You are viewing a plain text version of this content. The canonical link for it is here.
Posted to issues@iceberg.apache.org by "talatuyarer (via GitHub)" <gi...@apache.org> on 2025/04/16 06:11:18 UTC

[PR] Catalog: Add BigQuery Metastore Catalog Support [iceberg]

talatuyarer opened a new pull request, #12808:
URL: https://github.com/apache/iceberg/pull/12808

   This PR addresses comments from [PR's](https://github.com/apache/iceberg/pull/11039) that introduces initial support for using Google BigQuery as a Metastore Catalog for Apache Iceberg.  Unfortunately creator of the PR is on leave. I had to recreate a new PR for that.
   
   Key changes include:
   
   * Implementation of `BigQueryMetaStoreClient` and related classes to interact with the BigQuery API for Metastore operations.
   * Addition of `BigQueryMetastoreCatalog` to provide Iceberg catalog functionality backed by BigQuery.
   * Support for basic catalog operations such as creating/listing/deleting datasets (namespaces) and tables.
   * Integration with GCP credentials and BigQuery client setup.
   * Initial CatalogTests suite and infrastructure for BigQuery Metastore catalog.


-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org

For queries about this service, please contact Infrastructure at:
users@infra.apache.org


---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For additional commands, e-mail: issues-help@iceberg.apache.org


Re: [PR] Catalog: Add BigQuery Metastore Catalog Support [iceberg]

Posted by "talatuyarer (via GitHub)" <gi...@apache.org>.
talatuyarer commented on code in PR #12808:
URL: https://github.com/apache/iceberg/pull/12808#discussion_r2059700362


##########
bigquery/src/main/java/org/apache/iceberg/gcp/bigquery/BigQueryMetastoreCatalog.java:
##########
@@ -0,0 +1,400 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.iceberg.gcp.bigquery;
+
+import com.google.api.services.bigquery.model.Dataset;
+import com.google.api.services.bigquery.model.DatasetList.Datasets;
+import com.google.api.services.bigquery.model.DatasetReference;
+import com.google.api.services.bigquery.model.ExternalCatalogDatasetOptions;
+import com.google.api.services.bigquery.model.TableReference;
+import com.google.cloud.ServiceOptions;
+import com.google.cloud.bigquery.BigQueryOptions;
+import java.io.IOException;
+import java.security.GeneralSecurityException;
+import java.util.List;
+import java.util.Locale;
+import java.util.Map;
+import java.util.Objects;
+import java.util.Set;
+import org.apache.hadoop.conf.Configurable;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.iceberg.BaseMetastoreCatalog;
+import org.apache.iceberg.CatalogProperties;
+import org.apache.iceberg.CatalogUtil;
+import org.apache.iceberg.TableMetadata;
+import org.apache.iceberg.TableOperations;
+import org.apache.iceberg.catalog.Namespace;
+import org.apache.iceberg.catalog.SupportsNamespaces;
+import org.apache.iceberg.catalog.TableIdentifier;
+import org.apache.iceberg.exceptions.AlreadyExistsException;
+import org.apache.iceberg.exceptions.NoSuchNamespaceException;
+import org.apache.iceberg.exceptions.NoSuchTableException;
+import org.apache.iceberg.exceptions.ServiceFailureException;
+import org.apache.iceberg.exceptions.ValidationException;
+import org.apache.iceberg.io.FileIO;
+import org.apache.iceberg.relocated.com.google.common.annotations.VisibleForTesting;
+import org.apache.iceberg.relocated.com.google.common.base.Preconditions;
+import org.apache.iceberg.relocated.com.google.common.base.Strings;
+import org.apache.iceberg.relocated.com.google.common.collect.ImmutableList;
+import org.apache.iceberg.relocated.com.google.common.collect.ImmutableMap;
+import org.apache.iceberg.relocated.com.google.common.collect.Maps;
+import org.apache.iceberg.util.LocationUtil;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/** Iceberg Bigquery Metastore Catalog implementation. */
+public final class BigQueryMetastoreCatalog extends BaseMetastoreCatalog
+    implements SupportsNamespaces, Configurable {
+
+  // User provided properties.
+  public static final String PROJECT_ID = "gcp-project";
+  public static final String GCP_LOCATION = "gcp-location";
+  public static final String FILTER_UNSUPPORTED_TABLES = "filter-unsupported-tables";
+  public static final String TESTING_ENABLED = "testing-enabled";
+
+  public static final String HIVE_METASTORE_WAREHOUSE_DIR = "hive.metastore.warehouse.dir";
+
+  private static final Logger LOG = LoggerFactory.getLogger(BigQueryMetastoreCatalog.class);
+
+  private static final String DEFAULT_GCP_LOCATION = "us";
+
+  private String catalogName;
+  private Map<String, String> catalogProperties;
+  private FileIO fileIO;
+  private Configuration conf;
+  private String projectId;
+  private String location;
+  private BigQueryMetastoreClient client;
+  private boolean filterUnsupportedTables;
+
+  // Must have a no-arg constructor to be dynamically loaded
+  // initialize(String name, Map<String, String> properties) will be called to complete
+  // initialization
+  public BigQueryMetastoreCatalog() {}
+
+  @Override
+  public void initialize(String inputName, Map<String, String> properties) {
+    Preconditions.checkArgument(
+        properties.containsKey(PROJECT_ID), "GCP project must be specified");
+
+    projectId = properties.get(PROJECT_ID);
+    location = properties.getOrDefault(GCP_LOCATION, DEFAULT_GCP_LOCATION);
+    boolean testingEnabled =
+        Boolean.parseBoolean(properties.getOrDefault(TESTING_ENABLED, "false"));
+
+    BigQueryOptions options =
+        BigQueryOptions.newBuilder()
+            .setProjectId(projectId)
+            .setLocation(location)
+            .setRetrySettings(ServiceOptions.getDefaultRetrySettings())
+            .build();
+
+    try {
+      if (testingEnabled) {
+        client = new FakeBigQueryMetastoreClient(options);
+
+      } else {
+        client = new BigQueryMetastoreClientImpl(options);
+      }
+    } catch (IOException e) {
+      throw new ServiceFailureException(e, "Creating BigQuery client failed");
+    } catch (GeneralSecurityException e) {
+      throw new ValidationException(e, "Creating BigQuery client failed due to a security issue");
+    }
+    initialize(inputName, properties, projectId, location, client);
+  }
+
+  @VisibleForTesting
+  void initialize(
+      String inputName,
+      Map<String, String> properties,
+      String initialProjectId,
+      String initialLocation,
+      BigQueryMetastoreClient bigQueryMetaStoreClient) {
+    Preconditions.checkArgument(bigQueryMetaStoreClient != null, "BigQuery client can not be null");
+    this.catalogName = inputName;
+    this.catalogProperties = ImmutableMap.copyOf(properties);
+    this.projectId = initialProjectId;
+    this.location = initialLocation;
+    this.client = bigQueryMetaStoreClient;
+
+    if (this.conf == null) {
+      LOG.warn("No configuration was set, using the default environment Configuration");
+      this.conf = new Configuration();
+    }
+
+    LOG.info("Using BigQuery Metastore Iceberg Catalog: {}", inputName);
+
+    if (properties.containsKey(CatalogProperties.WAREHOUSE_LOCATION)) {
+      // Iceberg always removes trailing slash to avoid paths like "<folder>//data" in file systems
+      // like s3.
+      this.conf.set(
+          HIVE_METASTORE_WAREHOUSE_DIR,
+          LocationUtil.stripTrailingSlash(properties.get(CatalogProperties.WAREHOUSE_LOCATION)));
+    }
+
+    String fileIoImpl =
+        properties.getOrDefault(
+            CatalogProperties.FILE_IO_IMPL, "org.apache.iceberg.hadoop.HadoopFileIO");
+    this.fileIO = CatalogUtil.loadFileIO(fileIoImpl, properties, conf);
+
+    this.filterUnsupportedTables =
+        Boolean.parseBoolean(properties.getOrDefault(FILTER_UNSUPPORTED_TABLES, "false"));
+  }
+
+  @Override
+  protected TableOperations newTableOps(TableIdentifier identifier) {
+    return new BigQueryTableOperations(
+        client,
+        fileIO,
+        projectId,
+        // Sometimes extensions have the namespace contain the table name too, so we are forced to
+        // allow invalid namespace and just take the first part here like other catalog
+        // implementations do.
+        identifier.namespace().level(0),
+        identifier.name(),
+        conf);
+  }
+
+  @Override
+  protected String defaultWarehouseLocation(TableIdentifier identifier) {
+    String locationUri = null;
+    DatasetReference datasetReference = toDatasetReference(identifier.namespace());
+    Dataset dataset = client.getDataset(datasetReference);
+    if (dataset != null && dataset.getExternalCatalogDatasetOptions() != null) {
+      locationUri = dataset.getExternalCatalogDatasetOptions().getDefaultStorageLocationUri();
+    }
+    return String.format(

Review Comment:
   I updated where I see this issue



-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org

For queries about this service, please contact Infrastructure at:
users@infra.apache.org


---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For additional commands, e-mail: issues-help@iceberg.apache.org


Re: [PR] Catalog: Add BigQuery Metastore Catalog Support [iceberg]

Posted by "talatuyarer (via GitHub)" <gi...@apache.org>.
talatuyarer commented on code in PR #12808:
URL: https://github.com/apache/iceberg/pull/12808#discussion_r2059823154


##########
bigquery/src/main/java/org/apache/iceberg/gcp/bigquery/BigQueryMetastoreCatalog.java:
##########
@@ -0,0 +1,402 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.iceberg.gcp.bigquery;
+
+import com.google.api.services.bigquery.model.Dataset;
+import com.google.api.services.bigquery.model.DatasetList.Datasets;
+import com.google.api.services.bigquery.model.DatasetReference;
+import com.google.api.services.bigquery.model.ExternalCatalogDatasetOptions;
+import com.google.api.services.bigquery.model.TableReference;
+import com.google.cloud.ServiceOptions;
+import com.google.cloud.bigquery.BigQueryOptions;
+import java.io.IOException;
+import java.security.GeneralSecurityException;
+import java.util.List;
+import java.util.Locale;
+import java.util.Map;
+import java.util.Objects;
+import java.util.Set;
+import org.apache.hadoop.conf.Configurable;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.iceberg.BaseMetastoreCatalog;
+import org.apache.iceberg.CatalogProperties;
+import org.apache.iceberg.CatalogUtil;
+import org.apache.iceberg.TableMetadata;
+import org.apache.iceberg.TableOperations;
+import org.apache.iceberg.catalog.Namespace;
+import org.apache.iceberg.catalog.SupportsNamespaces;
+import org.apache.iceberg.catalog.TableIdentifier;
+import org.apache.iceberg.exceptions.AlreadyExistsException;
+import org.apache.iceberg.exceptions.NoSuchNamespaceException;
+import org.apache.iceberg.exceptions.NoSuchTableException;
+import org.apache.iceberg.exceptions.ServiceFailureException;
+import org.apache.iceberg.exceptions.ValidationException;
+import org.apache.iceberg.io.FileIO;
+import org.apache.iceberg.relocated.com.google.common.annotations.VisibleForTesting;
+import org.apache.iceberg.relocated.com.google.common.base.Preconditions;
+import org.apache.iceberg.relocated.com.google.common.base.Strings;
+import org.apache.iceberg.relocated.com.google.common.collect.ImmutableList;
+import org.apache.iceberg.relocated.com.google.common.collect.ImmutableMap;
+import org.apache.iceberg.relocated.com.google.common.collect.Maps;
+import org.apache.iceberg.util.LocationUtil;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/** Iceberg Bigquery Metastore Catalog implementation. */
+public final class BigQueryMetastoreCatalog extends BaseMetastoreCatalog
+    implements SupportsNamespaces, Configurable {
+
+  // User provided properties.
+  public static final String PROPERTIES_KEY_GCP_PROJECT = "gcp-project";
+  public static final String PROPERTIES_KEY_GCP_LOCATION = "gcp-location";
+  public static final String PROPERTIES_KEY_FILTER_UNSUPPORTED_TABLES = "filter-unsupported-tables";
+  public static final String PROPERTIES_KEY_TESTING_ENABLED = "testing-enabled";

Review Comment:
   Removed



-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org

For queries about this service, please contact Infrastructure at:
users@infra.apache.org


---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For additional commands, e-mail: issues-help@iceberg.apache.org


Re: [PR] Catalog: Add BigQuery Metastore Catalog Support [iceberg]

Posted by "talatuyarer (via GitHub)" <gi...@apache.org>.
talatuyarer commented on code in PR #12808:
URL: https://github.com/apache/iceberg/pull/12808#discussion_r2059721091


##########
bigquery/src/test/java/org/apache/iceberg/gcp/bigquery/TestBigQueryCatalog.java:
##########
@@ -0,0 +1,673 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.iceberg.gcp.bigquery;
+
+import static org.apache.iceberg.CatalogUtil.ICEBERG_CATALOG_TYPE;
+import static org.apache.iceberg.CatalogUtil.ICEBERG_CATALOG_TYPE_BIGQUERY;
+import static org.apache.iceberg.gcp.bigquery.BigQueryMetastoreCatalog.PROJECT_ID;
+import static org.apache.iceberg.gcp.bigquery.BigQueryMetastoreCatalog.TESTING_ENABLED;
+import static org.apache.iceberg.types.Types.NestedField.optional;
+import static org.assertj.core.api.Assertions.assertThat;
+import static org.assertj.core.api.Assertions.assertThatThrownBy;
+
+import java.io.File;
+import java.util.List;
+import java.util.Map;
+import org.apache.iceberg.CatalogProperties;
+import org.apache.iceberg.CatalogUtil;
+import org.apache.iceberg.PartitionSpec;
+import org.apache.iceberg.Schema;
+import org.apache.iceberg.SortOrder;
+import org.apache.iceberg.Table;
+import org.apache.iceberg.TableUtil;
+import org.apache.iceberg.Transaction;
+import org.apache.iceberg.catalog.CatalogTests;
+import org.apache.iceberg.catalog.Namespace;
+import org.apache.iceberg.catalog.TableIdentifier;
+import org.apache.iceberg.exceptions.NoSuchNamespaceException;
+import org.apache.iceberg.exceptions.NoSuchTableException;
+import org.apache.iceberg.expressions.Expressions;
+import org.apache.iceberg.relocated.com.google.common.collect.ImmutableMap;
+import org.apache.iceberg.types.Types;
+import org.junit.jupiter.api.AfterEach;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Disabled;
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.io.TempDir;
+import org.junit.jupiter.params.ParameterizedTest;
+import org.junit.jupiter.params.provider.ValueSource;
+
+public class TestBigQueryCatalog extends CatalogTests<BigQueryMetastoreCatalog> {
+
+  private static final String GCP_PROJECT = "project-id";
+  private static final String CATALOG_ID = "catalog-name";
+
+  // another schema that is not the same
+  protected static final Schema OTHER_SCHEMA_OPTIONAL =
+      new Schema(optional(1, "some_id", Types.IntegerType.get()));
+
+  // Schema passed to create tables
+  protected static final Schema SCHEMA_OPTIONAL =
+      new Schema(
+          optional(3, "id", Types.IntegerType.get(), "unique ID 🤪"),
+          optional(4, "data", Types.StringType.get()));
+
+  // This is the actual schema for the table, with column IDs reassigned
+  protected static final Schema REPLACE_SCHEMA_OPTIONAL =
+      new Schema(
+          optional(2, "id", Types.IntegerType.get(), "unique ID 🤪"),
+          optional(3, "data", Types.StringType.get()));
+
+  protected static final PartitionSpec REPLACE_SPEC_OPTIONAL =
+      PartitionSpec.builderFor(REPLACE_SCHEMA_OPTIONAL).bucket("id", 16).withSpecId(1).build();
+
+  protected static final SortOrder REPLACE_WRITE_ORDER_OPTIONAL =
+      SortOrder.builderFor(REPLACE_SCHEMA_OPTIONAL)
+          .asc(Expressions.bucket("id", 16))
+          .asc("id")
+          .build();
+
+  @TempDir private File tempFolder;
+  private BigQueryMetastoreCatalog catalog;
+  private BigQueryMetastoreClient fakeBigQueryMetastoreClient;
+  private String warehouseLocation;
+
+  @BeforeEach
+  public void before() throws Exception {
+    catalog = initCatalog(CATALOG_ID, ImmutableMap.of());
+  }
+
+  @AfterEach
+  public void after() throws Exception {

Review Comment:
   Thank you for letting me know. Now it is fixed. 



-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org

For queries about this service, please contact Infrastructure at:
users@infra.apache.org


---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For additional commands, e-mail: issues-help@iceberg.apache.org


Re: [PR] Catalog: Add BigQuery Metastore Catalog Support [iceberg]

Posted by "nastra (via GitHub)" <gi...@apache.org>.
nastra closed pull request #12808: Catalog: Add BigQuery Metastore Catalog Support
URL: https://github.com/apache/iceberg/pull/12808


-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org

For queries about this service, please contact Infrastructure at:
users@infra.apache.org


---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For additional commands, e-mail: issues-help@iceberg.apache.org


Re: [PR] Catalog: Add BigQuery Metastore Catalog Support [iceberg]

Posted by "nastra (via GitHub)" <gi...@apache.org>.
nastra commented on code in PR #12808:
URL: https://github.com/apache/iceberg/pull/12808#discussion_r2063968935


##########
bigquery/src/main/java/org/apache/iceberg/gcp/bigquery/BigQueryMetastoreCatalog.java:
##########
@@ -0,0 +1,394 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.iceberg.gcp.bigquery;
+
+import com.google.api.services.bigquery.model.Dataset;
+import com.google.api.services.bigquery.model.DatasetList.Datasets;
+import com.google.api.services.bigquery.model.DatasetReference;
+import com.google.api.services.bigquery.model.ExternalCatalogDatasetOptions;
+import com.google.api.services.bigquery.model.TableReference;
+import com.google.cloud.ServiceOptions;
+import com.google.cloud.bigquery.BigQueryOptions;
+import java.io.IOException;
+import java.security.GeneralSecurityException;
+import java.util.List;
+import java.util.Locale;
+import java.util.Map;
+import java.util.Objects;
+import java.util.Set;
+import org.apache.hadoop.conf.Configurable;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.iceberg.BaseMetastoreCatalog;
+import org.apache.iceberg.CatalogProperties;
+import org.apache.iceberg.CatalogUtil;
+import org.apache.iceberg.TableMetadata;
+import org.apache.iceberg.TableOperations;
+import org.apache.iceberg.catalog.Namespace;
+import org.apache.iceberg.catalog.SupportsNamespaces;
+import org.apache.iceberg.catalog.TableIdentifier;
+import org.apache.iceberg.exceptions.AlreadyExistsException;
+import org.apache.iceberg.exceptions.NoSuchNamespaceException;
+import org.apache.iceberg.exceptions.NoSuchTableException;
+import org.apache.iceberg.exceptions.ServiceFailureException;
+import org.apache.iceberg.exceptions.ValidationException;
+import org.apache.iceberg.io.FileIO;
+import org.apache.iceberg.relocated.com.google.common.annotations.VisibleForTesting;
+import org.apache.iceberg.relocated.com.google.common.base.Preconditions;
+import org.apache.iceberg.relocated.com.google.common.base.Strings;
+import org.apache.iceberg.relocated.com.google.common.collect.ImmutableList;
+import org.apache.iceberg.relocated.com.google.common.collect.ImmutableMap;
+import org.apache.iceberg.relocated.com.google.common.collect.Maps;
+import org.apache.iceberg.util.LocationUtil;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/** Iceberg Bigquery Metastore Catalog implementation. */
+public class BigQueryMetastoreCatalog extends BaseMetastoreCatalog
+    implements SupportsNamespaces, Configurable {
+
+  // User provided properties.
+  public static final String PROJECT_ID = "gcp-project";
+  public static final String GCP_LOCATION = "gcp-location";
+  public static final String LIST_ALL_TABLES = "list-all-tables";
+
+  public static final String HIVE_METASTORE_WAREHOUSE_DIR = "hive.metastore.warehouse.dir";
+
+  private static final Logger LOG = LoggerFactory.getLogger(BigQueryMetastoreCatalog.class);
+
+  private static final String DEFAULT_GCP_LOCATION = "us";
+
+  private String catalogName;
+  private Map<String, String> catalogProperties;
+  private FileIO fileIO;
+  private Configuration conf;
+  private String projectId;
+  private String location;
+  private BigQueryMetastoreClient client;
+  private boolean listAllTables;
+
+  // Must have a no-arg constructor to be dynamically loaded
+  // initialize(String name, Map<String, String> properties) will be called to complete
+  // initialization
+  public BigQueryMetastoreCatalog() {}
+
+  @Override
+  public void initialize(String inputName, Map<String, String> properties) {
+    Preconditions.checkArgument(
+        properties.containsKey(PROJECT_ID), "GCP project must be specified");
+
+    projectId = properties.get(PROJECT_ID);
+    location = properties.getOrDefault(GCP_LOCATION, DEFAULT_GCP_LOCATION);
+
+    BigQueryOptions options =
+        BigQueryOptions.newBuilder()
+            .setProjectId(projectId)
+            .setLocation(location)
+            .setRetrySettings(ServiceOptions.getDefaultRetrySettings())
+            .build();
+
+    try {
+      client = new BigQueryMetastoreClientImpl(options);
+    } catch (IOException e) {
+      throw new ServiceFailureException(e, "Creating BigQuery client failed");
+    } catch (GeneralSecurityException e) {
+      throw new ValidationException(e, "Creating BigQuery client failed due to a security issue");
+    }
+
+    initialize(inputName, properties, projectId, location, client);
+  }
+
+  @VisibleForTesting
+  void initialize(
+      String inputName,
+      Map<String, String> properties,
+      String initialProjectId,
+      String initialLocation,
+      BigQueryMetastoreClient bigQueryMetaStoreClient) {
+    Preconditions.checkArgument(bigQueryMetaStoreClient != null, "BigQuery client can not be null");
+    this.catalogName = inputName;
+    this.catalogProperties = ImmutableMap.copyOf(properties);
+    this.projectId = initialProjectId;
+    this.location = initialLocation;
+    this.client = bigQueryMetaStoreClient;
+
+    if (this.conf == null) {
+      LOG.warn("No configuration was set, using the default environment Configuration");
+      this.conf = new Configuration();
+    }
+
+    LOG.info("Using BigQuery Metastore Iceberg Catalog: {}", inputName);
+
+    if (properties.containsKey(CatalogProperties.WAREHOUSE_LOCATION)) {
+      this.conf.set(
+          HIVE_METASTORE_WAREHOUSE_DIR,
+          LocationUtil.stripTrailingSlash(properties.get(CatalogProperties.WAREHOUSE_LOCATION)));
+    }
+
+    String fileIoImpl =
+        properties.getOrDefault(
+            CatalogProperties.FILE_IO_IMPL, "org.apache.iceberg.hadoop.HadoopFileIO");
+    this.fileIO = CatalogUtil.loadFileIO(fileIoImpl, properties, conf);
+
+    this.listAllTables = Boolean.parseBoolean(properties.getOrDefault(LIST_ALL_TABLES, "true"));
+  }
+
+  @Override
+  protected TableOperations newTableOps(TableIdentifier identifier) {
+    return new BigQueryTableOperations(client, fileIO, toTableReference(identifier), conf);
+  }
+
+  @Override
+  protected String defaultWarehouseLocation(TableIdentifier identifier) {
+    String locationUri = null;
+    DatasetReference datasetReference = toDatasetReference(identifier.namespace());
+    Dataset dataset = client.load(datasetReference);
+    if (dataset != null && dataset.getExternalCatalogDatasetOptions() != null) {
+      locationUri = dataset.getExternalCatalogDatasetOptions().getDefaultStorageLocationUri();
+    }
+
+    return String.format(
+        "%s/%s",
+        Strings.isNullOrEmpty(locationUri)
+            ? createDefaultStorageLocationUri(datasetReference.getDatasetId())
+            : LocationUtil.stripTrailingSlash(locationUri),
+        identifier.name());
+  }
+
+  @Override
+  public List<TableIdentifier> listTables(Namespace namespace) {
+    validateNamespace(namespace);
+
+    return client.list(toDatasetReference(namespace), listAllTables).stream()
+        .map(
+            table -> TableIdentifier.of(namespace.level(0), table.getTableReference().getTableId()))
+        .collect(ImmutableList.toImmutableList());
+  }
+
+  @Override
+  public boolean dropTable(TableIdentifier identifier, boolean purge) {
+    try {
+      TableOperations ops = newTableOps(identifier);
+      TableMetadata lastMetadata = ops.current();
+
+      client.delete(toTableReference(identifier));
+
+      if (purge && lastMetadata != null) {
+        CatalogUtil.dropTableData(ops.io(), lastMetadata);
+      }
+    } catch (NoSuchTableException e) {
+      return false;
+    }
+
+    return true;
+  }
+
+  @Override
+  public void renameTable(TableIdentifier from, TableIdentifier to) {
+    validateNamespace(from.namespace());
+    validateNamespace(to.namespace());
+
+    if (!namespaceExists(to.namespace())) {
+      throw new NoSuchNamespaceException("Namespace does not exist: %s", to.namespace());
+    }
+
+    if (!namespaceExists(from.namespace())) {
+      throw new NoSuchNamespaceException("Namespace does not exist: %s", from.namespace());
+    }
+
+    // Check if the source table exists

Review Comment:
   nit: I would remove this comment, since it's obvious from the code



-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org

For queries about this service, please contact Infrastructure at:
users@infra.apache.org


---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For additional commands, e-mail: issues-help@iceberg.apache.org


Re: [PR] Catalog: Add BigQuery Metastore Catalog Support [iceberg]

Posted by "nastra (via GitHub)" <gi...@apache.org>.
nastra commented on code in PR #12808:
URL: https://github.com/apache/iceberg/pull/12808#discussion_r2063963749


##########
bigquery/src/main/java/org/apache/iceberg/gcp/bigquery/BigQueryMetastoreCatalog.java:
##########
@@ -0,0 +1,394 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.iceberg.gcp.bigquery;
+
+import com.google.api.services.bigquery.model.Dataset;
+import com.google.api.services.bigquery.model.DatasetList.Datasets;
+import com.google.api.services.bigquery.model.DatasetReference;
+import com.google.api.services.bigquery.model.ExternalCatalogDatasetOptions;
+import com.google.api.services.bigquery.model.TableReference;
+import com.google.cloud.ServiceOptions;
+import com.google.cloud.bigquery.BigQueryOptions;
+import java.io.IOException;
+import java.security.GeneralSecurityException;
+import java.util.List;
+import java.util.Locale;
+import java.util.Map;
+import java.util.Objects;
+import java.util.Set;
+import org.apache.hadoop.conf.Configurable;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.iceberg.BaseMetastoreCatalog;
+import org.apache.iceberg.CatalogProperties;
+import org.apache.iceberg.CatalogUtil;
+import org.apache.iceberg.TableMetadata;
+import org.apache.iceberg.TableOperations;
+import org.apache.iceberg.catalog.Namespace;
+import org.apache.iceberg.catalog.SupportsNamespaces;
+import org.apache.iceberg.catalog.TableIdentifier;
+import org.apache.iceberg.exceptions.AlreadyExistsException;
+import org.apache.iceberg.exceptions.NoSuchNamespaceException;
+import org.apache.iceberg.exceptions.NoSuchTableException;
+import org.apache.iceberg.exceptions.ServiceFailureException;
+import org.apache.iceberg.exceptions.ValidationException;
+import org.apache.iceberg.io.FileIO;
+import org.apache.iceberg.relocated.com.google.common.annotations.VisibleForTesting;
+import org.apache.iceberg.relocated.com.google.common.base.Preconditions;
+import org.apache.iceberg.relocated.com.google.common.base.Strings;
+import org.apache.iceberg.relocated.com.google.common.collect.ImmutableList;
+import org.apache.iceberg.relocated.com.google.common.collect.ImmutableMap;
+import org.apache.iceberg.relocated.com.google.common.collect.Maps;
+import org.apache.iceberg.util.LocationUtil;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/** Iceberg Bigquery Metastore Catalog implementation. */
+public class BigQueryMetastoreCatalog extends BaseMetastoreCatalog
+    implements SupportsNamespaces, Configurable {
+
+  // User provided properties.
+  public static final String PROJECT_ID = "gcp-project";
+  public static final String GCP_LOCATION = "gcp-location";
+  public static final String LIST_ALL_TABLES = "list-all-tables";
+
+  public static final String HIVE_METASTORE_WAREHOUSE_DIR = "hive.metastore.warehouse.dir";
+
+  private static final Logger LOG = LoggerFactory.getLogger(BigQueryMetastoreCatalog.class);
+
+  private static final String DEFAULT_GCP_LOCATION = "us";
+
+  private String catalogName;
+  private Map<String, String> catalogProperties;
+  private FileIO fileIO;
+  private Configuration conf;
+  private String projectId;
+  private String location;
+  private BigQueryMetastoreClient client;
+  private boolean listAllTables;
+
+  // Must have a no-arg constructor to be dynamically loaded
+  // initialize(String name, Map<String, String> properties) will be called to complete
+  // initialization
+  public BigQueryMetastoreCatalog() {}
+
+  @Override
+  public void initialize(String inputName, Map<String, String> properties) {
+    Preconditions.checkArgument(
+        properties.containsKey(PROJECT_ID), "GCP project must be specified");

Review Comment:
   ```suggestion
           properties.containsKey(PROJECT_ID), "Invalid GCP project: %s must be specified", PROJECT_ID);
   ```
   
   We try to be descriptive in the error messages and so ideally you want to mention the property that actually needs to be specified in this case



-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org

For queries about this service, please contact Infrastructure at:
users@infra.apache.org


---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For additional commands, e-mail: issues-help@iceberg.apache.org


Re: [PR] Catalog: Add BigQuery Metastore Catalog Support [iceberg]

Posted by "ebyhr (via GitHub)" <gi...@apache.org>.
ebyhr commented on code in PR #12808:
URL: https://github.com/apache/iceberg/pull/12808#discussion_r2048088148


##########
bigquery/src/main/java/org/apache/iceberg/gcp/bigquery/BigQueryMetaStoreClientImpl.java:
##########
@@ -0,0 +1,620 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.iceberg.gcp.bigquery;
+
+import com.google.api.client.googleapis.javanet.GoogleNetHttpTransport;
+import com.google.api.client.http.HttpHeaders;
+import com.google.api.client.http.HttpResponse;
+import com.google.api.client.http.HttpResponseException;
+import com.google.api.client.http.HttpStatusCodes;
+import com.google.api.client.json.gson.GsonFactory;
+import com.google.api.client.util.Data;
+import com.google.api.services.bigquery.Bigquery;
+import com.google.api.services.bigquery.BigqueryScopes;
+import com.google.api.services.bigquery.model.Dataset;
+import com.google.api.services.bigquery.model.DatasetList;
+import com.google.api.services.bigquery.model.DatasetList.Datasets;
+import com.google.api.services.bigquery.model.DatasetReference;
+import com.google.api.services.bigquery.model.ExternalCatalogDatasetOptions;
+import com.google.api.services.bigquery.model.ExternalCatalogTableOptions;
+import com.google.api.services.bigquery.model.Table;
+import com.google.api.services.bigquery.model.TableList;
+import com.google.api.services.bigquery.model.TableList.Tables;
+import com.google.api.services.bigquery.model.TableReference;
+import com.google.api.services.bigquery.model.TableSchema;
+import com.google.auth.http.HttpCredentialsAdapter;
+import com.google.auth.oauth2.GoogleCredentials;
+import com.google.cloud.BaseServiceException;
+import com.google.cloud.ExceptionHandler;
+import com.google.cloud.bigquery.BigQueryErrorMessages;
+import com.google.cloud.bigquery.BigQueryOptions;
+import com.google.cloud.bigquery.BigQueryRetryConfig;
+import com.google.cloud.bigquery.BigQueryRetryHelper;
+import java.io.IOException;
+import java.nio.charset.StandardCharsets;
+import java.security.GeneralSecurityException;
+import java.util.List;
+import java.util.Locale;
+import java.util.Map;
+import java.util.Set;
+import java.util.concurrent.Callable;
+import java.util.stream.Collectors;
+import java.util.stream.Stream;
+import org.apache.iceberg.BaseMetastoreTableOperations;
+import org.apache.iceberg.exceptions.AlreadyExistsException;
+import org.apache.iceberg.exceptions.BadRequestException;
+import org.apache.iceberg.exceptions.ForbiddenException;
+import org.apache.iceberg.exceptions.NoSuchIcebergTableException;
+import org.apache.iceberg.exceptions.NoSuchNamespaceException;
+import org.apache.iceberg.exceptions.NoSuchTableException;
+import org.apache.iceberg.exceptions.NotAuthorizedException;
+import org.apache.iceberg.exceptions.RuntimeIOException;
+import org.apache.iceberg.exceptions.ServiceFailureException;
+import org.apache.iceberg.exceptions.ServiceUnavailableException;
+import org.apache.iceberg.exceptions.ValidationException;
+import org.apache.iceberg.relocated.com.google.common.base.Preconditions;
+import org.apache.iceberg.relocated.com.google.common.collect.Lists;
+import org.apache.iceberg.relocated.com.google.common.collect.Maps;
+
+/** A client of Google Bigquery Metastore functions over the BigQuery service. */
+public final class BigQueryMetaStoreClientImpl implements BigQueryMetaStoreClient {
+
+  public static final ExceptionHandler.Interceptor EXCEPTION_HANDLER_INTERCEPTOR =
+      new ExceptionHandler.Interceptor() {
+
+        private static final long serialVersionUID = -8429573486870467828L;
+
+        @Override
+        public RetryResult afterEval(Exception exception, RetryResult retryResult) {
+          return ExceptionHandler.Interceptor.RetryResult.CONTINUE_EVALUATION;
+        }
+
+        @Override
+        public RetryResult beforeEval(Exception exception) {
+          if (exception instanceof BaseServiceException) {
+            boolean retriable = ((BaseServiceException) exception).isRetryable();
+            return retriable
+                ? ExceptionHandler.Interceptor.RetryResult.RETRY
+                : ExceptionHandler.Interceptor.RetryResult.CONTINUE_EVALUATION;
+          }
+          return ExceptionHandler.Interceptor.RetryResult.CONTINUE_EVALUATION;
+        }
+      };
+
+  private final Bigquery client;
+  private final BigQueryOptions bigqueryOptions;
+
+  // Retry config with error messages and regex for rate limit exceeded errors.
+  private static final BigQueryRetryConfig DEFAULT_RETRY_CONFIG =
+      BigQueryRetryConfig.newBuilder()
+          .retryOnMessage(BigQueryErrorMessages.RATE_LIMIT_EXCEEDED_MSG)
+          .retryOnMessage(BigQueryErrorMessages.JOB_RATE_LIMIT_EXCEEDED_MSG)
+          .retryOnRegEx(BigQueryErrorMessages.RetryRegExPatterns.RATE_LIMIT_EXCEEDED_REGEX)
+          .build();
+
+  public static final ExceptionHandler BIGQUERY_EXCEPTION_HANDLER =
+      ExceptionHandler.newBuilder()
+          .abortOn(RuntimeException.class)
+          .retryOn(java.net.ConnectException.class) // retry on Connection Exception
+          .retryOn(java.net.UnknownHostException.class) // retry on UnknownHostException

Review Comment:
   "retry on UnknownHostException" is obvious when reading the code. It would be nice to explain **_why_** we retry on such exceptions. 



##########
bigquery/src/main/java/org/apache/iceberg/gcp/bigquery/BigQueryMetaStoreClient.java:
##########
@@ -0,0 +1,129 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.iceberg.gcp.bigquery;
+
+import com.google.api.services.bigquery.model.Dataset;
+import com.google.api.services.bigquery.model.DatasetList.Datasets;
+import com.google.api.services.bigquery.model.DatasetReference;
+import com.google.api.services.bigquery.model.Table;
+import com.google.api.services.bigquery.model.TableList.Tables;
+import com.google.api.services.bigquery.model.TableReference;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+
+/**
+ * A client of Google BigQuery Metastore functions over the BigQuery service. Uses the Google
+ * BigQuery API.
+ */
+public interface BigQueryMetaStoreClient {

Review Comment:
   nit: BigQueryMetaStoreClient -> BigQueryMetastoreClient? 



##########
bigquery/src/test/java/org/apache/iceberg/gcp/bigquery/BigQueryMetastoreTestUtils.java:
##########
@@ -0,0 +1,86 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.iceberg.gcp.bigquery;
+
+import static org.apache.iceberg.types.Types.NestedField.required;
+import static org.assertj.core.api.Assertions.assertThat;
+
+import com.google.api.services.bigquery.model.ExternalCatalogTableOptions;
+import com.google.api.services.bigquery.model.StorageDescriptor;
+import com.google.api.services.bigquery.model.Table;
+import com.google.api.services.bigquery.model.TableReference;
+import java.io.File;
+import java.io.IOException;
+import java.util.Collections;
+import java.util.Optional;
+import org.apache.commons.io.FileUtils;
+import org.apache.commons.io.filefilter.TrueFileFilter;
+import org.apache.iceberg.Schema;
+import org.apache.iceberg.catalog.TableIdentifier;
+import org.apache.iceberg.types.Types;
+
+/** Test utility methods for BigQuery Metastore Iceberg catalog. */
+public final class BigQueryMetastoreTestUtils {
+  private BigQueryMetastoreTestUtils() {}
+
+  public static final String METADATA_LOCATION_PROP = "metadata_location";
+
+  public static Schema getTestSchema() {
+    return new Schema(
+        required(1, "id", Types.IntegerType.get(), "unique ID"),
+        required(2, "data", Types.StringType.get()));
+  }
+
+  public static Table createTestTable(
+      File tempFolder,
+      BigQueryMetastoreCatalog bigQueryMetastoreCatalog,
+      TableReference tableReference)
+      throws IOException {
+    Schema schema = getTestSchema();
+    TableIdentifier tableIdentifier =
+        TableIdentifier.of(tableReference.getDatasetId(), tableReference.getTableId());
+    String tableDir = tempFolder.toPath().resolve(tableReference.getTableId()).toString();
+
+    bigQueryMetastoreCatalog
+        .buildTable(tableIdentifier, schema)
+        .withLocation(tableDir)
+        .createTransaction()
+        .commitTransaction();
+
+    Optional<String> metadataLocation = getIcebergMetadataFilePath(tableDir);
+    assertThat(metadataLocation.isPresent());

Review Comment:
   ```suggestion
       assertThat(metadataLocation).isPresent();
   ```



##########
bigquery/src/main/java/org/apache/iceberg/gcp/bigquery/BigQueryMetaStoreClientImpl.java:
##########
@@ -0,0 +1,620 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.iceberg.gcp.bigquery;
+
+import com.google.api.client.googleapis.javanet.GoogleNetHttpTransport;
+import com.google.api.client.http.HttpHeaders;
+import com.google.api.client.http.HttpResponse;
+import com.google.api.client.http.HttpResponseException;
+import com.google.api.client.http.HttpStatusCodes;
+import com.google.api.client.json.gson.GsonFactory;
+import com.google.api.client.util.Data;
+import com.google.api.services.bigquery.Bigquery;
+import com.google.api.services.bigquery.BigqueryScopes;
+import com.google.api.services.bigquery.model.Dataset;
+import com.google.api.services.bigquery.model.DatasetList;
+import com.google.api.services.bigquery.model.DatasetList.Datasets;
+import com.google.api.services.bigquery.model.DatasetReference;
+import com.google.api.services.bigquery.model.ExternalCatalogDatasetOptions;
+import com.google.api.services.bigquery.model.ExternalCatalogTableOptions;
+import com.google.api.services.bigquery.model.Table;
+import com.google.api.services.bigquery.model.TableList;
+import com.google.api.services.bigquery.model.TableList.Tables;
+import com.google.api.services.bigquery.model.TableReference;
+import com.google.api.services.bigquery.model.TableSchema;
+import com.google.auth.http.HttpCredentialsAdapter;
+import com.google.auth.oauth2.GoogleCredentials;
+import com.google.cloud.BaseServiceException;
+import com.google.cloud.ExceptionHandler;
+import com.google.cloud.bigquery.BigQueryErrorMessages;
+import com.google.cloud.bigquery.BigQueryOptions;
+import com.google.cloud.bigquery.BigQueryRetryConfig;
+import com.google.cloud.bigquery.BigQueryRetryHelper;
+import java.io.IOException;
+import java.nio.charset.StandardCharsets;
+import java.security.GeneralSecurityException;
+import java.util.List;
+import java.util.Locale;
+import java.util.Map;
+import java.util.Set;
+import java.util.concurrent.Callable;
+import java.util.stream.Collectors;
+import java.util.stream.Stream;
+import org.apache.iceberg.BaseMetastoreTableOperations;
+import org.apache.iceberg.exceptions.AlreadyExistsException;
+import org.apache.iceberg.exceptions.BadRequestException;
+import org.apache.iceberg.exceptions.ForbiddenException;
+import org.apache.iceberg.exceptions.NoSuchIcebergTableException;
+import org.apache.iceberg.exceptions.NoSuchNamespaceException;
+import org.apache.iceberg.exceptions.NoSuchTableException;
+import org.apache.iceberg.exceptions.NotAuthorizedException;
+import org.apache.iceberg.exceptions.RuntimeIOException;
+import org.apache.iceberg.exceptions.ServiceFailureException;
+import org.apache.iceberg.exceptions.ServiceUnavailableException;
+import org.apache.iceberg.exceptions.ValidationException;
+import org.apache.iceberg.relocated.com.google.common.base.Preconditions;
+import org.apache.iceberg.relocated.com.google.common.collect.Lists;
+import org.apache.iceberg.relocated.com.google.common.collect.Maps;
+
+/** A client of Google Bigquery Metastore functions over the BigQuery service. */
+public final class BigQueryMetaStoreClientImpl implements BigQueryMetaStoreClient {
+
+  public static final ExceptionHandler.Interceptor EXCEPTION_HANDLER_INTERCEPTOR =
+      new ExceptionHandler.Interceptor() {
+
+        private static final long serialVersionUID = -8429573486870467828L;
+
+        @Override
+        public RetryResult afterEval(Exception exception, RetryResult retryResult) {
+          return ExceptionHandler.Interceptor.RetryResult.CONTINUE_EVALUATION;
+        }
+
+        @Override
+        public RetryResult beforeEval(Exception exception) {
+          if (exception instanceof BaseServiceException) {
+            boolean retriable = ((BaseServiceException) exception).isRetryable();
+            return retriable
+                ? ExceptionHandler.Interceptor.RetryResult.RETRY
+                : ExceptionHandler.Interceptor.RetryResult.CONTINUE_EVALUATION;
+          }
+          return ExceptionHandler.Interceptor.RetryResult.CONTINUE_EVALUATION;
+        }
+      };
+
+  private final Bigquery client;
+  private final BigQueryOptions bigqueryOptions;
+
+  // Retry config with error messages and regex for rate limit exceeded errors.
+  private static final BigQueryRetryConfig DEFAULT_RETRY_CONFIG =
+      BigQueryRetryConfig.newBuilder()
+          .retryOnMessage(BigQueryErrorMessages.RATE_LIMIT_EXCEEDED_MSG)
+          .retryOnMessage(BigQueryErrorMessages.JOB_RATE_LIMIT_EXCEEDED_MSG)
+          .retryOnRegEx(BigQueryErrorMessages.RetryRegExPatterns.RATE_LIMIT_EXCEEDED_REGEX)
+          .build();
+
+  public static final ExceptionHandler BIGQUERY_EXCEPTION_HANDLER =
+      ExceptionHandler.newBuilder()
+          .abortOn(RuntimeException.class)
+          .retryOn(java.net.ConnectException.class) // retry on Connection Exception
+          .retryOn(java.net.UnknownHostException.class) // retry on UnknownHostException
+          .addInterceptors(EXCEPTION_HANDLER_INTERCEPTOR)
+          .build();
+
+  /** Constructs a client of the Google BigQuery service. */
+  public BigQueryMetaStoreClientImpl(BigQueryOptions options)
+      throws IOException, GeneralSecurityException {
+    // Initialize client that will be used to send requests. This client only needs to be created
+    // once, and can be reused for multiple requests
+    HttpCredentialsAdapter httpCredentialsAdapter =
+        new HttpCredentialsAdapter(
+            GoogleCredentials.getApplicationDefault().createScoped(BigqueryScopes.all()));
+    this.client =
+        new Bigquery.Builder(
+                GoogleNetHttpTransport.newTrustedTransport(),
+                GsonFactory.getDefaultInstance(),
+                httpRequest -> {
+                  httpCredentialsAdapter.initialize(httpRequest);
+                  // Instead of throwing exceptions, analyze the HttpResponse object and inspect its
+                  // status code. This will allow BigQuery API errors to be converted into Iceberg
+                  // exceptions.
+                  httpRequest.setThrowExceptionOnExecuteError(false);
+                })
+            .setApplicationName("BigQuery Iceberg Catalog Plugin")
+            .build();
+    this.bigqueryOptions = options;
+  }
+
+  @Override
+  public Dataset createDataset(Dataset dataset) {
+    // TODO (b/399885863): Ensure Dataset creation is idempotent when handling retries.
+    Dataset response = null;
+    try {
+      response =
+          BigQueryRetryHelper.runWithRetries(
+              new Callable<Dataset>() {
+                @Override
+                public Dataset call() {
+                  return create(dataset);
+                }
+              },
+              bigqueryOptions.getRetrySettings(),
+              BIGQUERY_EXCEPTION_HANDLER,
+              bigqueryOptions.getClock(),
+              DEFAULT_RETRY_CONFIG);
+    } catch (BigQueryRetryHelper.BigQueryRetryHelperException e) {
+      handleBigQueryRetryException(e);
+    }
+    return response;
+  }
+
+  @SuppressWarnings("FormatStringAnnotation")
+  private Dataset create(Dataset dataset) {
+    try {
+      HttpResponse response =
+          client
+              .datasets()
+              .insert(dataset.getDatasetReference().getProjectId(), dataset)
+              .executeUnparsed();
+      return convertExceptionIfUnsuccessful(response).parseAs(Dataset.class);
+    } catch (IOException e) {
+      throw new RuntimeIOException(e);
+    } catch (AlreadyExistsException e) {
+      throw new AlreadyExistsException("Namespace already exists: " + dataset.getId());
+    }
+  }
+
+  @Override
+  @SuppressWarnings("FormatStringAnnotation")
+  public Dataset getDataset(DatasetReference datasetReference) {
+    try {
+      HttpResponse response =
+          client
+              .datasets()
+              .get(datasetReference.getProjectId(), datasetReference.getDatasetId())
+              .executeUnparsed();
+      if (response.getStatusCode() == HttpStatusCodes.STATUS_CODE_NOT_FOUND) {
+        throw new NoSuchNamespaceException(
+            "Namespace does not exist: " + datasetReference.getDatasetId());
+      }
+      return convertExceptionIfUnsuccessful(response).parseAs(Dataset.class);
+    } catch (IOException e) {
+      throw new RuntimeIOException(e);
+    }
+  }
+
+  @Override
+  @SuppressWarnings("FormatStringAnnotation")
+  public void deleteDataset(DatasetReference datasetReference) {
+    try {
+      HttpResponse response =
+          client
+              .datasets()
+              .delete(datasetReference.getProjectId(), datasetReference.getDatasetId())
+              .executeUnparsed();
+      if (response.getStatusCode() == HttpStatusCodes.STATUS_CODE_NOT_FOUND) {
+        throw new NoSuchNamespaceException(
+            "Namespace does not exist: " + datasetReference.getDatasetId());
+      }
+      convertExceptionIfUnsuccessful(response);
+    } catch (IOException e) {
+      throw new RuntimeIOException(e);
+    }
+  }
+
+  @Override
+  public Dataset setDatasetParameters(
+      DatasetReference datasetReference, Map<String, String> parameters) {
+    Dataset dataset = getDataset(datasetReference);
+    ExternalCatalogDatasetOptions externalCatalogDatasetOptions =
+        dataset.getExternalCatalogDatasetOptions() == null
+            ? new ExternalCatalogDatasetOptions()
+            : dataset.getExternalCatalogDatasetOptions();
+    Map<String, String> finalParameters =
+        externalCatalogDatasetOptions.getParameters() == null
+            ? Maps.newHashMap()
+            : externalCatalogDatasetOptions.getParameters();
+    finalParameters.putAll(parameters);
+
+    dataset.setExternalCatalogDatasetOptions(
+        externalCatalogDatasetOptions.setParameters(finalParameters));
+
+    return updateDataset(dataset);
+  }
+
+  @Override
+  public Dataset removeDatasetParameters(
+      DatasetReference datasetReference, Set<String> parameters) {
+    Dataset dataset = getDataset(datasetReference);
+    ExternalCatalogDatasetOptions externalCatalogDatasetOptions =
+        dataset.getExternalCatalogDatasetOptions() == null
+            ? new ExternalCatalogDatasetOptions()
+            : dataset.getExternalCatalogDatasetOptions();
+    Map<String, String> finalParameters =
+        externalCatalogDatasetOptions.getParameters() == null
+            ? Maps.newHashMap()
+            : externalCatalogDatasetOptions.getParameters();
+    parameters.forEach(finalParameters::remove);
+
+    dataset.setExternalCatalogDatasetOptions(
+        externalCatalogDatasetOptions.setParameters(finalParameters));
+
+    return updateDataset(dataset);
+  }
+
+  @Override
+  public List<Datasets> listDatasets(String projectId) {
+    try {
+      String nextPageToken = null;
+      List<Datasets> datasets = Lists.newArrayList();
+      do {
+        HttpResponse pageResponse =
+            client.datasets().list(projectId).setPageToken(nextPageToken).executeUnparsed();
+        DatasetList result =
+            convertExceptionIfUnsuccessful(pageResponse).parseAs(DatasetList.class);
+        nextPageToken = result.getNextPageToken();
+        if (result.getDatasets() != null) {
+          datasets.addAll(result.getDatasets());
+        }
+      } while (nextPageToken != null && !nextPageToken.isEmpty());
+      return datasets;
+    } catch (IOException e) {
+      throw new RuntimeIOException(e);
+    }
+  }
+
+  @Override
+  public Table createTable(Table table) {
+    // Ensure it is an Iceberg table supported by the BQ metastore catalog.
+    validateTable(table);
+    // TODO (b/399885863): Ensure table creation is idempotent when handling retries.

Review Comment:
   This repository isn't only for Google engineers :)



##########
bigquery/src/main/java/org/apache/iceberg/gcp/bigquery/BigQueryMetaStoreClientImpl.java:
##########
@@ -0,0 +1,620 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.iceberg.gcp.bigquery;
+
+import com.google.api.client.googleapis.javanet.GoogleNetHttpTransport;
+import com.google.api.client.http.HttpHeaders;
+import com.google.api.client.http.HttpResponse;
+import com.google.api.client.http.HttpResponseException;
+import com.google.api.client.http.HttpStatusCodes;
+import com.google.api.client.json.gson.GsonFactory;
+import com.google.api.client.util.Data;
+import com.google.api.services.bigquery.Bigquery;
+import com.google.api.services.bigquery.BigqueryScopes;
+import com.google.api.services.bigquery.model.Dataset;
+import com.google.api.services.bigquery.model.DatasetList;
+import com.google.api.services.bigquery.model.DatasetList.Datasets;
+import com.google.api.services.bigquery.model.DatasetReference;
+import com.google.api.services.bigquery.model.ExternalCatalogDatasetOptions;
+import com.google.api.services.bigquery.model.ExternalCatalogTableOptions;
+import com.google.api.services.bigquery.model.Table;
+import com.google.api.services.bigquery.model.TableList;
+import com.google.api.services.bigquery.model.TableList.Tables;
+import com.google.api.services.bigquery.model.TableReference;
+import com.google.api.services.bigquery.model.TableSchema;
+import com.google.auth.http.HttpCredentialsAdapter;
+import com.google.auth.oauth2.GoogleCredentials;
+import com.google.cloud.BaseServiceException;
+import com.google.cloud.ExceptionHandler;
+import com.google.cloud.bigquery.BigQueryErrorMessages;
+import com.google.cloud.bigquery.BigQueryOptions;
+import com.google.cloud.bigquery.BigQueryRetryConfig;
+import com.google.cloud.bigquery.BigQueryRetryHelper;
+import java.io.IOException;
+import java.nio.charset.StandardCharsets;
+import java.security.GeneralSecurityException;
+import java.util.List;
+import java.util.Locale;
+import java.util.Map;
+import java.util.Set;
+import java.util.concurrent.Callable;
+import java.util.stream.Collectors;
+import java.util.stream.Stream;
+import org.apache.iceberg.BaseMetastoreTableOperations;
+import org.apache.iceberg.exceptions.AlreadyExistsException;
+import org.apache.iceberg.exceptions.BadRequestException;
+import org.apache.iceberg.exceptions.ForbiddenException;
+import org.apache.iceberg.exceptions.NoSuchIcebergTableException;
+import org.apache.iceberg.exceptions.NoSuchNamespaceException;
+import org.apache.iceberg.exceptions.NoSuchTableException;
+import org.apache.iceberg.exceptions.NotAuthorizedException;
+import org.apache.iceberg.exceptions.RuntimeIOException;
+import org.apache.iceberg.exceptions.ServiceFailureException;
+import org.apache.iceberg.exceptions.ServiceUnavailableException;
+import org.apache.iceberg.exceptions.ValidationException;
+import org.apache.iceberg.relocated.com.google.common.base.Preconditions;
+import org.apache.iceberg.relocated.com.google.common.collect.Lists;
+import org.apache.iceberg.relocated.com.google.common.collect.Maps;
+
+/** A client of Google Bigquery Metastore functions over the BigQuery service. */
+public final class BigQueryMetaStoreClientImpl implements BigQueryMetaStoreClient {
+
+  public static final ExceptionHandler.Interceptor EXCEPTION_HANDLER_INTERCEPTOR =
+      new ExceptionHandler.Interceptor() {
+
+        private static final long serialVersionUID = -8429573486870467828L;
+
+        @Override
+        public RetryResult afterEval(Exception exception, RetryResult retryResult) {
+          return ExceptionHandler.Interceptor.RetryResult.CONTINUE_EVALUATION;
+        }
+
+        @Override
+        public RetryResult beforeEval(Exception exception) {
+          if (exception instanceof BaseServiceException) {
+            boolean retriable = ((BaseServiceException) exception).isRetryable();
+            return retriable
+                ? ExceptionHandler.Interceptor.RetryResult.RETRY
+                : ExceptionHandler.Interceptor.RetryResult.CONTINUE_EVALUATION;
+          }
+          return ExceptionHandler.Interceptor.RetryResult.CONTINUE_EVALUATION;
+        }
+      };
+
+  private final Bigquery client;
+  private final BigQueryOptions bigqueryOptions;
+
+  // Retry config with error messages and regex for rate limit exceeded errors.
+  private static final BigQueryRetryConfig DEFAULT_RETRY_CONFIG =
+      BigQueryRetryConfig.newBuilder()
+          .retryOnMessage(BigQueryErrorMessages.RATE_LIMIT_EXCEEDED_MSG)
+          .retryOnMessage(BigQueryErrorMessages.JOB_RATE_LIMIT_EXCEEDED_MSG)
+          .retryOnRegEx(BigQueryErrorMessages.RetryRegExPatterns.RATE_LIMIT_EXCEEDED_REGEX)
+          .build();
+
+  public static final ExceptionHandler BIGQUERY_EXCEPTION_HANDLER =
+      ExceptionHandler.newBuilder()
+          .abortOn(RuntimeException.class)
+          .retryOn(java.net.ConnectException.class) // retry on Connection Exception
+          .retryOn(java.net.UnknownHostException.class) // retry on UnknownHostException
+          .addInterceptors(EXCEPTION_HANDLER_INTERCEPTOR)
+          .build();
+
+  /** Constructs a client of the Google BigQuery service. */
+  public BigQueryMetaStoreClientImpl(BigQueryOptions options)
+      throws IOException, GeneralSecurityException {
+    // Initialize client that will be used to send requests. This client only needs to be created
+    // once, and can be reused for multiple requests
+    HttpCredentialsAdapter httpCredentialsAdapter =
+        new HttpCredentialsAdapter(
+            GoogleCredentials.getApplicationDefault().createScoped(BigqueryScopes.all()));
+    this.client =
+        new Bigquery.Builder(
+                GoogleNetHttpTransport.newTrustedTransport(),
+                GsonFactory.getDefaultInstance(),
+                httpRequest -> {
+                  httpCredentialsAdapter.initialize(httpRequest);
+                  // Instead of throwing exceptions, analyze the HttpResponse object and inspect its
+                  // status code. This will allow BigQuery API errors to be converted into Iceberg
+                  // exceptions.
+                  httpRequest.setThrowExceptionOnExecuteError(false);
+                })
+            .setApplicationName("BigQuery Metastore Iceberg Catalog Plugin")
+            .build();
+    this.bigqueryOptions = options;
+  }
+
+  @Override
+  public Dataset createDataset(Dataset dataset) {
+    // TODO (b/399885863): Ensure Dataset creation is idempotent when handling retries.
+    Dataset response = null;
+    try {
+      response =
+          BigQueryRetryHelper.runWithRetries(
+              new Callable<Dataset>() {
+                @Override
+                public Dataset call() {
+                  return create(dataset);
+                }
+              },

Review Comment:
   We could replace these code with lambda `() -> create(dataset)`



##########
bigquery/src/main/java/org/apache/iceberg/gcp/bigquery/BigQueryMetaStoreClient.java:
##########
@@ -0,0 +1,129 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.iceberg.gcp.bigquery;
+
+import com.google.api.services.bigquery.model.Dataset;
+import com.google.api.services.bigquery.model.DatasetList.Datasets;
+import com.google.api.services.bigquery.model.DatasetReference;
+import com.google.api.services.bigquery.model.Table;
+import com.google.api.services.bigquery.model.TableList.Tables;
+import com.google.api.services.bigquery.model.TableReference;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+
+/**
+ * A client of Google BigQuery Metastore functions over the BigQuery service. Uses the Google
+ * BigQuery API.
+ */
+public interface BigQueryMetaStoreClient {
+
+  /**
+   * Creates and returns a new dataset.
+   *
+   * @param dataset the dataset to create
+   */
+  Dataset createDataset(Dataset dataset);
+
+  /**
+   * Returns a dataset.
+   *
+   * @param datasetReference full dataset reference
+   */
+  Dataset getDataset(DatasetReference datasetReference);
+
+  /**
+   * Deletes a dataset.
+   *
+   * @param datasetReference full dataset reference
+   */
+  void deleteDataset(DatasetReference datasetReference);
+
+  /**
+   * Updates parameters of a dataset or adds them if did not exist, leaving already-existing ones
+   * intact.
+   *
+   * @param datasetReference full dataset reference
+   * @param parameters metadata parameters to add
+   * @return dataset after patch
+   */
+  Dataset setDatasetParameters(DatasetReference datasetReference, Map<String, String> parameters);
+
+  /**
+   * Removes given set of keys of parameters of a dataset. Ignores keys that do not exist already.
+   *
+   * @param datasetReference full dataset reference
+   * @param parameters metadata parameter keys to remove
+   * @return dataset after patch
+   */
+  Dataset removeDatasetParameters(DatasetReference datasetReference, Set<String> parameters);
+
+  /**
+   * Lists datasets under a given project
+   *
+   * @param projectId the identifier of the project to list datasets under
+   */
+  List<Datasets> listDatasets(String projectId);
+
+  /**
+   * Creates and returns a new table.
+   *
+   * @param table body of the table to create
+   */
+  Table createTable(Table table);
+
+  /**
+   * Returns a table.
+   *
+   * @param tableReference full table reference
+   */
+  Table getTable(TableReference tableReference);
+
+  /**
+   * Updates the catalog table options of an Iceberg table and returns the updated table.
+   *
+   * @param tableReference full table reference
+   * @param table to patch
+   */
+  Table patchTable(TableReference tableReference, Table table);
+
+  /**
+   * Renames a table.
+   *
+   * @param tableToRename full table reference
+   * @param newTableId new table identifier
+   */
+  Table renameTable(TableReference tableToRename, String newTableId);

Review Comment:
   This method is unused. 



##########
bigquery/src/main/java/org/apache/iceberg/gcp/bigquery/FakeBigQueryMetaStoreClient.java:
##########
@@ -0,0 +1,228 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.iceberg.gcp.bigquery;
+
+import com.google.api.services.bigquery.model.Dataset;
+import com.google.api.services.bigquery.model.DatasetList;
+import com.google.api.services.bigquery.model.DatasetReference;
+import com.google.api.services.bigquery.model.ExternalCatalogDatasetOptions;
+import com.google.api.services.bigquery.model.Table;
+import com.google.api.services.bigquery.model.TableList;
+import com.google.api.services.bigquery.model.TableReference;
+import com.google.cloud.bigquery.BigQueryOptions;
+import java.io.IOException;
+import java.security.GeneralSecurityException;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.UUID;
+import java.util.stream.Collectors;
+import org.apache.iceberg.exceptions.AlreadyExistsException;
+import org.apache.iceberg.exceptions.CommitFailedException;
+import org.apache.iceberg.exceptions.NamespaceNotEmptyException;
+import org.apache.iceberg.exceptions.NoSuchNamespaceException;
+import org.apache.iceberg.exceptions.NoSuchTableException;
+import org.apache.iceberg.relocated.com.google.common.collect.Maps;
+
+public class FakeBigQueryMetaStoreClient implements BigQueryMetaStoreClient {

Review Comment:
   How about renaming to FakeBigQueryMetaStoreClient? 



##########
bigquery/src/main/java/org/apache/iceberg/gcp/bigquery/BigQueryMetaStoreClientImpl.java:
##########
@@ -0,0 +1,620 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.iceberg.gcp.bigquery;
+
+import com.google.api.client.googleapis.javanet.GoogleNetHttpTransport;
+import com.google.api.client.http.HttpHeaders;
+import com.google.api.client.http.HttpResponse;
+import com.google.api.client.http.HttpResponseException;
+import com.google.api.client.http.HttpStatusCodes;
+import com.google.api.client.json.gson.GsonFactory;
+import com.google.api.client.util.Data;
+import com.google.api.services.bigquery.Bigquery;
+import com.google.api.services.bigquery.BigqueryScopes;
+import com.google.api.services.bigquery.model.Dataset;
+import com.google.api.services.bigquery.model.DatasetList;
+import com.google.api.services.bigquery.model.DatasetList.Datasets;
+import com.google.api.services.bigquery.model.DatasetReference;
+import com.google.api.services.bigquery.model.ExternalCatalogDatasetOptions;
+import com.google.api.services.bigquery.model.ExternalCatalogTableOptions;
+import com.google.api.services.bigquery.model.Table;
+import com.google.api.services.bigquery.model.TableList;
+import com.google.api.services.bigquery.model.TableList.Tables;
+import com.google.api.services.bigquery.model.TableReference;
+import com.google.api.services.bigquery.model.TableSchema;
+import com.google.auth.http.HttpCredentialsAdapter;
+import com.google.auth.oauth2.GoogleCredentials;
+import com.google.cloud.BaseServiceException;
+import com.google.cloud.ExceptionHandler;
+import com.google.cloud.bigquery.BigQueryErrorMessages;
+import com.google.cloud.bigquery.BigQueryOptions;
+import com.google.cloud.bigquery.BigQueryRetryConfig;
+import com.google.cloud.bigquery.BigQueryRetryHelper;
+import java.io.IOException;
+import java.nio.charset.StandardCharsets;
+import java.security.GeneralSecurityException;
+import java.util.List;
+import java.util.Locale;
+import java.util.Map;
+import java.util.Set;
+import java.util.concurrent.Callable;
+import java.util.stream.Collectors;
+import java.util.stream.Stream;
+import org.apache.iceberg.BaseMetastoreTableOperations;
+import org.apache.iceberg.exceptions.AlreadyExistsException;
+import org.apache.iceberg.exceptions.BadRequestException;
+import org.apache.iceberg.exceptions.ForbiddenException;
+import org.apache.iceberg.exceptions.NoSuchIcebergTableException;
+import org.apache.iceberg.exceptions.NoSuchNamespaceException;
+import org.apache.iceberg.exceptions.NoSuchTableException;
+import org.apache.iceberg.exceptions.NotAuthorizedException;
+import org.apache.iceberg.exceptions.RuntimeIOException;
+import org.apache.iceberg.exceptions.ServiceFailureException;
+import org.apache.iceberg.exceptions.ServiceUnavailableException;
+import org.apache.iceberg.exceptions.ValidationException;
+import org.apache.iceberg.relocated.com.google.common.base.Preconditions;
+import org.apache.iceberg.relocated.com.google.common.collect.Lists;
+import org.apache.iceberg.relocated.com.google.common.collect.Maps;
+
+/** A client of Google Bigquery Metastore functions over the BigQuery service. */
+public final class BigQueryMetaStoreClientImpl implements BigQueryMetaStoreClient {
+
+  public static final ExceptionHandler.Interceptor EXCEPTION_HANDLER_INTERCEPTOR =
+      new ExceptionHandler.Interceptor() {
+
+        private static final long serialVersionUID = -8429573486870467828L;
+
+        @Override
+        public RetryResult afterEval(Exception exception, RetryResult retryResult) {
+          return ExceptionHandler.Interceptor.RetryResult.CONTINUE_EVALUATION;
+        }
+
+        @Override
+        public RetryResult beforeEval(Exception exception) {
+          if (exception instanceof BaseServiceException) {
+            boolean retriable = ((BaseServiceException) exception).isRetryable();
+            return retriable
+                ? ExceptionHandler.Interceptor.RetryResult.RETRY
+                : ExceptionHandler.Interceptor.RetryResult.CONTINUE_EVALUATION;
+          }
+          return ExceptionHandler.Interceptor.RetryResult.CONTINUE_EVALUATION;
+        }
+      };
+
+  private final Bigquery client;
+  private final BigQueryOptions bigqueryOptions;
+
+  // Retry config with error messages and regex for rate limit exceeded errors.
+  private static final BigQueryRetryConfig DEFAULT_RETRY_CONFIG =
+      BigQueryRetryConfig.newBuilder()
+          .retryOnMessage(BigQueryErrorMessages.RATE_LIMIT_EXCEEDED_MSG)
+          .retryOnMessage(BigQueryErrorMessages.JOB_RATE_LIMIT_EXCEEDED_MSG)
+          .retryOnRegEx(BigQueryErrorMessages.RetryRegExPatterns.RATE_LIMIT_EXCEEDED_REGEX)
+          .build();
+
+  public static final ExceptionHandler BIGQUERY_EXCEPTION_HANDLER =
+      ExceptionHandler.newBuilder()
+          .abortOn(RuntimeException.class)
+          .retryOn(java.net.ConnectException.class) // retry on Connection Exception
+          .retryOn(java.net.UnknownHostException.class) // retry on UnknownHostException
+          .addInterceptors(EXCEPTION_HANDLER_INTERCEPTOR)
+          .build();
+
+  /** Constructs a client of the Google BigQuery service. */
+  public BigQueryMetaStoreClientImpl(BigQueryOptions options)
+      throws IOException, GeneralSecurityException {
+    // Initialize client that will be used to send requests. This client only needs to be created
+    // once, and can be reused for multiple requests
+    HttpCredentialsAdapter httpCredentialsAdapter =
+        new HttpCredentialsAdapter(
+            GoogleCredentials.getApplicationDefault().createScoped(BigqueryScopes.all()));
+    this.client =
+        new Bigquery.Builder(
+                GoogleNetHttpTransport.newTrustedTransport(),
+                GsonFactory.getDefaultInstance(),
+                httpRequest -> {
+                  httpCredentialsAdapter.initialize(httpRequest);
+                  // Instead of throwing exceptions, analyze the HttpResponse object and inspect its
+                  // status code. This will allow BigQuery API errors to be converted into Iceberg
+                  // exceptions.
+                  httpRequest.setThrowExceptionOnExecuteError(false);
+                })
+            .setApplicationName("BigQuery Metastore Iceberg Catalog Plugin")
+            .build();
+    this.bigqueryOptions = options;
+  }
+
+  @Override
+  public Dataset createDataset(Dataset dataset) {
+    // TODO (b/399885863): Ensure Dataset creation is idempotent when handling retries.

Review Comment:
   What is `b/399885863`? 



##########
bigquery/src/main/java/org/apache/iceberg/gcp/bigquery/BigQueryMetaStoreClientImpl.java:
##########
@@ -0,0 +1,620 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.iceberg.gcp.bigquery;
+
+import com.google.api.client.googleapis.javanet.GoogleNetHttpTransport;
+import com.google.api.client.http.HttpHeaders;
+import com.google.api.client.http.HttpResponse;
+import com.google.api.client.http.HttpResponseException;
+import com.google.api.client.http.HttpStatusCodes;
+import com.google.api.client.json.gson.GsonFactory;
+import com.google.api.client.util.Data;
+import com.google.api.services.bigquery.Bigquery;
+import com.google.api.services.bigquery.BigqueryScopes;
+import com.google.api.services.bigquery.model.Dataset;
+import com.google.api.services.bigquery.model.DatasetList;
+import com.google.api.services.bigquery.model.DatasetList.Datasets;
+import com.google.api.services.bigquery.model.DatasetReference;
+import com.google.api.services.bigquery.model.ExternalCatalogDatasetOptions;
+import com.google.api.services.bigquery.model.ExternalCatalogTableOptions;
+import com.google.api.services.bigquery.model.Table;
+import com.google.api.services.bigquery.model.TableList;
+import com.google.api.services.bigquery.model.TableList.Tables;
+import com.google.api.services.bigquery.model.TableReference;
+import com.google.api.services.bigquery.model.TableSchema;
+import com.google.auth.http.HttpCredentialsAdapter;
+import com.google.auth.oauth2.GoogleCredentials;
+import com.google.cloud.BaseServiceException;
+import com.google.cloud.ExceptionHandler;
+import com.google.cloud.bigquery.BigQueryErrorMessages;
+import com.google.cloud.bigquery.BigQueryOptions;
+import com.google.cloud.bigquery.BigQueryRetryConfig;
+import com.google.cloud.bigquery.BigQueryRetryHelper;
+import java.io.IOException;
+import java.nio.charset.StandardCharsets;
+import java.security.GeneralSecurityException;
+import java.util.List;
+import java.util.Locale;
+import java.util.Map;
+import java.util.Set;
+import java.util.concurrent.Callable;
+import java.util.stream.Collectors;
+import java.util.stream.Stream;
+import org.apache.iceberg.BaseMetastoreTableOperations;
+import org.apache.iceberg.exceptions.AlreadyExistsException;
+import org.apache.iceberg.exceptions.BadRequestException;
+import org.apache.iceberg.exceptions.ForbiddenException;
+import org.apache.iceberg.exceptions.NoSuchIcebergTableException;
+import org.apache.iceberg.exceptions.NoSuchNamespaceException;
+import org.apache.iceberg.exceptions.NoSuchTableException;
+import org.apache.iceberg.exceptions.NotAuthorizedException;
+import org.apache.iceberg.exceptions.RuntimeIOException;
+import org.apache.iceberg.exceptions.ServiceFailureException;
+import org.apache.iceberg.exceptions.ServiceUnavailableException;
+import org.apache.iceberg.exceptions.ValidationException;
+import org.apache.iceberg.relocated.com.google.common.base.Preconditions;
+import org.apache.iceberg.relocated.com.google.common.collect.Lists;
+import org.apache.iceberg.relocated.com.google.common.collect.Maps;
+
+/** A client of Google Bigquery Metastore functions over the BigQuery service. */
+public final class BigQueryMetaStoreClientImpl implements BigQueryMetaStoreClient {
+
+  public static final ExceptionHandler.Interceptor EXCEPTION_HANDLER_INTERCEPTOR =
+      new ExceptionHandler.Interceptor() {
+
+        private static final long serialVersionUID = -8429573486870467828L;
+
+        @Override
+        public RetryResult afterEval(Exception exception, RetryResult retryResult) {
+          return ExceptionHandler.Interceptor.RetryResult.CONTINUE_EVALUATION;
+        }
+
+        @Override
+        public RetryResult beforeEval(Exception exception) {
+          if (exception instanceof BaseServiceException) {
+            boolean retriable = ((BaseServiceException) exception).isRetryable();
+            return retriable
+                ? ExceptionHandler.Interceptor.RetryResult.RETRY
+                : ExceptionHandler.Interceptor.RetryResult.CONTINUE_EVALUATION;
+          }
+          return ExceptionHandler.Interceptor.RetryResult.CONTINUE_EVALUATION;
+        }
+      };
+
+  private final Bigquery client;
+  private final BigQueryOptions bigqueryOptions;
+
+  // Retry config with error messages and regex for rate limit exceeded errors.
+  private static final BigQueryRetryConfig DEFAULT_RETRY_CONFIG =
+      BigQueryRetryConfig.newBuilder()
+          .retryOnMessage(BigQueryErrorMessages.RATE_LIMIT_EXCEEDED_MSG)
+          .retryOnMessage(BigQueryErrorMessages.JOB_RATE_LIMIT_EXCEEDED_MSG)
+          .retryOnRegEx(BigQueryErrorMessages.RetryRegExPatterns.RATE_LIMIT_EXCEEDED_REGEX)
+          .build();
+
+  public static final ExceptionHandler BIGQUERY_EXCEPTION_HANDLER =
+      ExceptionHandler.newBuilder()
+          .abortOn(RuntimeException.class)
+          .retryOn(java.net.ConnectException.class) // retry on Connection Exception
+          .retryOn(java.net.UnknownHostException.class) // retry on UnknownHostException
+          .addInterceptors(EXCEPTION_HANDLER_INTERCEPTOR)
+          .build();
+
+  /** Constructs a client of the Google BigQuery service. */
+  public BigQueryMetaStoreClientImpl(BigQueryOptions options)
+      throws IOException, GeneralSecurityException {
+    // Initialize client that will be used to send requests. This client only needs to be created
+    // once, and can be reused for multiple requests
+    HttpCredentialsAdapter httpCredentialsAdapter =
+        new HttpCredentialsAdapter(
+            GoogleCredentials.getApplicationDefault().createScoped(BigqueryScopes.all()));
+    this.client =
+        new Bigquery.Builder(
+                GoogleNetHttpTransport.newTrustedTransport(),
+                GsonFactory.getDefaultInstance(),
+                httpRequest -> {
+                  httpCredentialsAdapter.initialize(httpRequest);
+                  // Instead of throwing exceptions, analyze the HttpResponse object and inspect its
+                  // status code. This will allow BigQuery API errors to be converted into Iceberg
+                  // exceptions.
+                  httpRequest.setThrowExceptionOnExecuteError(false);
+                })
+            .setApplicationName("BigQuery Metastore Iceberg Catalog Plugin")
+            .build();
+    this.bigqueryOptions = options;
+  }
+
+  @Override
+  public Dataset createDataset(Dataset dataset) {
+    // TODO (b/399885863): Ensure Dataset creation is idempotent when handling retries.
+    Dataset response = null;
+    try {
+      response =
+          BigQueryRetryHelper.runWithRetries(
+              new Callable<Dataset>() {
+                @Override
+                public Dataset call() {
+                  return create(dataset);
+                }
+              },
+              bigqueryOptions.getRetrySettings(),
+              BIGQUERY_EXCEPTION_HANDLER,
+              bigqueryOptions.getClock(),
+              DEFAULT_RETRY_CONFIG);
+    } catch (BigQueryRetryHelper.BigQueryRetryHelperException e) {
+      handleBigQueryRetryException(e);
+    }
+    return response;
+  }
+
+  @SuppressWarnings("FormatStringAnnotation")
+  private Dataset create(Dataset dataset) {
+    try {
+      HttpResponse response =
+          client
+              .datasets()
+              .insert(dataset.getDatasetReference().getProjectId(), dataset)
+              .executeUnparsed();
+      return convertExceptionIfUnsuccessful(response).parseAs(Dataset.class);
+    } catch (IOException e) {
+      throw new RuntimeIOException(e);
+    } catch (AlreadyExistsException e) {
+      throw new AlreadyExistsException("Namespace already exists: " + dataset.getId());
+    }
+  }
+
+  @Override
+  @SuppressWarnings("FormatStringAnnotation")
+  public Dataset getDataset(DatasetReference datasetReference) {
+    try {
+      HttpResponse response =
+          client
+              .datasets()
+              .get(datasetReference.getProjectId(), datasetReference.getDatasetId())
+              .executeUnparsed();
+      if (response.getStatusCode() == HttpStatusCodes.STATUS_CODE_NOT_FOUND) {
+        throw new NoSuchNamespaceException(
+            "Namespace does not exist: " + datasetReference.getDatasetId());
+      }
+      return convertExceptionIfUnsuccessful(response).parseAs(Dataset.class);
+    } catch (IOException e) {
+      throw new RuntimeIOException(e);
+    }
+  }
+
+  @Override
+  @SuppressWarnings("FormatStringAnnotation")
+  public void deleteDataset(DatasetReference datasetReference) {
+    try {
+      HttpResponse response =
+          client
+              .datasets()
+              .delete(datasetReference.getProjectId(), datasetReference.getDatasetId())
+              .executeUnparsed();
+      if (response.getStatusCode() == HttpStatusCodes.STATUS_CODE_NOT_FOUND) {
+        throw new NoSuchNamespaceException(
+            "Namespace does not exist: " + datasetReference.getDatasetId());
+      }
+      convertExceptionIfUnsuccessful(response);
+    } catch (IOException e) {
+      throw new RuntimeIOException(e);
+    }
+  }
+
+  @Override
+  public Dataset setDatasetParameters(
+      DatasetReference datasetReference, Map<String, String> parameters) {
+    Dataset dataset = getDataset(datasetReference);
+    ExternalCatalogDatasetOptions externalCatalogDatasetOptions =
+        dataset.getExternalCatalogDatasetOptions() == null
+            ? new ExternalCatalogDatasetOptions()
+            : dataset.getExternalCatalogDatasetOptions();
+    Map<String, String> finalParameters =
+        externalCatalogDatasetOptions.getParameters() == null
+            ? Maps.newHashMap()
+            : externalCatalogDatasetOptions.getParameters();
+    finalParameters.putAll(parameters);
+
+    dataset.setExternalCatalogDatasetOptions(
+        externalCatalogDatasetOptions.setParameters(finalParameters));
+
+    return updateDataset(dataset);
+  }
+
+  @Override
+  public Dataset removeDatasetParameters(
+      DatasetReference datasetReference, Set<String> parameters) {
+    Dataset dataset = getDataset(datasetReference);
+    ExternalCatalogDatasetOptions externalCatalogDatasetOptions =
+        dataset.getExternalCatalogDatasetOptions() == null
+            ? new ExternalCatalogDatasetOptions()
+            : dataset.getExternalCatalogDatasetOptions();
+    Map<String, String> finalParameters =
+        externalCatalogDatasetOptions.getParameters() == null
+            ? Maps.newHashMap()
+            : externalCatalogDatasetOptions.getParameters();
+    parameters.forEach(finalParameters::remove);
+
+    dataset.setExternalCatalogDatasetOptions(
+        externalCatalogDatasetOptions.setParameters(finalParameters));
+
+    return updateDataset(dataset);
+  }
+
+  @Override
+  public List<Datasets> listDatasets(String projectId) {
+    try {
+      String nextPageToken = null;
+      List<Datasets> datasets = Lists.newArrayList();
+      do {
+        HttpResponse pageResponse =
+            client.datasets().list(projectId).setPageToken(nextPageToken).executeUnparsed();
+        DatasetList result =
+            convertExceptionIfUnsuccessful(pageResponse).parseAs(DatasetList.class);
+        nextPageToken = result.getNextPageToken();
+        if (result.getDatasets() != null) {
+          datasets.addAll(result.getDatasets());
+        }
+      } while (nextPageToken != null && !nextPageToken.isEmpty());
+      return datasets;
+    } catch (IOException e) {
+      throw new RuntimeIOException(e);
+    }
+  }
+
+  @Override
+  public Table createTable(Table table) {
+    // Ensure it is an Iceberg table supported by the BigQuery metastore catalog.
+    validateTable(table);
+    // TODO (b/399885863): Ensure table creation is idempotent when handling retries.
+    Table response = null;
+    try {
+      response =
+          BigQueryRetryHelper.runWithRetries(
+              new Callable<Table>() {
+                @Override
+                public Table call() {
+                  return create(table);
+                }
+              },
+              bigqueryOptions.getRetrySettings(),
+              BIGQUERY_EXCEPTION_HANDLER,
+              bigqueryOptions.getClock(),
+              DEFAULT_RETRY_CONFIG);
+    } catch (BigQueryRetryHelper.BigQueryRetryHelperException e) {
+      handleBigQueryRetryException(e);
+    }
+    return response;
+  }
+
+  @SuppressWarnings("FormatStringAnnotation")
+  private Table create(Table table) {
+    try {
+      HttpResponse response =
+          client
+              .tables()
+              .insert(
+                  Preconditions.checkNotNull(table.getTableReference()).getProjectId(),
+                  Preconditions.checkNotNull(table.getTableReference()).getDatasetId(),
+                  table)
+              .executeUnparsed();
+      return convertExceptionIfUnsuccessful(response).parseAs(Table.class);
+    } catch (IOException e) {
+      throw new RuntimeIOException(e);
+    } catch (AlreadyExistsException e) {
+      throw new AlreadyExistsException("Table already exists", e);
+    }
+  }
+
+  @Override
+  @SuppressWarnings("FormatStringAnnotation")
+  public Table getTable(TableReference tableReference) {
+    try {
+      HttpResponse response =
+          client
+              .tables()
+              .get(
+                  tableReference.getProjectId(),
+                  tableReference.getDatasetId(),
+                  tableReference.getTableId())
+              .executeUnparsed();
+      if (response.getStatusCode() == HttpStatusCodes.STATUS_CODE_NOT_FOUND) {
+        throw new NoSuchTableException(response.getStatusMessage());
+      }
+      return validateTable(convertExceptionIfUnsuccessful(response).parseAs(Table.class));
+    } catch (IOException e) {
+      throw new RuntimeIOException(e);
+    }
+  }
+
+  @Override
+  @SuppressWarnings("FormatStringAnnotation")
+  public Table patchTable(TableReference tableReference, Table table) {
+    // Ensure it is an Iceberg table supported by the BQ metastore catalog.
+    validateTable(table);
+
+    ExternalCatalogTableOptions newExternalCatalogTableOptions =
+        new ExternalCatalogTableOptions()
+            .setStorageDescriptor(table.getExternalCatalogTableOptions().getStorageDescriptor())
+            .setConnectionId(table.getExternalCatalogTableOptions().getConnectionId())
+            .setParameters(table.getExternalCatalogTableOptions().getParameters());
+    Table updatedTable =
+        new Table()
+            .setExternalCatalogTableOptions(newExternalCatalogTableOptions)
+            // Must set the schema as null for using schema auto-detect.
+            .setSchema(Data.nullOf(TableSchema.class));
+
+    Table response = null;
+    try {
+      response =
+          BigQueryRetryHelper.runWithRetries(
+              new Callable<Table>() {
+                @Override
+                public Table call() {
+                  return patch(tableReference, updatedTable, table.getEtag());
+                }
+              },
+              bigqueryOptions.getRetrySettings(),
+              BIGQUERY_EXCEPTION_HANDLER,
+              bigqueryOptions.getClock(),
+              DEFAULT_RETRY_CONFIG);
+    } catch (BigQueryRetryHelper.BigQueryRetryHelperException e) {
+      handleBigQueryRetryException(e);
+    }
+    return response;
+  }
+
+  @SuppressWarnings("FormatStringAnnotation")
+  private Table patch(TableReference tableReference, Table table, String etag) {
+    try {
+      HttpResponse response =
+          client
+              .tables()
+              .patch(
+                  tableReference.getProjectId(),
+                  tableReference.getDatasetId(),
+                  tableReference.getTableId(),
+                  table)
+              .setRequestHeaders(new HttpHeaders().setIfMatch(etag))
+              .executeUnparsed();
+
+      if (response.getStatusCode() == HttpStatusCodes.STATUS_CODE_NOT_FOUND) {
+        String responseString = response.parseAsString();
+        if (responseString.toLowerCase(Locale.ROOT).contains("not found: connection")) {
+          throw new BadRequestException(responseString);
+        }
+        throw new NoSuchTableException(response.getStatusMessage());
+      }
+      return convertExceptionIfUnsuccessful(response).parseAs(Table.class);
+    } catch (IOException e) {
+      throw new RuntimeIOException(e);
+    }
+  }
+
+  @Override
+  @SuppressWarnings("FormatStringAnnotation")
+  public Table renameTable(TableReference tableToRename, String newTableId) {
+    Table table = getTable(tableToRename); // Verify table first
+    Table patch =
+        new Table()
+            .setTableReference(
+                new TableReference()
+                    .setProjectId(table.getTableReference().getProjectId())
+                    .setDatasetId(table.getTableReference().getDatasetId())
+                    .setTableId(newTableId));
+
+    try {
+      HttpResponse response =
+          client
+              .tables()
+              .patch(
+                  tableToRename.getProjectId(),
+                  tableToRename.getDatasetId(),
+                  tableToRename.getTableId(),
+                  patch)
+              .setRequestHeaders(new HttpHeaders().setIfMatch(table.getEtag()))
+              .executeUnparsed();
+
+      if (response.getStatusCode() == HttpStatusCodes.STATUS_CODE_NOT_FOUND) {
+        throw new NoSuchTableException(response.getStatusMessage());
+      }
+      return convertExceptionIfUnsuccessful(response).parseAs(Table.class);
+    } catch (IOException e) {
+      throw new RuntimeIOException(e);
+    }
+  }
+
+  @Override
+  @SuppressWarnings("FormatStringAnnotation")
+  public void deleteTable(TableReference tableReference) {
+    try {
+      getTable(tableReference); // Fetching it to validate it is a BigQuery Metastore table first
+
+      HttpResponse response =
+          client
+              .tables()
+              .delete(
+                  tableReference.getProjectId(),
+                  tableReference.getDatasetId(),
+                  tableReference.getTableId())
+              .executeUnparsed();
+
+      if (response.getStatusCode() == HttpStatusCodes.STATUS_CODE_NOT_FOUND) {
+        throw new NoSuchTableException(response.getStatusMessage());
+      }
+      convertExceptionIfUnsuccessful(response);
+    } catch (IOException e) {
+      throw new RuntimeIOException(e);
+    }
+  }
+
+  @Override
+  @SuppressWarnings("FormatStringAnnotation")
+  public List<Tables> listTables(
+      DatasetReference datasetReference, boolean filterUnsupportedTables) {
+    try {
+      String nextPageToken = null;
+      Stream<Tables> tablesStream = Stream.empty();
+      do {
+        HttpResponse pageResponse =
+            client
+                .tables()
+                .list(datasetReference.getProjectId(), datasetReference.getDatasetId())
+                .setPageToken(nextPageToken)
+                .executeUnparsed();
+        if (pageResponse.getStatusCode() == HttpStatusCodes.STATUS_CODE_NOT_FOUND) {
+          throw new NoSuchNamespaceException(pageResponse.getStatusMessage());
+        }
+        TableList result = convertExceptionIfUnsuccessful(pageResponse).parseAs(TableList.class);
+        nextPageToken = result.getNextPageToken();
+        List<Tables> tablesPage = result.getTables();
+        Stream<Tables> tablesPageStream =
+            tablesPage == null ? Stream.empty() : result.getTables().stream();
+        tablesStream = Stream.concat(tablesStream, tablesPageStream);
+      } while (nextPageToken != null && !nextPageToken.isEmpty());
+
+      // TODO(b/345839927): The server should return more metadata here to distinguish Iceberg
+      // BQMS tables for us to filter out those results since invoking `getTable` on them would
+      // correctly raise a `NoSuchIcebergTableException` for being inoperable by this plugin.
+      if (filterUnsupportedTables) {
+        tablesStream =
+            tablesStream
+                .parallel()
+                .filter(
+                    table -> {
+                      try {
+                        getTable(table.getTableReference());
+                      } catch (NoSuchTableException e) {
+                        return false;
+                      }
+                      return true;
+                    });
+      }
+
+      return tablesStream.collect(Collectors.toList());
+    } catch (IOException e) {
+      throw new RuntimeIOException(e);
+    }
+  }
+
+  @SuppressWarnings("FormatStringAnnotation")
+  private Dataset updateDataset(Dataset dataset) {
+    try {
+      HttpResponse response =
+          client
+              .datasets()
+              .update(
+                  Preconditions.checkNotNull(dataset.getDatasetReference()).getProjectId(),
+                  Preconditions.checkNotNull(dataset.getDatasetReference().getDatasetId()),
+                  dataset)
+              .setRequestHeaders(new HttpHeaders().setIfMatch(dataset.getEtag()))
+              .executeUnparsed();
+      if (response.getStatusCode() == HttpStatusCodes.STATUS_CODE_NOT_FOUND) {
+        throw new NoSuchNamespaceException(response.getStatusMessage());
+      }
+      return convertExceptionIfUnsuccessful(response).parseAs(Dataset.class);
+    } catch (IOException e) {
+      throw new RuntimeIOException(e);
+    }
+  }
+
+  /**
+   * Returns true when it is a BigQuery Metastore Iceberg table, defined by having the
+   * ExternalCatalogTableOptions object and a parameter of METADATA_LOCATION_PROP as part of its
+   * parameters map.
+   *
+   * @param table to check
+   */
+  private boolean isValidIcebergTable(Table table) {

Review Comment:
   This method can be static. Same for convertExceptionIfUnsuccessful. 



##########
bigquery/src/main/java/org/apache/iceberg/gcp/bigquery/BigQueryMetaStoreClient.java:
##########
@@ -0,0 +1,129 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.iceberg.gcp.bigquery;
+
+import com.google.api.services.bigquery.model.Dataset;
+import com.google.api.services.bigquery.model.DatasetList.Datasets;
+import com.google.api.services.bigquery.model.DatasetReference;
+import com.google.api.services.bigquery.model.Table;
+import com.google.api.services.bigquery.model.TableList.Tables;
+import com.google.api.services.bigquery.model.TableReference;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+
+/**
+ * A client of Google BigQuery Metastore functions over the BigQuery service. Uses the Google
+ * BigQuery API.
+ */
+public interface BigQueryMetaStoreClient {
+
+  /**
+   * Creates and returns a new dataset.
+   *
+   * @param dataset the dataset to create
+   */
+  Dataset createDataset(Dataset dataset);

Review Comment:
   Return value of createDataset, setDatasetParameters, removeDatasetParameters are never used. 



##########
bigquery/src/main/java/org/apache/iceberg/gcp/bigquery/BigQueryTableOperations.java:
##########
@@ -0,0 +1,300 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.iceberg.gcp.bigquery;
+
+import com.google.api.services.bigquery.model.ExternalCatalogTableOptions;
+import com.google.api.services.bigquery.model.Table;
+import com.google.api.services.bigquery.model.TableReference;
+import java.util.Locale;
+import java.util.Map;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hive.common.StatsSetupConst;
+import org.apache.iceberg.BaseMetastoreOperations;
+import org.apache.iceberg.BaseMetastoreTableOperations;
+import org.apache.iceberg.SnapshotSummary;
+import org.apache.iceberg.TableMetadata;
+import org.apache.iceberg.TableProperties;
+import org.apache.iceberg.exceptions.AlreadyExistsException;
+import org.apache.iceberg.exceptions.CommitFailedException;
+import org.apache.iceberg.exceptions.CommitStateUnknownException;
+import org.apache.iceberg.exceptions.NoSuchTableException;
+import org.apache.iceberg.exceptions.ValidationException;
+import org.apache.iceberg.hadoop.ConfigProperties;
+import org.apache.iceberg.io.FileIO;
+import org.apache.iceberg.relocated.com.google.common.collect.Maps;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/** Handles BigQuery metastore table operations. */
+public final class BigQueryTableOperations extends BaseMetastoreTableOperations {
+
+  private static final Logger LOG = LoggerFactory.getLogger(BigQueryTableOperations.class);
+
+  public static final String TABLE_PROPERTIES_BQ_CONNECTION = "bq_connection";
+
+  private final BigQueryMetaStoreClient client;
+  private final FileIO fileIO;
+  private final TableReference tableReference;
+  private final Configuration conf;
+
+  BigQueryTableOperations(
+      BigQueryMetaStoreClient client,
+      FileIO fileIO,
+      String project,
+      String dataset,
+      String table,
+      Configuration conf) {
+    this.client = client;
+    this.fileIO = fileIO;
+    this.tableReference =
+        new TableReference().setProjectId(project).setDatasetId(dataset).setTableId(table);
+    this.conf = conf;
+  }
+
+  // The doRefresh method should provide implementation on how to get the metadata location.
+  @Override
+  public void doRefresh() {
+    // Must default to null.
+    String metadataLocation = null;
+    try {
+      metadataLocation =
+          getMetadataLocationOrThrow(
+              client.getTable(this.tableReference).getExternalCatalogTableOptions());
+    } catch (NoSuchTableException e) {
+      if (currentMetadataLocation() != null) {
+        // Re-throws the exception because the table must exist in this case.
+        throw e;
+      }
+    }
+    refreshFromMetadataLocation(metadataLocation);
+  }
+
+  // The doCommit method should provide implementation on how to update with metadata location
+  // atomically
+  @Override
+  public void doCommit(TableMetadata base, TableMetadata metadata) {
+    String newMetadataLocation =
+        base == null && metadata.metadataFileLocation() != null
+            ? metadata.metadataFileLocation()
+            : writeNewMetadata(metadata, currentVersion() + 1);
+    BaseMetastoreOperations.CommitStatus commitStatus =
+        BaseMetastoreOperations.CommitStatus.FAILURE;
+    try {
+      if (base == null) {
+        createTable(newMetadataLocation, metadata);
+      } else {
+        updateTable(base.metadataFileLocation(), newMetadataLocation, metadata);
+      }
+      commitStatus = BaseMetastoreOperations.CommitStatus.SUCCESS;
+    } catch (CommitFailedException | CommitStateUnknownException e) {
+      throw e;
+    } catch (Throwable e) {
+      LOG.error("Exception thrown on commit: ", e);
+
+      if (commitStatus == BaseMetastoreOperations.CommitStatus.FAILURE

Review Comment:
   `commitStatus == BaseMetastoreOperations.CommitStatus.FAILURE` is always true in the current logic. 



##########
bigquery/src/main/java/org/apache/iceberg/gcp/bigquery/BigQueryMetaStoreClientImpl.java:
##########
@@ -0,0 +1,620 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.iceberg.gcp.bigquery;
+
+import com.google.api.client.googleapis.javanet.GoogleNetHttpTransport;
+import com.google.api.client.http.HttpHeaders;
+import com.google.api.client.http.HttpResponse;
+import com.google.api.client.http.HttpResponseException;
+import com.google.api.client.http.HttpStatusCodes;
+import com.google.api.client.json.gson.GsonFactory;
+import com.google.api.client.util.Data;
+import com.google.api.services.bigquery.Bigquery;
+import com.google.api.services.bigquery.BigqueryScopes;
+import com.google.api.services.bigquery.model.Dataset;
+import com.google.api.services.bigquery.model.DatasetList;
+import com.google.api.services.bigquery.model.DatasetList.Datasets;
+import com.google.api.services.bigquery.model.DatasetReference;
+import com.google.api.services.bigquery.model.ExternalCatalogDatasetOptions;
+import com.google.api.services.bigquery.model.ExternalCatalogTableOptions;
+import com.google.api.services.bigquery.model.Table;
+import com.google.api.services.bigquery.model.TableList;
+import com.google.api.services.bigquery.model.TableList.Tables;
+import com.google.api.services.bigquery.model.TableReference;
+import com.google.api.services.bigquery.model.TableSchema;
+import com.google.auth.http.HttpCredentialsAdapter;
+import com.google.auth.oauth2.GoogleCredentials;
+import com.google.cloud.BaseServiceException;
+import com.google.cloud.ExceptionHandler;
+import com.google.cloud.bigquery.BigQueryErrorMessages;
+import com.google.cloud.bigquery.BigQueryOptions;
+import com.google.cloud.bigquery.BigQueryRetryConfig;
+import com.google.cloud.bigquery.BigQueryRetryHelper;
+import java.io.IOException;
+import java.nio.charset.StandardCharsets;
+import java.security.GeneralSecurityException;
+import java.util.List;
+import java.util.Locale;
+import java.util.Map;
+import java.util.Set;
+import java.util.concurrent.Callable;
+import java.util.stream.Collectors;
+import java.util.stream.Stream;
+import org.apache.iceberg.BaseMetastoreTableOperations;
+import org.apache.iceberg.exceptions.AlreadyExistsException;
+import org.apache.iceberg.exceptions.BadRequestException;
+import org.apache.iceberg.exceptions.ForbiddenException;
+import org.apache.iceberg.exceptions.NoSuchIcebergTableException;
+import org.apache.iceberg.exceptions.NoSuchNamespaceException;
+import org.apache.iceberg.exceptions.NoSuchTableException;
+import org.apache.iceberg.exceptions.NotAuthorizedException;
+import org.apache.iceberg.exceptions.RuntimeIOException;
+import org.apache.iceberg.exceptions.ServiceFailureException;
+import org.apache.iceberg.exceptions.ServiceUnavailableException;
+import org.apache.iceberg.exceptions.ValidationException;
+import org.apache.iceberg.relocated.com.google.common.base.Preconditions;
+import org.apache.iceberg.relocated.com.google.common.collect.Lists;
+import org.apache.iceberg.relocated.com.google.common.collect.Maps;
+
+/** A client of Google Bigquery Metastore functions over the BigQuery service. */
+public final class BigQueryMetaStoreClientImpl implements BigQueryMetaStoreClient {
+
+  public static final ExceptionHandler.Interceptor EXCEPTION_HANDLER_INTERCEPTOR =
+      new ExceptionHandler.Interceptor() {
+
+        private static final long serialVersionUID = -8429573486870467828L;
+
+        @Override
+        public RetryResult afterEval(Exception exception, RetryResult retryResult) {
+          return ExceptionHandler.Interceptor.RetryResult.CONTINUE_EVALUATION;
+        }
+
+        @Override
+        public RetryResult beforeEval(Exception exception) {
+          if (exception instanceof BaseServiceException) {
+            boolean retriable = ((BaseServiceException) exception).isRetryable();
+            return retriable
+                ? ExceptionHandler.Interceptor.RetryResult.RETRY
+                : ExceptionHandler.Interceptor.RetryResult.CONTINUE_EVALUATION;
+          }
+          return ExceptionHandler.Interceptor.RetryResult.CONTINUE_EVALUATION;
+        }
+      };
+
+  private final Bigquery client;
+  private final BigQueryOptions bigqueryOptions;
+
+  // Retry config with error messages and regex for rate limit exceeded errors.
+  private static final BigQueryRetryConfig DEFAULT_RETRY_CONFIG =
+      BigQueryRetryConfig.newBuilder()
+          .retryOnMessage(BigQueryErrorMessages.RATE_LIMIT_EXCEEDED_MSG)
+          .retryOnMessage(BigQueryErrorMessages.JOB_RATE_LIMIT_EXCEEDED_MSG)
+          .retryOnRegEx(BigQueryErrorMessages.RetryRegExPatterns.RATE_LIMIT_EXCEEDED_REGEX)
+          .build();
+
+  public static final ExceptionHandler BIGQUERY_EXCEPTION_HANDLER =
+      ExceptionHandler.newBuilder()
+          .abortOn(RuntimeException.class)
+          .retryOn(java.net.ConnectException.class) // retry on Connection Exception
+          .retryOn(java.net.UnknownHostException.class) // retry on UnknownHostException
+          .addInterceptors(EXCEPTION_HANDLER_INTERCEPTOR)
+          .build();
+
+  /** Constructs a client of the Google BigQuery service. */
+  public BigQueryMetaStoreClientImpl(BigQueryOptions options)
+      throws IOException, GeneralSecurityException {
+    // Initialize client that will be used to send requests. This client only needs to be created
+    // once, and can be reused for multiple requests
+    HttpCredentialsAdapter httpCredentialsAdapter =
+        new HttpCredentialsAdapter(
+            GoogleCredentials.getApplicationDefault().createScoped(BigqueryScopes.all()));
+    this.client =
+        new Bigquery.Builder(
+                GoogleNetHttpTransport.newTrustedTransport(),
+                GsonFactory.getDefaultInstance(),
+                httpRequest -> {
+                  httpCredentialsAdapter.initialize(httpRequest);
+                  // Instead of throwing exceptions, analyze the HttpResponse object and inspect its
+                  // status code. This will allow BigQuery API errors to be converted into Iceberg
+                  // exceptions.
+                  httpRequest.setThrowExceptionOnExecuteError(false);
+                })
+            .setApplicationName("BigQuery Metastore Iceberg Catalog Plugin")
+            .build();
+    this.bigqueryOptions = options;
+  }
+
+  @Override
+  public Dataset createDataset(Dataset dataset) {
+    // TODO (b/399885863): Ensure Dataset creation is idempotent when handling retries.
+    Dataset response = null;
+    try {
+      response =
+          BigQueryRetryHelper.runWithRetries(
+              new Callable<Dataset>() {
+                @Override
+                public Dataset call() {
+                  return create(dataset);
+                }
+              },
+              bigqueryOptions.getRetrySettings(),
+              BIGQUERY_EXCEPTION_HANDLER,
+              bigqueryOptions.getClock(),
+              DEFAULT_RETRY_CONFIG);
+    } catch (BigQueryRetryHelper.BigQueryRetryHelperException e) {
+      handleBigQueryRetryException(e);
+    }
+    return response;
+  }
+
+  @SuppressWarnings("FormatStringAnnotation")
+  private Dataset create(Dataset dataset) {
+    try {
+      HttpResponse response =
+          client
+              .datasets()
+              .insert(dataset.getDatasetReference().getProjectId(), dataset)
+              .executeUnparsed();
+      return convertExceptionIfUnsuccessful(response).parseAs(Dataset.class);
+    } catch (IOException e) {
+      throw new RuntimeIOException(e);
+    } catch (AlreadyExistsException e) {
+      throw new AlreadyExistsException("Namespace already exists: " + dataset.getId());
+    }
+  }
+
+  @Override
+  @SuppressWarnings("FormatStringAnnotation")
+  public Dataset getDataset(DatasetReference datasetReference) {
+    try {
+      HttpResponse response =
+          client
+              .datasets()
+              .get(datasetReference.getProjectId(), datasetReference.getDatasetId())
+              .executeUnparsed();
+      if (response.getStatusCode() == HttpStatusCodes.STATUS_CODE_NOT_FOUND) {
+        throw new NoSuchNamespaceException(
+            "Namespace does not exist: " + datasetReference.getDatasetId());
+      }
+      return convertExceptionIfUnsuccessful(response).parseAs(Dataset.class);
+    } catch (IOException e) {
+      throw new RuntimeIOException(e);
+    }
+  }
+
+  @Override
+  @SuppressWarnings("FormatStringAnnotation")
+  public void deleteDataset(DatasetReference datasetReference) {
+    try {
+      HttpResponse response =
+          client
+              .datasets()
+              .delete(datasetReference.getProjectId(), datasetReference.getDatasetId())
+              .executeUnparsed();
+      if (response.getStatusCode() == HttpStatusCodes.STATUS_CODE_NOT_FOUND) {
+        throw new NoSuchNamespaceException(
+            "Namespace does not exist: " + datasetReference.getDatasetId());
+      }
+      convertExceptionIfUnsuccessful(response);
+    } catch (IOException e) {
+      throw new RuntimeIOException(e);
+    }
+  }
+
+  @Override
+  public Dataset setDatasetParameters(
+      DatasetReference datasetReference, Map<String, String> parameters) {
+    Dataset dataset = getDataset(datasetReference);
+    ExternalCatalogDatasetOptions externalCatalogDatasetOptions =
+        dataset.getExternalCatalogDatasetOptions() == null
+            ? new ExternalCatalogDatasetOptions()
+            : dataset.getExternalCatalogDatasetOptions();
+    Map<String, String> finalParameters =
+        externalCatalogDatasetOptions.getParameters() == null
+            ? Maps.newHashMap()
+            : externalCatalogDatasetOptions.getParameters();
+    finalParameters.putAll(parameters);
+
+    dataset.setExternalCatalogDatasetOptions(
+        externalCatalogDatasetOptions.setParameters(finalParameters));
+
+    return updateDataset(dataset);

Review Comment:
   What happens if the dataset is changed between L223-237 concurrently? Same for removeDatasetParameters. 



##########
bigquery/src/main/java/org/apache/iceberg/gcp/bigquery/FakeBigQueryMetaStoreClient.java:
##########
@@ -0,0 +1,228 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.iceberg.gcp.bigquery;
+
+import com.google.api.services.bigquery.model.Dataset;
+import com.google.api.services.bigquery.model.DatasetList;
+import com.google.api.services.bigquery.model.DatasetReference;
+import com.google.api.services.bigquery.model.ExternalCatalogDatasetOptions;
+import com.google.api.services.bigquery.model.Table;
+import com.google.api.services.bigquery.model.TableList;
+import com.google.api.services.bigquery.model.TableReference;
+import com.google.cloud.bigquery.BigQueryOptions;
+import java.io.IOException;
+import java.security.GeneralSecurityException;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.UUID;
+import java.util.stream.Collectors;
+import org.apache.iceberg.exceptions.AlreadyExistsException;
+import org.apache.iceberg.exceptions.CommitFailedException;
+import org.apache.iceberg.exceptions.NamespaceNotEmptyException;
+import org.apache.iceberg.exceptions.NoSuchNamespaceException;
+import org.apache.iceberg.exceptions.NoSuchTableException;
+import org.apache.iceberg.relocated.com.google.common.collect.Maps;
+
+public class FakeBigQueryMetaStoreClient implements BigQueryMetaStoreClient {
+  private final Map<DatasetReference, Dataset> datasets = Maps.newHashMap();
+  private final Map<TableReference, Table> tables = Maps.newHashMap();
+
+  public FakeBigQueryMetaStoreClient(BigQueryOptions options)
+      throws IOException, GeneralSecurityException {}
+
+  public Map<DatasetReference, Dataset> datasets() {
+    return datasets;
+  }
+
+  public Map<TableReference, Table> tables() {
+    return tables;
+  }

Review Comment:
   Unused methods. 



##########
bigquery/src/main/java/org/apache/iceberg/gcp/bigquery/BigQueryMetaStoreClientImpl.java:
##########
@@ -0,0 +1,620 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.iceberg.gcp.bigquery;
+
+import com.google.api.client.googleapis.javanet.GoogleNetHttpTransport;
+import com.google.api.client.http.HttpHeaders;
+import com.google.api.client.http.HttpResponse;
+import com.google.api.client.http.HttpResponseException;
+import com.google.api.client.http.HttpStatusCodes;
+import com.google.api.client.json.gson.GsonFactory;
+import com.google.api.client.util.Data;
+import com.google.api.services.bigquery.Bigquery;
+import com.google.api.services.bigquery.BigqueryScopes;
+import com.google.api.services.bigquery.model.Dataset;
+import com.google.api.services.bigquery.model.DatasetList;
+import com.google.api.services.bigquery.model.DatasetList.Datasets;
+import com.google.api.services.bigquery.model.DatasetReference;
+import com.google.api.services.bigquery.model.ExternalCatalogDatasetOptions;
+import com.google.api.services.bigquery.model.ExternalCatalogTableOptions;
+import com.google.api.services.bigquery.model.Table;
+import com.google.api.services.bigquery.model.TableList;
+import com.google.api.services.bigquery.model.TableList.Tables;
+import com.google.api.services.bigquery.model.TableReference;
+import com.google.api.services.bigquery.model.TableSchema;
+import com.google.auth.http.HttpCredentialsAdapter;
+import com.google.auth.oauth2.GoogleCredentials;
+import com.google.cloud.BaseServiceException;
+import com.google.cloud.ExceptionHandler;
+import com.google.cloud.bigquery.BigQueryErrorMessages;
+import com.google.cloud.bigquery.BigQueryOptions;
+import com.google.cloud.bigquery.BigQueryRetryConfig;
+import com.google.cloud.bigquery.BigQueryRetryHelper;
+import java.io.IOException;
+import java.nio.charset.StandardCharsets;
+import java.security.GeneralSecurityException;
+import java.util.List;
+import java.util.Locale;
+import java.util.Map;
+import java.util.Set;
+import java.util.concurrent.Callable;
+import java.util.stream.Collectors;
+import java.util.stream.Stream;
+import org.apache.iceberg.BaseMetastoreTableOperations;
+import org.apache.iceberg.exceptions.AlreadyExistsException;
+import org.apache.iceberg.exceptions.BadRequestException;
+import org.apache.iceberg.exceptions.ForbiddenException;
+import org.apache.iceberg.exceptions.NoSuchIcebergTableException;
+import org.apache.iceberg.exceptions.NoSuchNamespaceException;
+import org.apache.iceberg.exceptions.NoSuchTableException;
+import org.apache.iceberg.exceptions.NotAuthorizedException;
+import org.apache.iceberg.exceptions.RuntimeIOException;
+import org.apache.iceberg.exceptions.ServiceFailureException;
+import org.apache.iceberg.exceptions.ServiceUnavailableException;
+import org.apache.iceberg.exceptions.ValidationException;
+import org.apache.iceberg.relocated.com.google.common.base.Preconditions;
+import org.apache.iceberg.relocated.com.google.common.collect.Lists;
+import org.apache.iceberg.relocated.com.google.common.collect.Maps;
+
+/** A client of Google Bigquery Metastore functions over the BigQuery service. */
+public final class BigQueryMetaStoreClientImpl implements BigQueryMetaStoreClient {
+
+  public static final ExceptionHandler.Interceptor EXCEPTION_HANDLER_INTERCEPTOR =
+      new ExceptionHandler.Interceptor() {
+
+        private static final long serialVersionUID = -8429573486870467828L;
+
+        @Override
+        public RetryResult afterEval(Exception exception, RetryResult retryResult) {
+          return ExceptionHandler.Interceptor.RetryResult.CONTINUE_EVALUATION;
+        }
+
+        @Override
+        public RetryResult beforeEval(Exception exception) {
+          if (exception instanceof BaseServiceException) {
+            boolean retriable = ((BaseServiceException) exception).isRetryable();
+            return retriable
+                ? ExceptionHandler.Interceptor.RetryResult.RETRY
+                : ExceptionHandler.Interceptor.RetryResult.CONTINUE_EVALUATION;
+          }
+          return ExceptionHandler.Interceptor.RetryResult.CONTINUE_EVALUATION;
+        }
+      };
+
+  private final Bigquery client;
+  private final BigQueryOptions bigqueryOptions;
+
+  // Retry config with error messages and regex for rate limit exceeded errors.
+  private static final BigQueryRetryConfig DEFAULT_RETRY_CONFIG =
+      BigQueryRetryConfig.newBuilder()
+          .retryOnMessage(BigQueryErrorMessages.RATE_LIMIT_EXCEEDED_MSG)
+          .retryOnMessage(BigQueryErrorMessages.JOB_RATE_LIMIT_EXCEEDED_MSG)
+          .retryOnRegEx(BigQueryErrorMessages.RetryRegExPatterns.RATE_LIMIT_EXCEEDED_REGEX)
+          .build();
+
+  public static final ExceptionHandler BIGQUERY_EXCEPTION_HANDLER =
+      ExceptionHandler.newBuilder()
+          .abortOn(RuntimeException.class)
+          .retryOn(java.net.ConnectException.class) // retry on Connection Exception
+          .retryOn(java.net.UnknownHostException.class) // retry on UnknownHostException
+          .addInterceptors(EXCEPTION_HANDLER_INTERCEPTOR)
+          .build();
+
+  /** Constructs a client of the Google BigQuery service. */
+  public BigQueryMetaStoreClientImpl(BigQueryOptions options)
+      throws IOException, GeneralSecurityException {
+    // Initialize client that will be used to send requests. This client only needs to be created
+    // once, and can be reused for multiple requests
+    HttpCredentialsAdapter httpCredentialsAdapter =
+        new HttpCredentialsAdapter(
+            GoogleCredentials.getApplicationDefault().createScoped(BigqueryScopes.all()));
+    this.client =
+        new Bigquery.Builder(
+                GoogleNetHttpTransport.newTrustedTransport(),
+                GsonFactory.getDefaultInstance(),
+                httpRequest -> {
+                  httpCredentialsAdapter.initialize(httpRequest);
+                  // Instead of throwing exceptions, analyze the HttpResponse object and inspect its
+                  // status code. This will allow BigQuery API errors to be converted into Iceberg
+                  // exceptions.
+                  httpRequest.setThrowExceptionOnExecuteError(false);
+                })
+            .setApplicationName("BigQuery Metastore Iceberg Catalog Plugin")
+            .build();
+    this.bigqueryOptions = options;
+  }
+
+  @Override
+  public Dataset createDataset(Dataset dataset) {
+    // TODO (b/399885863): Ensure Dataset creation is idempotent when handling retries.
+    Dataset response = null;
+    try {
+      response =
+          BigQueryRetryHelper.runWithRetries(
+              new Callable<Dataset>() {
+                @Override
+                public Dataset call() {
+                  return create(dataset);
+                }
+              },
+              bigqueryOptions.getRetrySettings(),
+              BIGQUERY_EXCEPTION_HANDLER,
+              bigqueryOptions.getClock(),
+              DEFAULT_RETRY_CONFIG);
+    } catch (BigQueryRetryHelper.BigQueryRetryHelperException e) {
+      handleBigQueryRetryException(e);
+    }
+    return response;
+  }
+
+  @SuppressWarnings("FormatStringAnnotation")
+  private Dataset create(Dataset dataset) {
+    try {
+      HttpResponse response =
+          client
+              .datasets()
+              .insert(dataset.getDatasetReference().getProjectId(), dataset)
+              .executeUnparsed();
+      return convertExceptionIfUnsuccessful(response).parseAs(Dataset.class);
+    } catch (IOException e) {
+      throw new RuntimeIOException(e);
+    } catch (AlreadyExistsException e) {
+      throw new AlreadyExistsException("Namespace already exists: " + dataset.getId());
+    }
+  }
+
+  @Override
+  @SuppressWarnings("FormatStringAnnotation")
+  public Dataset getDataset(DatasetReference datasetReference) {
+    try {
+      HttpResponse response =
+          client
+              .datasets()
+              .get(datasetReference.getProjectId(), datasetReference.getDatasetId())
+              .executeUnparsed();
+      if (response.getStatusCode() == HttpStatusCodes.STATUS_CODE_NOT_FOUND) {
+        throw new NoSuchNamespaceException(
+            "Namespace does not exist: " + datasetReference.getDatasetId());
+      }
+      return convertExceptionIfUnsuccessful(response).parseAs(Dataset.class);
+    } catch (IOException e) {
+      throw new RuntimeIOException(e);
+    }
+  }
+
+  @Override
+  @SuppressWarnings("FormatStringAnnotation")
+  public void deleteDataset(DatasetReference datasetReference) {
+    try {
+      HttpResponse response =
+          client
+              .datasets()
+              .delete(datasetReference.getProjectId(), datasetReference.getDatasetId())
+              .executeUnparsed();
+      if (response.getStatusCode() == HttpStatusCodes.STATUS_CODE_NOT_FOUND) {
+        throw new NoSuchNamespaceException(
+            "Namespace does not exist: " + datasetReference.getDatasetId());
+      }
+      convertExceptionIfUnsuccessful(response);
+    } catch (IOException e) {
+      throw new RuntimeIOException(e);
+    }
+  }
+
+  @Override
+  public Dataset setDatasetParameters(
+      DatasetReference datasetReference, Map<String, String> parameters) {
+    Dataset dataset = getDataset(datasetReference);
+    ExternalCatalogDatasetOptions externalCatalogDatasetOptions =
+        dataset.getExternalCatalogDatasetOptions() == null
+            ? new ExternalCatalogDatasetOptions()
+            : dataset.getExternalCatalogDatasetOptions();
+    Map<String, String> finalParameters =
+        externalCatalogDatasetOptions.getParameters() == null
+            ? Maps.newHashMap()
+            : externalCatalogDatasetOptions.getParameters();
+    finalParameters.putAll(parameters);
+
+    dataset.setExternalCatalogDatasetOptions(
+        externalCatalogDatasetOptions.setParameters(finalParameters));
+
+    return updateDataset(dataset);
+  }
+
+  @Override
+  public Dataset removeDatasetParameters(
+      DatasetReference datasetReference, Set<String> parameters) {
+    Dataset dataset = getDataset(datasetReference);
+    ExternalCatalogDatasetOptions externalCatalogDatasetOptions =
+        dataset.getExternalCatalogDatasetOptions() == null
+            ? new ExternalCatalogDatasetOptions()
+            : dataset.getExternalCatalogDatasetOptions();
+    Map<String, String> finalParameters =
+        externalCatalogDatasetOptions.getParameters() == null
+            ? Maps.newHashMap()
+            : externalCatalogDatasetOptions.getParameters();
+    parameters.forEach(finalParameters::remove);
+
+    dataset.setExternalCatalogDatasetOptions(
+        externalCatalogDatasetOptions.setParameters(finalParameters));
+
+    return updateDataset(dataset);
+  }
+
+  @Override
+  public List<Datasets> listDatasets(String projectId) {
+    try {
+      String nextPageToken = null;
+      List<Datasets> datasets = Lists.newArrayList();
+      do {
+        HttpResponse pageResponse =
+            client.datasets().list(projectId).setPageToken(nextPageToken).executeUnparsed();
+        DatasetList result =
+            convertExceptionIfUnsuccessful(pageResponse).parseAs(DatasetList.class);
+        nextPageToken = result.getNextPageToken();
+        if (result.getDatasets() != null) {
+          datasets.addAll(result.getDatasets());
+        }
+      } while (nextPageToken != null && !nextPageToken.isEmpty());
+      return datasets;
+    } catch (IOException e) {
+      throw new RuntimeIOException(e);
+    }
+  }
+
+  @Override
+  public Table createTable(Table table) {
+    // Ensure it is an Iceberg table supported by the BigQuery metastore catalog.
+    validateTable(table);
+    // TODO (b/399885863): Ensure table creation is idempotent when handling retries.
+    Table response = null;
+    try {
+      response =
+          BigQueryRetryHelper.runWithRetries(
+              new Callable<Table>() {
+                @Override
+                public Table call() {
+                  return create(table);
+                }
+              },
+              bigqueryOptions.getRetrySettings(),
+              BIGQUERY_EXCEPTION_HANDLER,
+              bigqueryOptions.getClock(),
+              DEFAULT_RETRY_CONFIG);
+    } catch (BigQueryRetryHelper.BigQueryRetryHelperException e) {
+      handleBigQueryRetryException(e);
+    }
+    return response;
+  }
+
+  @SuppressWarnings("FormatStringAnnotation")
+  private Table create(Table table) {
+    try {
+      HttpResponse response =
+          client
+              .tables()
+              .insert(
+                  Preconditions.checkNotNull(table.getTableReference()).getProjectId(),
+                  Preconditions.checkNotNull(table.getTableReference()).getDatasetId(),
+                  table)
+              .executeUnparsed();
+      return convertExceptionIfUnsuccessful(response).parseAs(Table.class);
+    } catch (IOException e) {
+      throw new RuntimeIOException(e);
+    } catch (AlreadyExistsException e) {
+      throw new AlreadyExistsException("Table already exists", e);

Review Comment:
   Can we include the table name in the error message? 



##########
bigquery/src/main/java/org/apache/iceberg/gcp/bigquery/BigQueryMetastoreCatalog.java:
##########
@@ -0,0 +1,403 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.iceberg.gcp.bigquery;
+
+import com.google.api.services.bigquery.model.Dataset;
+import com.google.api.services.bigquery.model.DatasetList.Datasets;
+import com.google.api.services.bigquery.model.DatasetReference;
+import com.google.api.services.bigquery.model.ExternalCatalogDatasetOptions;
+import com.google.api.services.bigquery.model.TableReference;
+import com.google.cloud.ServiceOptions;
+import com.google.cloud.bigquery.BigQueryOptions;
+import java.io.IOException;
+import java.security.GeneralSecurityException;
+import java.util.List;
+import java.util.Locale;
+import java.util.Map;
+import java.util.Objects;
+import java.util.Set;
+import org.apache.hadoop.conf.Configurable;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.iceberg.BaseMetastoreCatalog;
+import org.apache.iceberg.CatalogProperties;
+import org.apache.iceberg.CatalogUtil;
+import org.apache.iceberg.TableMetadata;
+import org.apache.iceberg.TableOperations;
+import org.apache.iceberg.catalog.Namespace;
+import org.apache.iceberg.catalog.SupportsNamespaces;
+import org.apache.iceberg.catalog.TableIdentifier;
+import org.apache.iceberg.exceptions.AlreadyExistsException;
+import org.apache.iceberg.exceptions.NoSuchNamespaceException;
+import org.apache.iceberg.exceptions.NoSuchTableException;
+import org.apache.iceberg.exceptions.ServiceFailureException;
+import org.apache.iceberg.exceptions.ValidationException;
+import org.apache.iceberg.io.FileIO;
+import org.apache.iceberg.relocated.com.google.common.annotations.VisibleForTesting;
+import org.apache.iceberg.relocated.com.google.common.base.Preconditions;
+import org.apache.iceberg.relocated.com.google.common.base.Strings;
+import org.apache.iceberg.relocated.com.google.common.collect.ImmutableList;
+import org.apache.iceberg.relocated.com.google.common.collect.ImmutableMap;
+import org.apache.iceberg.relocated.com.google.common.collect.Maps;
+import org.apache.iceberg.util.LocationUtil;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/** Iceberg Bigquery Metastore Catalog implementation. */
+public final class BigQueryMetastoreCatalog extends BaseMetastoreCatalog
+    implements SupportsNamespaces, Configurable {
+
+  // User provided properties.
+  public static final String PROPERTIES_KEY_GCP_PROJECT = "gcp-project";
+  public static final String PROPERTIES_KEY_GCP_LOCATION = "gcp-location";
+  public static final String PROPERTIES_KEY_FILTER_UNSUPPORTED_TABLES = "filter-unsupported-tables";
+  public static final String PROPERTIES_KEY_TESTING_ENABLED = "testing-enabled";
+
+  public static final String HIVE_METASTORE_WAREHOUSE_DIR = "hive.metastore.warehouse.dir";
+
+  private static final Logger LOG = LoggerFactory.getLogger(BigQueryMetastoreCatalog.class);
+
+  private static final String DEFAULT_GCP_LOCATION = "us";
+
+  private String catalogPluginName;
+  private Map<String, String> catalogProperties;
+  private FileIO fileIO;
+  private Configuration conf;
+  private String projectId;
+  private String location;
+  private BigQueryMetaStoreClient client;
+  private boolean filterUnsupportedTables;
+
+  // Must have a no-arg constructor to be dynamically loaded
+  // initialize(String name, Map<String, String> properties) will be called to complete
+  // initialization
+  public BigQueryMetastoreCatalog() {}
+
+  @Override
+  public void initialize(String inputName, Map<String, String> properties) {
+    if (!properties.containsKey(PROPERTIES_KEY_GCP_PROJECT)) {
+      throw new ValidationException("GCP project must be specified");
+    }
+    projectId = properties.get(PROPERTIES_KEY_GCP_PROJECT);
+    location = properties.getOrDefault(PROPERTIES_KEY_GCP_LOCATION, DEFAULT_GCP_LOCATION);
+    boolean testingEnabled =
+        Boolean.parseBoolean(properties.getOrDefault(PROPERTIES_KEY_TESTING_ENABLED, "false"));
+
+    BigQueryOptions options =
+        BigQueryOptions.newBuilder()
+            .setProjectId(projectId)
+            .setLocation(location)
+            .setRetrySettings(ServiceOptions.getDefaultRetrySettings())
+            .build();
+
+    try {
+      if (testingEnabled) {
+        client = new FakeBigQueryMetaStoreClient(options);
+
+      } else {
+        client = new BigQueryMetaStoreClientImpl(options);
+      }
+    } catch (IOException e) {
+      throw new ServiceFailureException(e, "Creating BigQuery client failed");
+    } catch (GeneralSecurityException e) {
+      throw new ValidationException(e, "Creating BigQuery client failed due to a security issue");
+    }
+    initialize(inputName, properties, projectId, location, client);
+  }
+
+  @VisibleForTesting
+  void initialize(
+      String inputName,
+      Map<String, String> properties,
+      String initialProjectId,
+      String initialLocation,
+      BigQueryMetaStoreClient bigQueryMetaStoreClient) {
+    this.catalogPluginName = inputName;
+    this.catalogProperties = ImmutableMap.copyOf(properties);
+    this.projectId = initialProjectId;
+    this.location = initialLocation;
+    this.client =
+        Preconditions.checkNotNull(bigQueryMetaStoreClient, "BigQuery client can not be null");
+
+    if (this.conf == null) {
+      LOG.warn("No configuration was set, using the default environment Configuration");
+      this.conf = new Configuration();
+    }
+
+    LOG.info("Using BigQuery Metastore Iceberg Catalog: {}", inputName);
+
+    if (properties.containsKey(CatalogProperties.WAREHOUSE_LOCATION)) {
+      // Iceberg always removes trailing slash to avoid paths like "<folder>//data" in file systems
+      // like s3.
+      this.conf.set(
+          HIVE_METASTORE_WAREHOUSE_DIR,
+          LocationUtil.stripTrailingSlash(properties.get(CatalogProperties.WAREHOUSE_LOCATION)));
+    }
+
+    String fileIoImpl =
+        properties.getOrDefault(
+            CatalogProperties.FILE_IO_IMPL, "org.apache.iceberg.hadoop.HadoopFileIO");
+    this.fileIO = CatalogUtil.loadFileIO(fileIoImpl, properties, conf);
+
+    this.filterUnsupportedTables =
+        Boolean.parseBoolean(
+            properties.getOrDefault(PROPERTIES_KEY_FILTER_UNSUPPORTED_TABLES, "false"));
+  }
+
+  @Override
+  protected TableOperations newTableOps(TableIdentifier identifier) {
+    return new BigQueryTableOperations(
+        client,
+        fileIO,
+        projectId,
+        // Sometimes extensions have the namespace contain the table name too, so we are forced to
+        // allow invalid namespace and just take the first part here like other catalog
+        // implementations do.
+        identifier.namespace().level(0),
+        identifier.name(),
+        conf);
+  }
+
+  @Override
+  protected String defaultWarehouseLocation(TableIdentifier identifier) {
+    String locationUri = null;
+    DatasetReference datasetReference = toDatasetReference(identifier.namespace());
+    Dataset dataset = client.getDataset(datasetReference);
+    if (dataset != null && dataset.getExternalCatalogDatasetOptions() != null) {
+      locationUri = dataset.getExternalCatalogDatasetOptions().getDefaultStorageLocationUri();
+    }
+    return String.format(
+        "%s/%s",
+        Strings.isNullOrEmpty(locationUri) ? datasetReference.getDatasetId() : locationUri,

Review Comment:
   Does `locationUri` never end with `/`? 



-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org

For queries about this service, please contact Infrastructure at:
users@infra.apache.org


---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For additional commands, e-mail: issues-help@iceberg.apache.org


Re: [PR] Catalog: Add BigQuery Metastore Catalog Support [iceberg]

Posted by "kravikumar (via GitHub)" <gi...@apache.org>.
kravikumar commented on code in PR #12808:
URL: https://github.com/apache/iceberg/pull/12808#discussion_r2047405893


##########
bigquery/src/main/java/org/apache/iceberg/gcp/bigquery/BigQueryMetaStoreClient.java:
##########
@@ -0,0 +1,133 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.iceberg.gcp.bigquery;
+
+import com.google.api.services.bigquery.model.Dataset;
+import com.google.api.services.bigquery.model.DatasetList.Datasets;
+import com.google.api.services.bigquery.model.DatasetReference;
+import com.google.api.services.bigquery.model.Table;
+import com.google.api.services.bigquery.model.TableList.Tables;
+import com.google.api.services.bigquery.model.TableReference;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+
+/**
+ * A client of Google BigQuery Metastore functions over the BigQuery service. Uses the Google
+ * BigQuery API.
+ */
+/**

Review Comment:
   nit: Can we remove this duplicate comment



##########
bigquery/src/main/java/org/apache/iceberg/gcp/bigquery/BigQueryMetaStoreClientImpl.java:
##########
@@ -0,0 +1,620 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.iceberg.gcp.bigquery;
+
+import com.google.api.client.googleapis.javanet.GoogleNetHttpTransport;
+import com.google.api.client.http.HttpHeaders;
+import com.google.api.client.http.HttpResponse;
+import com.google.api.client.http.HttpResponseException;
+import com.google.api.client.http.HttpStatusCodes;
+import com.google.api.client.json.gson.GsonFactory;
+import com.google.api.client.util.Data;
+import com.google.api.services.bigquery.Bigquery;
+import com.google.api.services.bigquery.BigqueryScopes;
+import com.google.api.services.bigquery.model.Dataset;
+import com.google.api.services.bigquery.model.DatasetList;
+import com.google.api.services.bigquery.model.DatasetList.Datasets;
+import com.google.api.services.bigquery.model.DatasetReference;
+import com.google.api.services.bigquery.model.ExternalCatalogDatasetOptions;
+import com.google.api.services.bigquery.model.ExternalCatalogTableOptions;
+import com.google.api.services.bigquery.model.Table;
+import com.google.api.services.bigquery.model.TableList;
+import com.google.api.services.bigquery.model.TableList.Tables;
+import com.google.api.services.bigquery.model.TableReference;
+import com.google.api.services.bigquery.model.TableSchema;
+import com.google.auth.http.HttpCredentialsAdapter;
+import com.google.auth.oauth2.GoogleCredentials;
+import com.google.cloud.BaseServiceException;
+import com.google.cloud.ExceptionHandler;
+import com.google.cloud.bigquery.BigQueryErrorMessages;
+import com.google.cloud.bigquery.BigQueryOptions;
+import com.google.cloud.bigquery.BigQueryRetryConfig;
+import com.google.cloud.bigquery.BigQueryRetryHelper;
+import java.io.IOException;
+import java.nio.charset.StandardCharsets;
+import java.security.GeneralSecurityException;
+import java.util.List;
+import java.util.Locale;
+import java.util.Map;
+import java.util.Set;
+import java.util.concurrent.Callable;
+import java.util.stream.Collectors;
+import java.util.stream.Stream;
+import org.apache.iceberg.BaseMetastoreTableOperations;
+import org.apache.iceberg.exceptions.AlreadyExistsException;
+import org.apache.iceberg.exceptions.BadRequestException;
+import org.apache.iceberg.exceptions.ForbiddenException;
+import org.apache.iceberg.exceptions.NoSuchIcebergTableException;
+import org.apache.iceberg.exceptions.NoSuchNamespaceException;
+import org.apache.iceberg.exceptions.NoSuchTableException;
+import org.apache.iceberg.exceptions.NotAuthorizedException;
+import org.apache.iceberg.exceptions.RuntimeIOException;
+import org.apache.iceberg.exceptions.ServiceFailureException;
+import org.apache.iceberg.exceptions.ServiceUnavailableException;
+import org.apache.iceberg.exceptions.ValidationException;
+import org.apache.iceberg.relocated.com.google.common.base.Preconditions;
+import org.apache.iceberg.relocated.com.google.common.collect.Lists;
+import org.apache.iceberg.relocated.com.google.common.collect.Maps;
+
+/** A client of Google Bigquery Metastore functions over the BigQuery service. */
+public final class BigQueryMetaStoreClientImpl implements BigQueryMetaStoreClient {
+
+  public static final ExceptionHandler.Interceptor EXCEPTION_HANDLER_INTERCEPTOR =
+      new ExceptionHandler.Interceptor() {
+
+        private static final long serialVersionUID = -8429573486870467828L;
+
+        @Override
+        public RetryResult afterEval(Exception exception, RetryResult retryResult) {
+          return ExceptionHandler.Interceptor.RetryResult.CONTINUE_EVALUATION;
+        }
+
+        @Override
+        public RetryResult beforeEval(Exception exception) {
+          if (exception instanceof BaseServiceException) {
+            boolean retriable = ((BaseServiceException) exception).isRetryable();
+            return retriable
+                ? ExceptionHandler.Interceptor.RetryResult.RETRY
+                : ExceptionHandler.Interceptor.RetryResult.CONTINUE_EVALUATION;
+          }
+          return ExceptionHandler.Interceptor.RetryResult.CONTINUE_EVALUATION;
+        }
+      };
+
+  private final Bigquery client;
+  private final BigQueryOptions bigqueryOptions;
+
+  // Retry config with error messages and regex for rate limit exceeded errors.
+  private static final BigQueryRetryConfig DEFAULT_RETRY_CONFIG =
+      BigQueryRetryConfig.newBuilder()
+          .retryOnMessage(BigQueryErrorMessages.RATE_LIMIT_EXCEEDED_MSG)
+          .retryOnMessage(BigQueryErrorMessages.JOB_RATE_LIMIT_EXCEEDED_MSG)
+          .retryOnRegEx(BigQueryErrorMessages.RetryRegExPatterns.RATE_LIMIT_EXCEEDED_REGEX)
+          .build();
+
+  public static final ExceptionHandler BIGQUERY_EXCEPTION_HANDLER =
+      ExceptionHandler.newBuilder()
+          .abortOn(RuntimeException.class)
+          .retryOn(java.net.ConnectException.class) // retry on Connection Exception
+          .retryOn(java.net.UnknownHostException.class) // retry on UnknownHostException
+          .addInterceptors(EXCEPTION_HANDLER_INTERCEPTOR)
+          .build();
+
+  /** Constructs a client of the Google BigQuery service. */
+  public BigQueryMetaStoreClientImpl(BigQueryOptions options)
+      throws IOException, GeneralSecurityException {
+    // Initialize client that will be used to send requests. This client only needs to be created
+    // once, and can be reused for multiple requests
+    HttpCredentialsAdapter httpCredentialsAdapter =
+        new HttpCredentialsAdapter(
+            GoogleCredentials.getApplicationDefault().createScoped(BigqueryScopes.all()));
+    this.client =
+        new Bigquery.Builder(
+                GoogleNetHttpTransport.newTrustedTransport(),
+                GsonFactory.getDefaultInstance(),
+                httpRequest -> {
+                  httpCredentialsAdapter.initialize(httpRequest);
+                  // Instead of throwing exceptions, analyze the HttpResponse object and inspect its
+                  // status code. This will allow BigQuery API errors to be converted into Iceberg
+                  // exceptions.
+                  httpRequest.setThrowExceptionOnExecuteError(false);
+                })
+            .setApplicationName("BigQuery Iceberg Catalog Plugin")
+            .build();
+    this.bigqueryOptions = options;
+  }
+
+  @Override
+  public Dataset createDataset(Dataset dataset) {
+    // TODO (b/399885863): Ensure Dataset creation is idempotent when handling retries.
+    Dataset response = null;
+    try {
+      response =
+          BigQueryRetryHelper.runWithRetries(
+              new Callable<Dataset>() {
+                @Override
+                public Dataset call() {
+                  return create(dataset);
+                }
+              },
+              bigqueryOptions.getRetrySettings(),
+              BIGQUERY_EXCEPTION_HANDLER,
+              bigqueryOptions.getClock(),
+              DEFAULT_RETRY_CONFIG);
+    } catch (BigQueryRetryHelper.BigQueryRetryHelperException e) {
+      handleBigQueryRetryException(e);
+    }
+    return response;
+  }
+
+  @SuppressWarnings("FormatStringAnnotation")
+  private Dataset create(Dataset dataset) {
+    try {
+      HttpResponse response =
+          client
+              .datasets()
+              .insert(dataset.getDatasetReference().getProjectId(), dataset)
+              .executeUnparsed();
+      return convertExceptionIfUnsuccessful(response).parseAs(Dataset.class);
+    } catch (IOException e) {
+      throw new RuntimeIOException(e);
+    } catch (AlreadyExistsException e) {
+      throw new AlreadyExistsException("Namespace already exists: " + dataset.getId());
+    }
+  }
+
+  @Override
+  @SuppressWarnings("FormatStringAnnotation")
+  public Dataset getDataset(DatasetReference datasetReference) {
+    try {
+      HttpResponse response =
+          client
+              .datasets()
+              .get(datasetReference.getProjectId(), datasetReference.getDatasetId())
+              .executeUnparsed();
+      if (response.getStatusCode() == HttpStatusCodes.STATUS_CODE_NOT_FOUND) {
+        throw new NoSuchNamespaceException(
+            "Namespace does not exist: " + datasetReference.getDatasetId());
+      }
+      return convertExceptionIfUnsuccessful(response).parseAs(Dataset.class);
+    } catch (IOException e) {
+      throw new RuntimeIOException(e);
+    }
+  }
+
+  @Override
+  @SuppressWarnings("FormatStringAnnotation")
+  public void deleteDataset(DatasetReference datasetReference) {
+    try {
+      HttpResponse response =
+          client
+              .datasets()
+              .delete(datasetReference.getProjectId(), datasetReference.getDatasetId())
+              .executeUnparsed();
+      if (response.getStatusCode() == HttpStatusCodes.STATUS_CODE_NOT_FOUND) {
+        throw new NoSuchNamespaceException(
+            "Namespace does not exist: " + datasetReference.getDatasetId());
+      }
+      convertExceptionIfUnsuccessful(response);
+    } catch (IOException e) {
+      throw new RuntimeIOException(e);
+    }
+  }
+
+  @Override
+  public Dataset setDatasetParameters(
+      DatasetReference datasetReference, Map<String, String> parameters) {
+    Dataset dataset = getDataset(datasetReference);
+    ExternalCatalogDatasetOptions externalCatalogDatasetOptions =
+        dataset.getExternalCatalogDatasetOptions() == null
+            ? new ExternalCatalogDatasetOptions()
+            : dataset.getExternalCatalogDatasetOptions();
+    Map<String, String> finalParameters =
+        externalCatalogDatasetOptions.getParameters() == null
+            ? Maps.newHashMap()
+            : externalCatalogDatasetOptions.getParameters();
+    finalParameters.putAll(parameters);
+
+    dataset.setExternalCatalogDatasetOptions(
+        externalCatalogDatasetOptions.setParameters(finalParameters));
+
+    return updateDataset(dataset);
+  }
+
+  @Override
+  public Dataset removeDatasetParameters(
+      DatasetReference datasetReference, Set<String> parameters) {
+    Dataset dataset = getDataset(datasetReference);
+    ExternalCatalogDatasetOptions externalCatalogDatasetOptions =
+        dataset.getExternalCatalogDatasetOptions() == null
+            ? new ExternalCatalogDatasetOptions()
+            : dataset.getExternalCatalogDatasetOptions();
+    Map<String, String> finalParameters =
+        externalCatalogDatasetOptions.getParameters() == null
+            ? Maps.newHashMap()
+            : externalCatalogDatasetOptions.getParameters();
+    parameters.forEach(finalParameters::remove);
+
+    dataset.setExternalCatalogDatasetOptions(
+        externalCatalogDatasetOptions.setParameters(finalParameters));
+
+    return updateDataset(dataset);
+  }
+
+  @Override
+  public List<Datasets> listDatasets(String projectId) {
+    try {
+      String nextPageToken = null;
+      List<Datasets> datasets = Lists.newArrayList();
+      do {
+        HttpResponse pageResponse =
+            client.datasets().list(projectId).setPageToken(nextPageToken).executeUnparsed();
+        DatasetList result =
+            convertExceptionIfUnsuccessful(pageResponse).parseAs(DatasetList.class);
+        nextPageToken = result.getNextPageToken();
+        if (result.getDatasets() != null) {
+          datasets.addAll(result.getDatasets());
+        }
+      } while (nextPageToken != null && !nextPageToken.isEmpty());
+      return datasets;
+    } catch (IOException e) {
+      throw new RuntimeIOException(e);
+    }
+  }
+
+  @Override
+  public Table createTable(Table table) {
+    // Ensure it is an Iceberg table supported by the BQ metastore catalog.
+    validateTable(table);
+    // TODO (b/399885863): Ensure table creation is idempotent when handling retries.

Review Comment:
   nit: Here and in other places - We might want to remove references to **b/** 



##########
bigquery/src/main/java/org/apache/iceberg/gcp/bigquery/BigQueryMetastoreCatalog.java:
##########
@@ -0,0 +1,403 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.iceberg.gcp.bigquery;
+
+import com.google.api.services.bigquery.model.Dataset;
+import com.google.api.services.bigquery.model.DatasetList.Datasets;
+import com.google.api.services.bigquery.model.DatasetReference;
+import com.google.api.services.bigquery.model.ExternalCatalogDatasetOptions;
+import com.google.api.services.bigquery.model.TableReference;
+import com.google.cloud.ServiceOptions;
+import com.google.cloud.bigquery.BigQueryOptions;
+import java.io.IOException;
+import java.security.GeneralSecurityException;
+import java.util.List;
+import java.util.Locale;
+import java.util.Map;
+import java.util.Objects;
+import java.util.Set;
+import org.apache.hadoop.conf.Configurable;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.iceberg.BaseMetastoreCatalog;
+import org.apache.iceberg.CatalogProperties;
+import org.apache.iceberg.CatalogUtil;
+import org.apache.iceberg.TableMetadata;
+import org.apache.iceberg.TableOperations;
+import org.apache.iceberg.catalog.Namespace;
+import org.apache.iceberg.catalog.SupportsNamespaces;
+import org.apache.iceberg.catalog.TableIdentifier;
+import org.apache.iceberg.exceptions.AlreadyExistsException;
+import org.apache.iceberg.exceptions.NoSuchNamespaceException;
+import org.apache.iceberg.exceptions.NoSuchTableException;
+import org.apache.iceberg.exceptions.ServiceFailureException;
+import org.apache.iceberg.exceptions.ValidationException;
+import org.apache.iceberg.io.FileIO;
+import org.apache.iceberg.relocated.com.google.common.annotations.VisibleForTesting;
+import org.apache.iceberg.relocated.com.google.common.base.Preconditions;
+import org.apache.iceberg.relocated.com.google.common.base.Strings;
+import org.apache.iceberg.relocated.com.google.common.collect.ImmutableList;
+import org.apache.iceberg.relocated.com.google.common.collect.ImmutableMap;
+import org.apache.iceberg.relocated.com.google.common.collect.Maps;
+import org.apache.iceberg.util.LocationUtil;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/** Iceberg Bigquery Metastore Catalog implementation. */
+public final class BigQueryMetastoreCatalog extends BaseMetastoreCatalog
+    implements SupportsNamespaces, Configurable {
+
+  // User provided properties.
+  public static final String PROPERTIES_KEY_GCP_PROJECT = "gcp-project";
+  public static final String PROPERTIES_KEY_GCP_LOCATION = "gcp-location";
+  public static final String PROPERTIES_KEY_FILTER_UNSUPPORTED_TABLES = "filter-unsupported-tables";
+  public static final String PROPERTIES_KEY_TESTING_ENABLED = "testing-enabled";
+
+  public static final String HIVE_METASTORE_WAREHOUSE_DIR = "hive.metastore.warehouse.dir";
+
+  private static final Logger LOG = LoggerFactory.getLogger(BigQueryMetastoreCatalog.class);
+
+  private static final String DEFAULT_GCP_LOCATION = "us";
+
+  private String catalogPluginName;
+  private Map<String, String> catalogProperties;
+  private FileIO fileIO;
+  private Configuration conf;
+  private String projectId;
+  private String location;
+  private BigQueryMetaStoreClient client;
+  private boolean filterUnsupportedTables;
+
+  // Must have a no-arg constructor to be dynamically loaded
+  // initialize(String name, Map<String, String> properties) will be called to complete
+  // initialization
+  public BigQueryMetastoreCatalog() {}
+
+  @Override
+  public void initialize(String inputName, Map<String, String> properties) {
+    if (!properties.containsKey(PROPERTIES_KEY_GCP_PROJECT)) {
+      throw new ValidationException("GCP project must be specified");
+    }
+    projectId = properties.get(PROPERTIES_KEY_GCP_PROJECT);
+    location = properties.getOrDefault(PROPERTIES_KEY_GCP_LOCATION, DEFAULT_GCP_LOCATION);
+    boolean testingEnabled =
+        Boolean.parseBoolean(properties.getOrDefault(PROPERTIES_KEY_TESTING_ENABLED, "false"));
+
+    BigQueryOptions options =
+        BigQueryOptions.newBuilder()
+            .setProjectId(projectId)
+            .setLocation(location)
+            .setRetrySettings(ServiceOptions.getDefaultRetrySettings())
+            .build();
+
+    try {
+      if (testingEnabled) {
+        client = new FakeBigQueryMetaStoreClient(options);
+
+      } else {
+        client = new BigQueryMetaStoreClientImpl(options);
+      }
+    } catch (IOException e) {
+      throw new ServiceFailureException(e, "Creating BigQuery client failed");
+    } catch (GeneralSecurityException e) {
+      throw new ValidationException(e, "Creating BigQuery client failed due to a security issue");
+    }
+    initialize(inputName, properties, projectId, location, client);
+  }
+
+  @VisibleForTesting
+  void initialize(
+      String inputName,
+      Map<String, String> properties,
+      String initialProjectId,
+      String initialLocation,
+      BigQueryMetaStoreClient bigQueryMetaStoreClient) {
+    this.catalogPluginName = inputName;
+    this.catalogProperties = ImmutableMap.copyOf(properties);
+    this.projectId = initialProjectId;
+    this.location = initialLocation;
+    this.client =
+        Preconditions.checkNotNull(bigQueryMetaStoreClient, "BigQuery client can not be null");
+
+    if (this.conf == null) {
+      LOG.warn("No configuration was set, using the default environment Configuration");
+      this.conf = new Configuration();
+    }
+
+    LOG.info("Using BigQuery Metastore Iceberg Catalog: {}", inputName);
+
+    if (properties.containsKey(CatalogProperties.WAREHOUSE_LOCATION)) {
+      // Iceberg always removes trailing slash to avoid paths like "<folder>//data" in file systems
+      // like s3.
+      this.conf.set(
+          HIVE_METASTORE_WAREHOUSE_DIR,
+          LocationUtil.stripTrailingSlash(properties.get(CatalogProperties.WAREHOUSE_LOCATION)));
+    }
+
+    String fileIoImpl =
+        properties.getOrDefault(
+            CatalogProperties.FILE_IO_IMPL, "org.apache.iceberg.hadoop.HadoopFileIO");

Review Comment:
   Can we default to `org.apache.iceberg.gcp.gcs.GCSFileIO` ? We will also need to update our documentation to indicate that `iceberg-gcp-bundle` be passed as a runtime jar if we are defaulting to `GCSFileIO`



-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org

For queries about this service, please contact Infrastructure at:
users@infra.apache.org


---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For additional commands, e-mail: issues-help@iceberg.apache.org


Re: [PR] Catalog: Add BigQuery Metastore Catalog Support [iceberg]

Posted by "talatuyarer (via GitHub)" <gi...@apache.org>.
talatuyarer commented on code in PR #12808:
URL: https://github.com/apache/iceberg/pull/12808#discussion_r2051080534


##########
bigquery/src/main/java/org/apache/iceberg/gcp/bigquery/BigQueryMetaStoreClient.java:
##########
@@ -0,0 +1,129 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.iceberg.gcp.bigquery;
+
+import com.google.api.services.bigquery.model.Dataset;
+import com.google.api.services.bigquery.model.DatasetList.Datasets;
+import com.google.api.services.bigquery.model.DatasetReference;
+import com.google.api.services.bigquery.model.Table;
+import com.google.api.services.bigquery.model.TableList.Tables;
+import com.google.api.services.bigquery.model.TableReference;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+
+/**
+ * A client of Google BigQuery Metastore functions over the BigQuery service. Uses the Google
+ * BigQuery API.
+ */
+public interface BigQueryMetaStoreClient {
+
+  /**
+   * Creates and returns a new dataset.
+   *
+   * @param dataset the dataset to create
+   */
+  Dataset createDataset(Dataset dataset);

Review Comment:
   I dont see any harm for that. Do you ? 



-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org

For queries about this service, please contact Infrastructure at:
users@infra.apache.org


---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For additional commands, e-mail: issues-help@iceberg.apache.org


Re: [PR] Catalog: Add BigQuery Metastore Catalog Support [iceberg]

Posted by "nastra (via GitHub)" <gi...@apache.org>.
nastra commented on code in PR #12808:
URL: https://github.com/apache/iceberg/pull/12808#discussion_r2055539916


##########
bigquery/src/test/java/org/apache/iceberg/gcp/bigquery/TestBigQueryCatalog.java:
##########
@@ -0,0 +1,673 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.iceberg.gcp.bigquery;
+
+import static org.apache.iceberg.CatalogUtil.ICEBERG_CATALOG_TYPE;
+import static org.apache.iceberg.CatalogUtil.ICEBERG_CATALOG_TYPE_BIGQUERY;
+import static org.apache.iceberg.gcp.bigquery.BigQueryMetastoreCatalog.PROJECT_ID;
+import static org.apache.iceberg.gcp.bigquery.BigQueryMetastoreCatalog.TESTING_ENABLED;
+import static org.apache.iceberg.types.Types.NestedField.optional;
+import static org.assertj.core.api.Assertions.assertThat;
+import static org.assertj.core.api.Assertions.assertThatThrownBy;
+
+import java.io.File;
+import java.util.List;
+import java.util.Map;
+import org.apache.iceberg.CatalogProperties;
+import org.apache.iceberg.CatalogUtil;
+import org.apache.iceberg.PartitionSpec;
+import org.apache.iceberg.Schema;
+import org.apache.iceberg.SortOrder;
+import org.apache.iceberg.Table;
+import org.apache.iceberg.TableUtil;
+import org.apache.iceberg.Transaction;
+import org.apache.iceberg.catalog.CatalogTests;
+import org.apache.iceberg.catalog.Namespace;
+import org.apache.iceberg.catalog.TableIdentifier;
+import org.apache.iceberg.exceptions.NoSuchNamespaceException;
+import org.apache.iceberg.exceptions.NoSuchTableException;
+import org.apache.iceberg.expressions.Expressions;
+import org.apache.iceberg.relocated.com.google.common.collect.ImmutableMap;
+import org.apache.iceberg.types.Types;
+import org.junit.jupiter.api.AfterEach;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Disabled;
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.io.TempDir;
+import org.junit.jupiter.params.ParameterizedTest;
+import org.junit.jupiter.params.provider.ValueSource;
+
+public class TestBigQueryCatalog extends CatalogTests<BigQueryMetastoreCatalog> {
+
+  private static final String GCP_PROJECT = "project-id";
+  private static final String CATALOG_ID = "catalog-name";
+
+  // another schema that is not the same
+  protected static final Schema OTHER_SCHEMA_OPTIONAL =
+      new Schema(optional(1, "some_id", Types.IntegerType.get()));
+
+  // Schema passed to create tables
+  protected static final Schema SCHEMA_OPTIONAL =
+      new Schema(
+          optional(3, "id", Types.IntegerType.get(), "unique ID 🤪"),
+          optional(4, "data", Types.StringType.get()));
+
+  // This is the actual schema for the table, with column IDs reassigned
+  protected static final Schema REPLACE_SCHEMA_OPTIONAL =
+      new Schema(
+          optional(2, "id", Types.IntegerType.get(), "unique ID 🤪"),
+          optional(3, "data", Types.StringType.get()));
+
+  protected static final PartitionSpec REPLACE_SPEC_OPTIONAL =
+      PartitionSpec.builderFor(REPLACE_SCHEMA_OPTIONAL).bucket("id", 16).withSpecId(1).build();
+
+  protected static final SortOrder REPLACE_WRITE_ORDER_OPTIONAL =
+      SortOrder.builderFor(REPLACE_SCHEMA_OPTIONAL)
+          .asc(Expressions.bucket("id", 16))
+          .asc("id")
+          .build();
+
+  @TempDir private File tempFolder;
+  private BigQueryMetastoreCatalog catalog;
+  private BigQueryMetastoreClient fakeBigQueryMetastoreClient;
+  private String warehouseLocation;
+
+  @BeforeEach
+  public void before() throws Exception {
+    catalog = initCatalog(CATALOG_ID, ImmutableMap.of());
+  }
+
+  @AfterEach
+  public void after() throws Exception {
+    // Drop all tables in all datasets
+    List<Namespace> namespaces = catalog().listNamespaces(Namespace.empty());
+    for (Namespace namespace : namespaces) {
+      List<TableIdentifier> tables = catalog().listTables(namespace);
+      for (TableIdentifier table : tables) {
+        try {
+          catalog().dropTable(table, true); // drop with purge
+        } catch (NoSuchTableException e) {
+          // Table might be dropped by a previous iteration, ignore
+        }
+      }
+
+      // Drop all datasets (except any initial ones)
+      try {
+        catalog().dropNamespace(namespace);
+      } catch (NoSuchNamespaceException e) {
+        // Namespace might be dropped by a previous iteration, ignore
+      }
+    }
+  }
+
+  @Override
+  protected boolean requiresNamespaceCreate() {
+    return true;
+  }
+
+  @Override
+  protected boolean supportsNamesWithSlashes() {
+    return false;
+  }
+
+  @Override
+  protected boolean supportsNamesWithDot() {
+    return false;
+  }
+
+  @Override
+  public BigQueryMetastoreCatalog catalog() {
+    return catalog;
+  }
+
+  @Override
+  protected BigQueryMetastoreCatalog initCatalog(
+      String catalogName, Map<String, String> additionalProperties) {
+
+    warehouseLocation = tempFolder.toPath().resolve("hive-warehouse").toString();
+
+    Map<String, String> properties =
+        Map.of(
+            ICEBERG_CATALOG_TYPE,
+            ICEBERG_CATALOG_TYPE_BIGQUERY,
+            PROJECT_ID,
+            GCP_PROJECT,
+            CatalogProperties.WAREHOUSE_LOCATION,
+            warehouseLocation,
+            TESTING_ENABLED,
+            "true",
+            CatalogProperties.TABLE_DEFAULT_PREFIX + "default-key1",
+            "catalog-default-key1",
+            CatalogProperties.TABLE_DEFAULT_PREFIX + "default-key2",
+            "catalog-default-key2",
+            CatalogProperties.TABLE_DEFAULT_PREFIX + "override-key3",
+            "catalog-default-key3",
+            CatalogProperties.TABLE_OVERRIDE_PREFIX + "override-key3",
+            "catalog-override-key3",
+            CatalogProperties.TABLE_OVERRIDE_PREFIX + "override-key4",
+            "catalog-override-key4");
+
+    BigQueryMetastoreCatalog tmpCatalog =
+        (BigQueryMetastoreCatalog)
+            CatalogUtil.buildIcebergCatalog(
+                catalogName,
+                ImmutableMap.<String, String>builder()
+                    .putAll(properties)
+                    .putAll(additionalProperties)
+                    .build(),
+                null);
+
+    return tmpCatalog;
+  }
+
+  @Test
+  public void testRenameTable() {
+    if (requiresNamespaceCreate()) {
+      catalog.createNamespace(NS);
+    }
+
+    assertThat(catalog.tableExists(TABLE))
+        .as("Source table should not exist before create")
+        .isFalse();
+
+    catalog.buildTable(TABLE, SCHEMA).create();
+    assertThat(catalog.tableExists(TABLE)).as("Table should exist after create").isTrue();
+
+    assertThat(catalog.tableExists(RENAMED_TABLE))
+        .as("Destination table should not exist before rename")
+        .isFalse();
+
+    assertThatThrownBy(() -> catalog.renameTable(TABLE, RENAMED_TABLE))
+        .isInstanceOf(UnsupportedOperationException.class)
+        .hasMessageContaining("Table rename operation is unsupported");
+
+    catalog.dropTable(TABLE);
+  }
+
+  @Disabled(
+      "BigQuery does not allow adding a REQUIRED column to an existing table schema. "
+          + "When this operation is attempted, an exception gets thrown.")
+  @Test
+  public void testOverrideTablePropertiesReplaceTransaction() {}
+
+  @Test
+  public void testConcurrentReplaceTransactionSchema() {

Review Comment:
   TBH I don't really understand why all of these custom tests are needed and also why certain tests are Disabled as they all pass. I was able to trim down the test class to
   ```
   public class TestBigQueryCatalog extends CatalogTests<BigQueryMetastoreCatalog> {
   
     @TempDir private File tempFolder;
     private BigQueryMetastoreCatalog catalog;
   
     @BeforeEach
     public void before() {
       catalog = initCatalog("catalog-name", ImmutableMap.of());
     }
   
     @AfterEach
     public void after() throws Exception {
       // Drop all tables in all datasets
       List<Namespace> namespaces = catalog().listNamespaces(Namespace.empty());
       for (Namespace namespace : namespaces) {
         List<TableIdentifier> tables = catalog().listTables(namespace);
         for (TableIdentifier table : tables) {
           try {
             catalog().dropTable(table, true); // drop with purge
           } catch (NoSuchTableException e) {
             // Table might be dropped by a previous iteration, ignore
           }
         }
   
         // Drop all datasets (except any initial ones)
         try {
           catalog().dropNamespace(namespace);
         } catch (NoSuchNamespaceException e) {
           // Namespace might be dropped by a previous iteration, ignore
         }
       }
     }
   
     @Override
     protected boolean requiresNamespaceCreate() {
       return true;
     }
   
     @Override
     protected boolean supportsNamesWithSlashes() {
       return false;
     }
   
     @Override
     protected boolean supportsNamesWithDot() {
       return false;
     }
   
     @Override
     public BigQueryMetastoreCatalog catalog() {
       return catalog;
     }
   
     @Override
     protected BigQueryMetastoreCatalog initCatalog(
         String catalogName, Map<String, String> additionalProperties) {
   
       String warehouseLocation = tempFolder.toPath().resolve("hive-warehouse").toString();
   
       Map<String, String> properties =
           Map.of(
               ICEBERG_CATALOG_TYPE,
               ICEBERG_CATALOG_TYPE_BIGQUERY,
               BigQueryMetastoreCatalog.PROJECT_ID,
               "projectId",
               CatalogProperties.WAREHOUSE_LOCATION,
               warehouseLocation,
               TESTING_ENABLED,
               "true",
               CatalogProperties.TABLE_DEFAULT_PREFIX + "default-key1",
               "catalog-default-key1",
               CatalogProperties.TABLE_DEFAULT_PREFIX + "default-key2",
               "catalog-default-key2",
               CatalogProperties.TABLE_DEFAULT_PREFIX + "override-key3",
               "catalog-default-key3",
               CatalogProperties.TABLE_OVERRIDE_PREFIX + "override-key3",
               "catalog-override-key3",
               CatalogProperties.TABLE_OVERRIDE_PREFIX + "override-key4",
               "catalog-override-key4");
   
       BigQueryMetastoreCatalog tmpCatalog =
           (BigQueryMetastoreCatalog)
               CatalogUtil.buildIcebergCatalog(
                   catalogName,
                   ImmutableMap.<String, String>builder()
                       .putAll(properties)
                       .putAll(additionalProperties)
                       .build(),
                   null);
   
       return tmpCatalog;
     }
   
     @Test
     public void testRenameTable() {
       if (requiresNamespaceCreate()) {
         catalog.createNamespace(NS);
       }
   
       assertThat(catalog.tableExists(TABLE))
           .as("Source table should not exist before create")
           .isFalse();
   
       catalog.buildTable(TABLE, SCHEMA).create();
       assertThat(catalog.tableExists(TABLE)).as("Table should exist after create").isTrue();
   
       assertThat(catalog.tableExists(RENAMED_TABLE))
           .as("Destination table should not exist before rename")
           .isFalse();
   
       assertThatThrownBy(() -> catalog.renameTable(TABLE, RENAMED_TABLE))
           .isInstanceOf(UnsupportedOperationException.class)
           .hasMessageContaining("Table rename operation is unsupported");
   
       catalog.dropTable(TABLE);
     }
   }
   ```



-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org

For queries about this service, please contact Infrastructure at:
users@infra.apache.org


---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For additional commands, e-mail: issues-help@iceberg.apache.org


Re: [PR] Catalog: Add BigQuery Metastore Catalog Support [iceberg]

Posted by "nastra (via GitHub)" <gi...@apache.org>.
nastra commented on code in PR #12808:
URL: https://github.com/apache/iceberg/pull/12808#discussion_r2054344779


##########
bigquery/src/main/java/org/apache/iceberg/gcp/bigquery/BigQueryMetastoreCatalog.java:
##########
@@ -0,0 +1,402 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.iceberg.gcp.bigquery;
+
+import com.google.api.services.bigquery.model.Dataset;
+import com.google.api.services.bigquery.model.DatasetList.Datasets;
+import com.google.api.services.bigquery.model.DatasetReference;
+import com.google.api.services.bigquery.model.ExternalCatalogDatasetOptions;
+import com.google.api.services.bigquery.model.TableReference;
+import com.google.cloud.ServiceOptions;
+import com.google.cloud.bigquery.BigQueryOptions;
+import java.io.IOException;
+import java.security.GeneralSecurityException;
+import java.util.List;
+import java.util.Locale;
+import java.util.Map;
+import java.util.Objects;
+import java.util.Set;
+import org.apache.hadoop.conf.Configurable;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.iceberg.BaseMetastoreCatalog;
+import org.apache.iceberg.CatalogProperties;
+import org.apache.iceberg.CatalogUtil;
+import org.apache.iceberg.TableMetadata;
+import org.apache.iceberg.TableOperations;
+import org.apache.iceberg.catalog.Namespace;
+import org.apache.iceberg.catalog.SupportsNamespaces;
+import org.apache.iceberg.catalog.TableIdentifier;
+import org.apache.iceberg.exceptions.AlreadyExistsException;
+import org.apache.iceberg.exceptions.NoSuchNamespaceException;
+import org.apache.iceberg.exceptions.NoSuchTableException;
+import org.apache.iceberg.exceptions.ServiceFailureException;
+import org.apache.iceberg.exceptions.ValidationException;
+import org.apache.iceberg.io.FileIO;
+import org.apache.iceberg.relocated.com.google.common.annotations.VisibleForTesting;
+import org.apache.iceberg.relocated.com.google.common.base.Preconditions;
+import org.apache.iceberg.relocated.com.google.common.base.Strings;
+import org.apache.iceberg.relocated.com.google.common.collect.ImmutableList;
+import org.apache.iceberg.relocated.com.google.common.collect.ImmutableMap;
+import org.apache.iceberg.relocated.com.google.common.collect.Maps;
+import org.apache.iceberg.util.LocationUtil;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/** Iceberg Bigquery Metastore Catalog implementation. */
+public final class BigQueryMetastoreCatalog extends BaseMetastoreCatalog
+    implements SupportsNamespaces, Configurable {
+
+  // User provided properties.
+  public static final String PROPERTIES_KEY_GCP_PROJECT = "gcp-project";

Review Comment:
   let's omit the `PROPERTIES_KEY` prefix as I don't think that helps in readability. Also the project doesn't use something like this in any other place around the codebase



-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org

For queries about this service, please contact Infrastructure at:
users@infra.apache.org


---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For additional commands, e-mail: issues-help@iceberg.apache.org


Re: [PR] Catalog: Add BigQuery Metastore Catalog Support [iceberg]

Posted by "nastra (via GitHub)" <gi...@apache.org>.
nastra commented on code in PR #12808:
URL: https://github.com/apache/iceberg/pull/12808#discussion_r2054346375


##########
gcp/src/main/java/org/apache/iceberg/gcp/GCPProperties.java:
##########
@@ -28,6 +28,9 @@
 import org.apache.iceberg.util.PropertyUtil;
 
 public class GCPProperties implements Serializable {
+  /** The GCP project ID. Required. */
+  public static final String PROJECT_ID = "gcp-project";

Review Comment:
   why is this and the gcp-location needed in this class for BigQuery? I think these changes here should be reverted as this class holds stuff for the FileIO implementation and not for a particular catalog impl



-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org

For queries about this service, please contact Infrastructure at:
users@infra.apache.org


---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For additional commands, e-mail: issues-help@iceberg.apache.org


Re: [PR] Catalog: Add BigQuery Metastore Catalog Support [iceberg]

Posted by "nastra (via GitHub)" <gi...@apache.org>.
nastra commented on code in PR #12808:
URL: https://github.com/apache/iceberg/pull/12808#discussion_r2055633040


##########
bigquery/src/main/java/org/apache/iceberg/gcp/bigquery/BigQueryMetastoreCatalog.java:
##########
@@ -0,0 +1,400 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.iceberg.gcp.bigquery;
+
+import com.google.api.services.bigquery.model.Dataset;
+import com.google.api.services.bigquery.model.DatasetList.Datasets;
+import com.google.api.services.bigquery.model.DatasetReference;
+import com.google.api.services.bigquery.model.ExternalCatalogDatasetOptions;
+import com.google.api.services.bigquery.model.TableReference;
+import com.google.cloud.ServiceOptions;
+import com.google.cloud.bigquery.BigQueryOptions;
+import java.io.IOException;
+import java.security.GeneralSecurityException;
+import java.util.List;
+import java.util.Locale;
+import java.util.Map;
+import java.util.Objects;
+import java.util.Set;
+import org.apache.hadoop.conf.Configurable;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.iceberg.BaseMetastoreCatalog;
+import org.apache.iceberg.CatalogProperties;
+import org.apache.iceberg.CatalogUtil;
+import org.apache.iceberg.TableMetadata;
+import org.apache.iceberg.TableOperations;
+import org.apache.iceberg.catalog.Namespace;
+import org.apache.iceberg.catalog.SupportsNamespaces;
+import org.apache.iceberg.catalog.TableIdentifier;
+import org.apache.iceberg.exceptions.AlreadyExistsException;
+import org.apache.iceberg.exceptions.NoSuchNamespaceException;
+import org.apache.iceberg.exceptions.NoSuchTableException;
+import org.apache.iceberg.exceptions.ServiceFailureException;
+import org.apache.iceberg.exceptions.ValidationException;
+import org.apache.iceberg.io.FileIO;
+import org.apache.iceberg.relocated.com.google.common.annotations.VisibleForTesting;
+import org.apache.iceberg.relocated.com.google.common.base.Preconditions;
+import org.apache.iceberg.relocated.com.google.common.base.Strings;
+import org.apache.iceberg.relocated.com.google.common.collect.ImmutableList;
+import org.apache.iceberg.relocated.com.google.common.collect.ImmutableMap;
+import org.apache.iceberg.relocated.com.google.common.collect.Maps;
+import org.apache.iceberg.util.LocationUtil;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/** Iceberg Bigquery Metastore Catalog implementation. */
+public final class BigQueryMetastoreCatalog extends BaseMetastoreCatalog
+    implements SupportsNamespaces, Configurable {
+
+  // User provided properties.
+  public static final String PROJECT_ID = "gcp-project";
+  public static final String GCP_LOCATION = "gcp-location";
+  public static final String FILTER_UNSUPPORTED_TABLES = "filter-unsupported-tables";
+  public static final String TESTING_ENABLED = "testing-enabled";
+
+  public static final String HIVE_METASTORE_WAREHOUSE_DIR = "hive.metastore.warehouse.dir";
+
+  private static final Logger LOG = LoggerFactory.getLogger(BigQueryMetastoreCatalog.class);
+
+  private static final String DEFAULT_GCP_LOCATION = "us";
+
+  private String catalogName;
+  private Map<String, String> catalogProperties;
+  private FileIO fileIO;
+  private Configuration conf;
+  private String projectId;
+  private String location;
+  private BigQueryMetastoreClient client;
+  private boolean filterUnsupportedTables;
+
+  // Must have a no-arg constructor to be dynamically loaded
+  // initialize(String name, Map<String, String> properties) will be called to complete
+  // initialization
+  public BigQueryMetastoreCatalog() {}
+
+  @Override
+  public void initialize(String inputName, Map<String, String> properties) {
+    Preconditions.checkArgument(
+        properties.containsKey(PROJECT_ID), "GCP project must be specified");
+
+    projectId = properties.get(PROJECT_ID);
+    location = properties.getOrDefault(GCP_LOCATION, DEFAULT_GCP_LOCATION);
+    boolean testingEnabled =
+        Boolean.parseBoolean(properties.getOrDefault(TESTING_ENABLED, "false"));
+
+    BigQueryOptions options =
+        BigQueryOptions.newBuilder()
+            .setProjectId(projectId)
+            .setLocation(location)
+            .setRetrySettings(ServiceOptions.getDefaultRetrySettings())
+            .build();
+
+    try {
+      if (testingEnabled) {
+        client = new FakeBigQueryMetastoreClient(options);
+
+      } else {
+        client = new BigQueryMetastoreClientImpl(options);
+      }
+    } catch (IOException e) {
+      throw new ServiceFailureException(e, "Creating BigQuery client failed");
+    } catch (GeneralSecurityException e) {
+      throw new ValidationException(e, "Creating BigQuery client failed due to a security issue");
+    }
+    initialize(inputName, properties, projectId, location, client);
+  }
+
+  @VisibleForTesting
+  void initialize(
+      String inputName,
+      Map<String, String> properties,
+      String initialProjectId,
+      String initialLocation,
+      BigQueryMetastoreClient bigQueryMetaStoreClient) {
+    Preconditions.checkArgument(bigQueryMetaStoreClient != null, "BigQuery client can not be null");
+    this.catalogName = inputName;
+    this.catalogProperties = ImmutableMap.copyOf(properties);
+    this.projectId = initialProjectId;
+    this.location = initialLocation;
+    this.client = bigQueryMetaStoreClient;
+
+    if (this.conf == null) {
+      LOG.warn("No configuration was set, using the default environment Configuration");
+      this.conf = new Configuration();
+    }
+
+    LOG.info("Using BigQuery Metastore Iceberg Catalog: {}", inputName);
+
+    if (properties.containsKey(CatalogProperties.WAREHOUSE_LOCATION)) {
+      // Iceberg always removes trailing slash to avoid paths like "<folder>//data" in file systems
+      // like s3.
+      this.conf.set(
+          HIVE_METASTORE_WAREHOUSE_DIR,
+          LocationUtil.stripTrailingSlash(properties.get(CatalogProperties.WAREHOUSE_LOCATION)));
+    }
+
+    String fileIoImpl =
+        properties.getOrDefault(
+            CatalogProperties.FILE_IO_IMPL, "org.apache.iceberg.hadoop.HadoopFileIO");
+    this.fileIO = CatalogUtil.loadFileIO(fileIoImpl, properties, conf);
+
+    this.filterUnsupportedTables =
+        Boolean.parseBoolean(properties.getOrDefault(FILTER_UNSUPPORTED_TABLES, "false"));
+  }
+
+  @Override
+  protected TableOperations newTableOps(TableIdentifier identifier) {
+    return new BigQueryTableOperations(
+        client,
+        fileIO,
+        projectId,
+        // Sometimes extensions have the namespace contain the table name too, so we are forced to
+        // allow invalid namespace and just take the first part here like other catalog
+        // implementations do.
+        identifier.namespace().level(0),
+        identifier.name(),
+        conf);
+  }
+
+  @Override
+  protected String defaultWarehouseLocation(TableIdentifier identifier) {
+    String locationUri = null;
+    DatasetReference datasetReference = toDatasetReference(identifier.namespace());
+    Dataset dataset = client.getDataset(datasetReference);
+    if (dataset != null && dataset.getExternalCatalogDatasetOptions() != null) {
+      locationUri = dataset.getExternalCatalogDatasetOptions().getDefaultStorageLocationUri();
+    }
+    return String.format(
+        "%s/%s",
+        Strings.isNullOrEmpty(locationUri)
+            ? datasetReference.getDatasetId()
+            : LocationUtil.stripTrailingSlash(locationUri),
+        identifier.name());
+  }
+
+  @Override
+  public List<TableIdentifier> listTables(Namespace namespace) {
+    validateNamespace(namespace);
+
+    return client.listTables(toDatasetReference(namespace), filterUnsupportedTables).stream()
+        .map(
+            table -> TableIdentifier.of(namespace.level(0), table.getTableReference().getTableId()))
+        .collect(ImmutableList.toImmutableList());
+  }
+
+  @Override
+  public boolean dropTable(TableIdentifier identifier, boolean purge) {
+    try {
+      TableOperations ops = newTableOps(identifier);
+      TableMetadata lastMetadata = ops.current();
+
+      client.deleteTable(toTableReference(identifier));
+
+      if (purge && lastMetadata != null) {
+        CatalogUtil.dropTableData(ops.io(), lastMetadata);
+      }
+    } catch (NoSuchTableException e) { // Not catching a NoSuchIcebergTableException on purpose
+      return false; // The documentation says just return false in this case
+    }
+    return true;
+  }
+
+  @Override
+  public void renameTable(TableIdentifier from, TableIdentifier to) {
+    validateNamespace(from.namespace());
+    validateNamespace(to.namespace());
+
+    try {
+      client.getDataset(toDatasetReference(to.namespace()));
+      client.getDataset(toDatasetReference(from.namespace()));
+    } catch (NoSuchNamespaceException e) {
+      throw e;
+    }
+
+    TableReference fromTableRef = toTableReference(from);
+    TableReference toTableRef = toTableReference(to);
+
+    // Check if the source table exists
+    try {
+      client.getTable(fromTableRef);
+    } catch (NoSuchTableException e) {
+      throw new NoSuchTableException("Table does not exist: %s", from.name());
+    }
+
+    // Check if the destination table already exists
+    try {
+      client.getTable(toTableRef);
+      // If getTable succeeds, the destination table exists
+      throw new AlreadyExistsException("Table already exists: %s", to.name());
+    } catch (NoSuchTableException e) {
+      // Destination table does not exist, proceed with throwing exception
+    }
+
+    if (!from.namespace().equals(to.namespace())) {
+      throw new ValidationException("New table name must be in the same namespace");
+    }
+
+    // TODO(b/354981675): Enable once supported by the API.
+    throw new UnsupportedOperationException("Table rename operation is unsupported.");
+  }
+
+  @Override
+  public void createNamespace(Namespace namespace, Map<String, String> metadata) {
+    Dataset builder = new Dataset();
+    DatasetReference datasetReference = toDatasetReference(namespace);
+    builder.setLocation(this.location);
+    builder.setDatasetReference(datasetReference);
+    builder.setExternalCatalogDatasetOptions(
+        BigQueryMetastoreUtils.createExternalCatalogDatasetOptions(
+            datasetReference.getDatasetId(), metadata));
+
+    client.createDataset(builder);
+  }
+
+  /**
+   * Since this catalog only supports one-level namespaces, it always returns an empty list unless
+   * passed an empty namespace to list all namespaces within the catalog.
+   */
+  @Override
+  public List<Namespace> listNamespaces(Namespace namespace) {
+    if (namespace.levels().length != 0) {
+      // BQMS does not support namespaces under database or tables, returns empty.
+      // It is called when dropping a namespace to make sure it's empty (listTables is called as
+      // well), returns empty to unblock deletion.
+      return ImmutableList.of();
+    }
+    return client.listDatasets(projectId).stream()
+        .map(BigQueryMetastoreCatalog::getNamespace)
+        .collect(ImmutableList.toImmutableList());
+  }
+
+  @Override
+  public boolean dropNamespace(Namespace namespace) {
+    try {
+      client.deleteDataset(toDatasetReference(namespace));
+      // We don't delete the data folder for safety, which aligns with Hive Metastore's default
+      // behavior.
+      // We can support database or catalog level config controlling file deletion in the future.
+      return true;
+    } catch (NoSuchNamespaceException e) {
+      return false;
+    }
+  }
+
+  @Override
+  public boolean setProperties(Namespace namespace, Map<String, String> properties) {
+    Dataset dataset = client.getDataset(toDatasetReference(namespace));
+
+    ExternalCatalogDatasetOptions existingOptions = dataset.getExternalCatalogDatasetOptions();
+    Map<String, String> existingParameters =
+        existingOptions != null ? existingOptions.getParameters() : null;
+
+    Map<String, String> newParameters = Maps.newHashMap();
+    if (existingParameters != null) {
+      newParameters.putAll(existingParameters);
+    }
+    newParameters.putAll(properties);
+
+    if (Objects.equals(existingParameters, newParameters)) {
+      // No change in parameters detected
+      return false;
+    }
+
+    client.setDatasetParameters(toDatasetReference(namespace), properties);
+    return true;
+  }
+
+  @Override
+  public boolean removeProperties(Namespace namespace, Set<String> properties) {
+    client.removeDatasetParameters(toDatasetReference(namespace), properties);
+    return true;
+  }
+
+  @SuppressWarnings("FormatStringAnnotation")
+  @Override
+  public Map<String, String> loadNamespaceMetadata(Namespace namespace) {
+    try {
+      return getMetadata(client.getDataset(toDatasetReference(namespace)));
+    } catch (IllegalArgumentException e) {

Review Comment:
   it seems a bit weird to handle IAE like this and then throw a NSNE



-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org

For queries about this service, please contact Infrastructure at:
users@infra.apache.org


---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For additional commands, e-mail: issues-help@iceberg.apache.org


Re: [PR] Catalog: Add BigQuery Metastore Catalog Support [iceberg]

Posted by "nastra (via GitHub)" <gi...@apache.org>.
nastra commented on code in PR #12808:
URL: https://github.com/apache/iceberg/pull/12808#discussion_r2066021568


##########
bigquery/src/main/java/org/apache/iceberg/gcp/bigquery/BigQueryTableOperations.java:
##########
@@ -0,0 +1,297 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.iceberg.gcp.bigquery;
+
+import com.google.api.services.bigquery.model.ExternalCatalogTableOptions;
+import com.google.api.services.bigquery.model.Table;
+import com.google.api.services.bigquery.model.TableReference;
+import java.util.Locale;
+import java.util.Map;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hive.common.StatsSetupConst;
+import org.apache.iceberg.BaseMetastoreOperations;
+import org.apache.iceberg.BaseMetastoreTableOperations;
+import org.apache.iceberg.SnapshotSummary;
+import org.apache.iceberg.TableMetadata;
+import org.apache.iceberg.TableProperties;
+import org.apache.iceberg.exceptions.AlreadyExistsException;
+import org.apache.iceberg.exceptions.CommitFailedException;
+import org.apache.iceberg.exceptions.CommitStateUnknownException;
+import org.apache.iceberg.exceptions.NoSuchTableException;
+import org.apache.iceberg.exceptions.ValidationException;
+import org.apache.iceberg.hadoop.ConfigProperties;
+import org.apache.iceberg.io.FileIO;
+import org.apache.iceberg.relocated.com.google.common.collect.Maps;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/** Handles BigQuery metastore table operations. */
+public final class BigQueryTableOperations extends BaseMetastoreTableOperations {
+
+  private static final Logger LOG = LoggerFactory.getLogger(BigQueryTableOperations.class);
+
+  private static final String TABLE_PROPERTIES_BQ_CONNECTION = "bq_connection";
+
+  private final BigQueryMetastoreClient client;
+  private final FileIO fileIO;
+  private final TableReference tableReference;
+  private final Configuration conf;
+
+  BigQueryTableOperations(
+      BigQueryMetastoreClient client,
+      FileIO fileIO,
+      TableReference tableReference,
+      Configuration conf) {
+    this.client = client;
+    this.fileIO = fileIO;
+    this.tableReference = tableReference;
+    this.conf = conf;
+  }
+
+  // The doRefresh method should provide implementation on how to get the metadata location.
+  @Override
+  public void doRefresh() {
+    // Must default to null.
+    String metadataLocation = null;
+    try {
+      metadataLocation =
+          getMetadataLocationOrThrow(
+              client.load(this.tableReference).getExternalCatalogTableOptions());
+    } catch (NoSuchTableException e) {
+      if (currentMetadataLocation() != null) {
+        // Re-throws the exception because the table must exist in this case.
+        throw e;
+      }
+    }
+    refreshFromMetadataLocation(metadataLocation);
+  }
+
+  // The doCommit method should provide implementation on how to update with metadata location
+  // atomically
+  @Override
+  public void doCommit(TableMetadata base, TableMetadata metadata) {
+    String newMetadataLocation =
+        base == null && metadata.metadataFileLocation() != null
+            ? metadata.metadataFileLocation()
+            : writeNewMetadata(metadata, currentVersion() + 1);
+    BaseMetastoreOperations.CommitStatus commitStatus =
+        BaseMetastoreOperations.CommitStatus.FAILURE;
+    try {
+      if (base == null) {
+        createTable(newMetadataLocation, metadata);
+      } else {
+        updateTable(base.metadataFileLocation(), newMetadataLocation, metadata);
+      }
+      commitStatus = BaseMetastoreOperations.CommitStatus.SUCCESS;
+    } catch (CommitFailedException | CommitStateUnknownException e) {
+      throw e;
+    } catch (Throwable e) {
+      LOG.error("Exception thrown on commit: ", e);
+      if (e instanceof AlreadyExistsException) {
+        throw e;
+      }
+      commitStatus =
+          BaseMetastoreOperations.CommitStatus.valueOf(
+              checkCommitStatus(newMetadataLocation, metadata).name());
+      if (commitStatus == BaseMetastoreOperations.CommitStatus.FAILURE) {
+        throw new CommitFailedException(e, "Failed to commit");
+      }
+      if (commitStatus == BaseMetastoreOperations.CommitStatus.UNKNOWN) {
+        throw new CommitStateUnknownException(e);
+      }
+    } finally {
+      try {
+        if (commitStatus == BaseMetastoreOperations.CommitStatus.FAILURE) {
+          LOG.warn("Failed to commit updates to table {}", tableName());
+          io().deleteFile(newMetadataLocation);
+        }
+      } catch (RuntimeException e) {
+        LOG.error(
+            "Failed to cleanup metadata file at {} for table {}",
+            newMetadataLocation,
+            tableName(),
+            e);
+      }
+    }
+  }
+
+  @Override
+  public String tableName() {
+    return String.format("%s.%s", tableReference.getDatasetId(), tableReference.getTableId());
+  }
+
+  @Override
+  public FileIO io() {
+    return fileIO;
+  }
+
+  private void createTable(String newMetadataLocation, TableMetadata metadata) {
+    LOG.debug("Creating a new Iceberg table: {}", tableName());
+    Table tableBuilder = makeNewTable(metadata, newMetadataLocation);
+    tableBuilder.setTableReference(this.tableReference);
+    addConnectionIfProvided(tableBuilder, metadata.properties());
+
+    client.create(tableBuilder);
+  }
+
+  private void addConnectionIfProvided(Table tableBuilder, Map<String, String> metadataProperties) {
+    if (metadataProperties.containsKey(TABLE_PROPERTIES_BQ_CONNECTION)) {
+      tableBuilder
+          .getExternalCatalogTableOptions()
+          .setConnectionId(metadataProperties.get(TABLE_PROPERTIES_BQ_CONNECTION));
+    }
+  }
+
+  /** Update table properties with concurrent update detection using etag. */
+  private void updateTable(
+      String oldMetadataLocation, String newMetadataLocation, TableMetadata metadata) {
+    Table table = client.load(this.tableReference);
+    if (table.getEtag().isEmpty()) {
+      throw new ValidationException(
+          "Etag of legacy table %s is empty, manually update the table via the BigQuery API or"
+              + " recreate and retry",
+          tableName());
+    }
+    ExternalCatalogTableOptions options = table.getExternalCatalogTableOptions();
+    addConnectionIfProvided(table, metadata.properties());
+
+    // If `metadataLocationFromMetastore` is different from metadata location of base, it means
+    // someone has updated metadata location in metastore, which is a conflict update.
+    String metadataLocationFromMetastore =
+        options.getParameters().getOrDefault(METADATA_LOCATION_PROP, "");
+    if (!metadataLocationFromMetastore.isEmpty()
+        && !metadataLocationFromMetastore.equals(oldMetadataLocation)) {
+      throw new CommitFailedException(
+          "Cannot commit base metadata location '%s' is not same as the current table metadata location '%s' for"
+              + " %s.%s",
+          oldMetadataLocation,
+          metadataLocationFromMetastore,
+          tableReference.getDatasetId(),
+          tableReference.getTableId());
+    }
+
+    options.setParameters(buildTableParameters(newMetadataLocation, metadata));
+    try {
+      client.update(tableReference, table);
+    } catch (ValidationException e) {
+      if (e.getMessage().toLowerCase(Locale.ENGLISH).contains("etag mismatch")) {
+        throw new CommitFailedException(
+            "Updating table failed due to conflict updates (etag mismatch). Retry the update");
+      }
+
+      throw e;
+    }
+  }
+
+  // To make the table queryable from Hive, the user would likely be setting the HIVE_ENGINE_ENABLED
+  // parameter.
+  //
+  // TODO: We need to make a decision on how to make the table queryable from Hive.
+  // (could be a server side change or a client side change - that's TBD).
+  private Table makeNewTable(TableMetadata metadata, String metadataFileLocation) {
+    boolean hiveEngineEnabled = hiveEngineEnabled(metadata);
+    return new Table()
+        .setExternalCatalogTableOptions(
+            BigQueryMetastoreUtils.createExternalCatalogTableOptions(
+                metadata.location(),
+                buildTableParameters(metadataFileLocation, metadata),
+                hiveEngineEnabled));
+  }
+
+  // Follow Iceberg's HiveTableOperations to populate more table parameters for HMS compatibility.
+  private Map<String, String> buildTableParameters(
+      String metadataFileLocation, TableMetadata metadata) {
+    Map<String, String> parameters = Maps.newHashMap(metadata.properties());
+    if (metadata.uuid() != null) {
+      parameters.put(TableProperties.UUID, metadata.uuid());
+    }
+    if (currentMetadataLocation() != null && !currentMetadataLocation().isEmpty()) {
+      parameters.put(PREVIOUS_METADATA_LOCATION_PROP, currentMetadataLocation());
+    }
+    parameters.put(METADATA_LOCATION_PROP, metadataFileLocation);
+    parameters.put(TABLE_TYPE_PROP, ICEBERG_TABLE_TYPE_VALUE);
+    // Follow HMS to use the EXTERNAL type.
+    parameters.put("EXTERNAL", "TRUE");
+
+    // Hive style basic statistics.
+    updateParametersWithSnapshotMetadata(metadata, parameters);
+    // More Iceberg metadata can be exposed, e.g., statistic, schema, partition spec, as HMS do. But
+    // we should be careful that these metadata could be huge and make the metadata API response
+    // less readable (e.g., list tables). Users can always inspect these metadata in Spark, so they
+    // are not set for now.
+    return parameters;
+  }
+
+  /** Adds Hive-style basic statistics from snapshot metadata if it exists. */
+  private static void updateParametersWithSnapshotMetadata(
+      TableMetadata metadata, Map<String, String> parameters) {
+    if (metadata.currentSnapshot() == null) {
+      return;
+    }
+
+    Map<String, String> summary = metadata.currentSnapshot().summary();
+    if (summary.get(SnapshotSummary.TOTAL_DATA_FILES_PROP) != null) {
+      parameters.put(StatsSetupConst.NUM_FILES, summary.get(SnapshotSummary.TOTAL_DATA_FILES_PROP));
+    }
+
+    if (summary.get(SnapshotSummary.TOTAL_RECORDS_PROP) != null) {
+      parameters.put(StatsSetupConst.ROW_COUNT, summary.get(SnapshotSummary.TOTAL_RECORDS_PROP));
+    }
+
+    if (summary.get(SnapshotSummary.TOTAL_FILE_SIZE_PROP) != null) {
+      parameters.put(StatsSetupConst.TOTAL_SIZE, summary.get(SnapshotSummary.TOTAL_FILE_SIZE_PROP));
+    }
+  }
+
+  private String getMetadataLocationOrThrow(ExternalCatalogTableOptions tableOptions) {

Review Comment:
   ```suggestion
     private String loadMetadataLocationOrThrow(ExternalCatalogTableOptions tableOptions) {
   ```



-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org

For queries about this service, please contact Infrastructure at:
users@infra.apache.org


---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For additional commands, e-mail: issues-help@iceberg.apache.org


Re: [PR] Catalog: Add BigQuery Metastore Catalog Support [iceberg]

Posted by "nastra (via GitHub)" <gi...@apache.org>.
nastra commented on code in PR #12808:
URL: https://github.com/apache/iceberg/pull/12808#discussion_r2066086062


##########
bigquery/src/test/java/org/apache/iceberg/gcp/bigquery/BigQueryTableOperationsTest.java:
##########
@@ -0,0 +1,274 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.iceberg.gcp.bigquery;
+
+import static org.apache.iceberg.gcp.bigquery.BigQueryMetastoreCatalog.PROJECT_ID;
+import static org.apache.iceberg.types.Types.NestedField.required;
+import static org.assertj.core.api.Assertions.assertThat;
+import static org.assertj.core.api.Assertions.assertThatThrownBy;
+import static org.mockito.Mockito.any;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.reset;
+import static org.mockito.Mockito.times;
+import static org.mockito.Mockito.verify;
+import static org.mockito.Mockito.when;
+
+import com.google.api.services.bigquery.model.Dataset;
+import com.google.api.services.bigquery.model.DatasetReference;
+import com.google.api.services.bigquery.model.ExternalCatalogDatasetOptions;
+import com.google.api.services.bigquery.model.ExternalCatalogTableOptions;
+import com.google.api.services.bigquery.model.StorageDescriptor;
+import com.google.api.services.bigquery.model.Table;
+import com.google.api.services.bigquery.model.TableReference;
+import java.io.File;
+import java.io.IOException;
+import java.util.Collections;
+import java.util.Optional;
+import org.apache.commons.io.FileUtils;
+import org.apache.commons.io.filefilter.TrueFileFilter;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.iceberg.CatalogProperties;
+import org.apache.iceberg.PartitionSpec;
+import org.apache.iceberg.Schema;
+import org.apache.iceberg.catalog.TableIdentifier;
+import org.apache.iceberg.exceptions.CommitFailedException;
+import org.apache.iceberg.exceptions.NoSuchTableException;
+import org.apache.iceberg.exceptions.ValidationException;
+import org.apache.iceberg.relocated.com.google.common.collect.ImmutableMap;
+import org.apache.iceberg.types.Types;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.io.TempDir;
+import org.mockito.ArgumentCaptor;
+
+public class BigQueryTableOperationsTest {
+
+  @TempDir private File tempFolder;
+  public static final String METADATA_LOCATION_PROP = "metadata_location";
+  private static final String GCP_PROJECT = "my-project";
+  private static final String GCP_REGION = "us";
+  private static final String DATASET_ID = "db";
+  private static final String TABLE_ID = "tbl";
+  private static final TableIdentifier SPARK_TABLE_ID = TableIdentifier.of(DATASET_ID, TABLE_ID);

Review Comment:
   I would just call this `IDENTIFIER` or `IDENT` since this isn't really related to Spark. Also the `DATASET_ID` and `TABLE_ID` should rather use Iceberg semantics -> `NS` / `TABLE`



-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org

For queries about this service, please contact Infrastructure at:
users@infra.apache.org


---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For additional commands, e-mail: issues-help@iceberg.apache.org


Re: [PR] Catalog: Add BigQuery Metastore Catalog Support [iceberg]

Posted by "danielcweeks (via GitHub)" <gi...@apache.org>.
danielcweeks commented on code in PR #12808:
URL: https://github.com/apache/iceberg/pull/12808#discussion_r2072202094


##########
bigquery/src/main/java/org/apache/iceberg/gcp/bigquery/BigQueryTableOperations.java:
##########
@@ -0,0 +1,296 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.iceberg.gcp.bigquery;
+
+import com.google.api.services.bigquery.model.ExternalCatalogTableOptions;
+import com.google.api.services.bigquery.model.Table;
+import com.google.api.services.bigquery.model.TableReference;
+import java.util.Locale;
+import java.util.Map;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hive.common.StatsSetupConst;
+import org.apache.iceberg.BaseMetastoreOperations;
+import org.apache.iceberg.BaseMetastoreTableOperations;
+import org.apache.iceberg.SnapshotSummary;
+import org.apache.iceberg.TableMetadata;
+import org.apache.iceberg.TableProperties;
+import org.apache.iceberg.exceptions.AlreadyExistsException;
+import org.apache.iceberg.exceptions.CommitFailedException;
+import org.apache.iceberg.exceptions.CommitStateUnknownException;
+import org.apache.iceberg.exceptions.NoSuchTableException;
+import org.apache.iceberg.exceptions.ValidationException;
+import org.apache.iceberg.hadoop.ConfigProperties;
+import org.apache.iceberg.io.FileIO;
+import org.apache.iceberg.relocated.com.google.common.collect.Maps;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/** Handles BigQuery metastore table operations. */
+public final class BigQueryTableOperations extends BaseMetastoreTableOperations {
+
+  private static final Logger LOG = LoggerFactory.getLogger(BigQueryTableOperations.class);
+
+  private static final String TABLE_PROPERTIES_BQ_CONNECTION = "bq_connection";
+
+  private final BigQueryMetastoreClient client;
+  private final FileIO fileIO;
+  private final TableReference tableReference;
+  private final Configuration conf;
+
+  BigQueryTableOperations(
+      BigQueryMetastoreClient client,
+      FileIO fileIO,
+      TableReference tableReference,
+      Configuration conf) {
+    this.client = client;
+    this.fileIO = fileIO;
+    this.tableReference = tableReference;
+    this.conf = conf;
+  }
+
+  // The doRefresh method should provide implementation on how to get the metadata location.
+  @Override
+  public void doRefresh() {
+    // Must default to null.
+    String metadataLocation = null;
+    try {
+      metadataLocation =
+          loadMetadataLocationOrThrow(
+              client.load(this.tableReference).getExternalCatalogTableOptions());
+    } catch (NoSuchTableException e) {
+      if (currentMetadataLocation() != null) {
+        // Re-throws the exception because the table must exist in this case.
+        throw e;
+      }
+    }
+    refreshFromMetadataLocation(metadataLocation);
+  }
+
+  // The doCommit method should provide implementation on how to update with metadata location
+  // atomically
+  @Override
+  public void doCommit(TableMetadata base, TableMetadata metadata) {
+    String newMetadataLocation =
+        base == null && metadata.metadataFileLocation() != null
+            ? metadata.metadataFileLocation()
+            : writeNewMetadata(metadata, currentVersion() + 1);
+    BaseMetastoreOperations.CommitStatus commitStatus =
+        BaseMetastoreOperations.CommitStatus.FAILURE;
+    try {
+      if (base == null) {
+        createTable(newMetadataLocation, metadata);
+      } else {
+        updateTable(base.metadataFileLocation(), newMetadataLocation, metadata);
+      }
+      commitStatus = BaseMetastoreOperations.CommitStatus.SUCCESS;
+    } catch (CommitFailedException | CommitStateUnknownException e) {
+      throw e;
+    } catch (Throwable e) {
+      LOG.error("Exception thrown on commit: ", e);
+      if (e instanceof AlreadyExistsException) {
+        throw e;
+      }
+      commitStatus =
+          BaseMetastoreOperations.CommitStatus.valueOf(
+              checkCommitStatus(newMetadataLocation, metadata).name());
+      if (commitStatus == BaseMetastoreOperations.CommitStatus.FAILURE) {
+        throw new CommitFailedException(e, "Failed to commit");
+      }
+      if (commitStatus == BaseMetastoreOperations.CommitStatus.UNKNOWN) {
+        throw new CommitStateUnknownException(e);
+      }
+    } finally {
+      try {
+        if (commitStatus == BaseMetastoreOperations.CommitStatus.FAILURE) {
+          LOG.warn("Failed to commit updates to table {}", tableName());
+          io().deleteFile(newMetadataLocation);
+        }
+      } catch (RuntimeException e) {
+        LOG.error(
+            "Failed to cleanup metadata file at {} for table {}",
+            newMetadataLocation,
+            tableName(),
+            e);
+      }
+    }
+  }
+
+  @Override
+  public String tableName() {
+    return String.format("%s.%s", tableReference.getDatasetId(), tableReference.getTableId());
+  }
+
+  @Override
+  public FileIO io() {
+    return fileIO;
+  }
+
+  private void createTable(String newMetadataLocation, TableMetadata metadata) {
+    LOG.debug("Creating a new Iceberg table: {}", tableName());
+    Table tableBuilder = makeNewTable(metadata, newMetadataLocation);
+    tableBuilder.setTableReference(this.tableReference);
+    addConnectionIfProvided(tableBuilder, metadata.properties());
+
+    client.create(tableBuilder);
+  }
+
+  private void addConnectionIfProvided(Table tableBuilder, Map<String, String> metadataProperties) {
+    if (metadataProperties.containsKey(TABLE_PROPERTIES_BQ_CONNECTION)) {
+      tableBuilder
+          .getExternalCatalogTableOptions()
+          .setConnectionId(metadataProperties.get(TABLE_PROPERTIES_BQ_CONNECTION));
+    }
+  }
+
+  /** Update table properties with concurrent update detection using etag. */
+  private void updateTable(
+      String oldMetadataLocation, String newMetadataLocation, TableMetadata metadata) {
+    Table table = client.load(this.tableReference);
+    if (table.getEtag().isEmpty()) {
+      throw new ValidationException(
+          "Etag of legacy table %s is empty, manually update the table via the BigQuery API or"
+              + " recreate and retry",
+          tableName());
+    }
+    ExternalCatalogTableOptions options = table.getExternalCatalogTableOptions();
+    addConnectionIfProvided(table, metadata.properties());
+
+    // If `metadataLocationFromMetastore` is different from metadata location of base, it means
+    // someone has updated metadata location in metastore, which is a conflict update.
+    String metadataLocationFromMetastore =
+        options.getParameters().getOrDefault(METADATA_LOCATION_PROP, "");
+    if (!metadataLocationFromMetastore.isEmpty()
+        && !metadataLocationFromMetastore.equals(oldMetadataLocation)) {
+      throw new CommitFailedException(
+          "Cannot commit base metadata location '%s' is not same as the current table metadata location '%s' for"
+              + " %s.%s",
+          oldMetadataLocation,
+          metadataLocationFromMetastore,
+          tableReference.getDatasetId(),
+          tableReference.getTableId());
+    }
+
+    options.setParameters(buildTableParameters(newMetadataLocation, metadata));
+    try {
+      client.update(tableReference, table);
+    } catch (ValidationException e) {
+      if (e.getMessage().toLowerCase(Locale.ENGLISH).contains("etag mismatch")) {
+        throw new CommitFailedException(
+            "Updating table failed due to conflict updates (etag mismatch). Retry the update");
+      }
+
+      throw e;
+    }
+  }
+
+  // To make the table queryable from Hive, the user would likely be setting the HIVE_ENGINE_ENABLED
+  // parameter.
+  //
+  // TODO: We need to make a decision on how to make the table queryable from Hive.
+  // (could be a server side change or a client side change - that's TBD).
+  private Table makeNewTable(TableMetadata metadata, String metadataFileLocation) {
+    return new Table()
+        .setExternalCatalogTableOptions(
+            BigQueryMetastoreUtils.createExternalCatalogTableOptions(
+                metadata.location(),
+                buildTableParameters(metadataFileLocation, metadata),
+                hiveEngineEnabled(metadata)));

Review Comment:
   I think I commented elsewhere, but I don't feel like this should be a flag.  It seems like we would just always include those properties.  I would assume spark and other engines would ignore it entirely and ideally it would just be set by the sever. 



-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org

For queries about this service, please contact Infrastructure at:
users@infra.apache.org


---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For additional commands, e-mail: issues-help@iceberg.apache.org


Re: [PR] Catalog: Add BigQuery Metastore Catalog Support [iceberg]

Posted by "danielcweeks (via GitHub)" <gi...@apache.org>.
danielcweeks commented on code in PR #12808:
URL: https://github.com/apache/iceberg/pull/12808#discussion_r2072147433


##########
bigquery/src/main/java/org/apache/iceberg/gcp/bigquery/BigQueryMetastoreCatalog.java:
##########
@@ -0,0 +1,366 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.iceberg.gcp.bigquery;
+
+import com.google.api.services.bigquery.model.Dataset;
+import com.google.api.services.bigquery.model.DatasetList.Datasets;
+import com.google.api.services.bigquery.model.DatasetReference;
+import com.google.api.services.bigquery.model.ExternalCatalogDatasetOptions;
+import com.google.api.services.bigquery.model.TableReference;
+import com.google.cloud.ServiceOptions;
+import com.google.cloud.bigquery.BigQueryOptions;
+import java.io.IOException;
+import java.io.UncheckedIOException;
+import java.security.GeneralSecurityException;
+import java.util.List;
+import java.util.Locale;
+import java.util.Map;
+import java.util.Objects;
+import java.util.Set;
+import org.apache.hadoop.conf.Configurable;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.iceberg.BaseMetastoreCatalog;
+import org.apache.iceberg.CatalogProperties;
+import org.apache.iceberg.CatalogUtil;
+import org.apache.iceberg.TableMetadata;
+import org.apache.iceberg.TableOperations;
+import org.apache.iceberg.catalog.Namespace;
+import org.apache.iceberg.catalog.SupportsNamespaces;
+import org.apache.iceberg.catalog.TableIdentifier;
+import org.apache.iceberg.exceptions.NoSuchNamespaceException;
+import org.apache.iceberg.exceptions.NoSuchTableException;
+import org.apache.iceberg.io.FileIO;
+import org.apache.iceberg.relocated.com.google.common.annotations.VisibleForTesting;
+import org.apache.iceberg.relocated.com.google.common.base.Preconditions;
+import org.apache.iceberg.relocated.com.google.common.base.Strings;
+import org.apache.iceberg.relocated.com.google.common.collect.ImmutableList;
+import org.apache.iceberg.relocated.com.google.common.collect.ImmutableMap;
+import org.apache.iceberg.relocated.com.google.common.collect.Maps;
+import org.apache.iceberg.util.LocationUtil;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/** Iceberg Bigquery Metastore Catalog implementation. */
+public class BigQueryMetastoreCatalog extends BaseMetastoreCatalog
+    implements SupportsNamespaces, Configurable {
+
+  // User provided properties.
+  public static final String PROJECT_ID = "gcp.bigquery.project-id";
+  public static final String GCP_LOCATION = "gcp.bigquery.location";
+  public static final String LIST_ALL_TABLES = "gcp.bigquery.list-all-tables";
+
+  private static final Logger LOG = LoggerFactory.getLogger(BigQueryMetastoreCatalog.class);
+
+  private static final String DEFAULT_GCP_LOCATION = "us";
+
+  private String catalogName;
+  private Map<String, String> catalogProperties;
+  private FileIO fileIO;
+  private Configuration conf;
+  private String projectId;
+  private String projectLocation;
+  private BigQueryMetastoreClient client;
+  private boolean listAllTables;
+  private String warehouseLocation;
+
+  public BigQueryMetastoreCatalog() {}
+
+  @Override
+  public void initialize(String name, Map<String, String> properties) {
+    Preconditions.checkArgument(
+        properties.containsKey(PROJECT_ID),
+        "Invalid GCP project: %s must be specified",
+        PROJECT_ID);
+
+    this.projectId = properties.get(PROJECT_ID);
+    this.projectLocation = properties.getOrDefault(GCP_LOCATION, DEFAULT_GCP_LOCATION);
+
+    BigQueryOptions options =
+        BigQueryOptions.newBuilder()
+            .setProjectId(projectId)
+            .setLocation(projectLocation)
+            .setRetrySettings(ServiceOptions.getDefaultRetrySettings())
+            .build();
+
+    try {
+      client = new BigQueryMetastoreClientImpl(options);
+    } catch (IOException e) {
+      throw new UncheckedIOException("Creating BigQuery client failed", e);
+    } catch (GeneralSecurityException e) {
+      throw new RuntimeException("Creating BigQuery client failed due to a security issue", e);
+    }
+
+    initialize(name, properties, projectId, projectLocation, client);
+  }
+
+  @VisibleForTesting
+  void initialize(
+      String name,
+      Map<String, String> properties,
+      String initialProjectId,
+      String initialLocation,
+      BigQueryMetastoreClient bigQueryMetaStoreClient) {
+    Preconditions.checkArgument(bigQueryMetaStoreClient != null, "Invalid BigQuery client: null");
+    this.catalogName = name;
+    this.catalogProperties = ImmutableMap.copyOf(properties);
+    this.projectId = initialProjectId;
+    this.projectLocation = initialLocation;
+    this.client = bigQueryMetaStoreClient;
+
+    if (this.conf == null) {
+      LOG.warn("No configuration was set, using the default environment Configuration");
+      this.conf = new Configuration();
+    }
+
+    LOG.info("Using BigQuery Metastore Iceberg Catalog: {}", name);
+
+    if (properties.containsKey(CatalogProperties.WAREHOUSE_LOCATION)) {
+      this.warehouseLocation =
+          LocationUtil.stripTrailingSlash(properties.get(CatalogProperties.WAREHOUSE_LOCATION));
+    }
+
+    this.fileIO =
+        CatalogUtil.loadFileIO(
+            properties.getOrDefault(
+                CatalogProperties.FILE_IO_IMPL, "org.apache.iceberg.io.ResolvingFileIO"),
+            properties,
+            conf);
+
+    this.listAllTables = Boolean.parseBoolean(properties.getOrDefault(LIST_ALL_TABLES, "true"));
+  }
+
+  @Override
+  protected TableOperations newTableOps(TableIdentifier identifier) {
+    return new BigQueryTableOperations(client, fileIO, toTableReference(identifier), conf);
+  }
+
+  @Override
+  protected String defaultWarehouseLocation(TableIdentifier identifier) {
+    String locationUri = null;
+    DatasetReference datasetReference = toDatasetReference(identifier.namespace());
+    Dataset dataset = client.load(datasetReference);
+    if (dataset != null && dataset.getExternalCatalogDatasetOptions() != null) {
+      locationUri = dataset.getExternalCatalogDatasetOptions().getDefaultStorageLocationUri();
+    }
+
+    return String.format(
+        "%s/%s",
+        Strings.isNullOrEmpty(locationUri)
+            ? createDefaultStorageLocationUri(datasetReference.getDatasetId())
+            : LocationUtil.stripTrailingSlash(locationUri),
+        identifier.name());
+  }
+
+  @Override
+  public List<TableIdentifier> listTables(Namespace namespace) {
+    validateNamespace(namespace);
+
+    return client.list(toDatasetReference(namespace), listAllTables).stream()
+        .map(
+            table -> TableIdentifier.of(namespace.level(0), table.getTableReference().getTableId()))
+        .collect(ImmutableList.toImmutableList());
+  }
+
+  @Override
+  public boolean dropTable(TableIdentifier identifier, boolean purge) {
+    try {
+      TableOperations ops = newTableOps(identifier);
+      TableMetadata lastMetadata = ops.current();
+
+      client.delete(toTableReference(identifier));
+
+      if (purge && lastMetadata != null) {
+        CatalogUtil.dropTableData(ops.io(), lastMetadata);
+      }
+    } catch (NoSuchTableException e) {
+      return false;
+    }
+
+    return true;
+  }
+
+  @Override
+  public void renameTable(TableIdentifier from, TableIdentifier to) {
+    // TODO: Enable once supported by BigQuery API.
+    throw new UnsupportedOperationException("Table rename operation is unsupported.");
+  }
+
+  @Override
+  public void createNamespace(Namespace namespace, Map<String, String> metadata) {
+    Dataset builder = new Dataset();
+    DatasetReference datasetReference = toDatasetReference(namespace);
+    builder.setLocation(this.projectLocation);
+    builder.setDatasetReference(datasetReference);
+    builder.setExternalCatalogDatasetOptions(
+        BigQueryMetastoreUtils.createExternalCatalogDatasetOptions(
+            createDefaultStorageLocationUri(datasetReference.getDatasetId()), metadata));
+
+    client.create(builder);
+  }
+
+  @Override
+  public List<Namespace> listNamespaces() {
+    try {
+      return listNamespaces(Namespace.empty());
+    } catch (NoSuchNamespaceException e) {
+      return ImmutableList.of();
+    }
+  }
+
+  /**
+   * Since this catalog only supports one-level namespaces, it always returns an empty list unless
+   * passed an empty namespace to list all namespaces within the catalog.
+   */
+  @Override
+  public List<Namespace> listNamespaces(Namespace namespace) {
+    List<Datasets> allDatasets = client.list(projectId);
+
+    ImmutableList<Namespace> namespaces =
+        allDatasets.stream().map(this::toNamespace).collect(ImmutableList.toImmutableList());
+    if (namespaces.isEmpty()) {
+      throw new NoSuchNamespaceException("Namespace does not exist: %s", namespace);
+    }
+
+    return namespaces;
+  }
+
+  @Override
+  public boolean dropNamespace(Namespace namespace) {
+    try {
+      client.delete(toDatasetReference(namespace));
+      // We don't delete the data folder for safety, which aligns with Hive Metastore's default
+      // behavior.
+      // We can support database or catalog level config controlling file deletion in the future.
+      return true;
+    } catch (NoSuchNamespaceException e) {
+      return false;
+    }
+  }
+
+  @Override
+  public boolean setProperties(Namespace namespace, Map<String, String> properties) {
+    Dataset dataset = client.load(toDatasetReference(namespace));
+
+    ExternalCatalogDatasetOptions existingOptions = dataset.getExternalCatalogDatasetOptions();
+    Map<String, String> existingParameters =
+        existingOptions != null ? existingOptions.getParameters() : null;
+
+    Map<String, String> newParameters = Maps.newHashMap();
+    if (existingParameters != null) {
+      newParameters.putAll(existingParameters);
+    }
+
+    newParameters.putAll(properties);
+
+    if (Objects.equals(existingParameters, newParameters)) {
+      // No change in parameters detected
+      return false;
+    }
+
+    client.setParameters(toDatasetReference(namespace), properties);
+    return true;
+  }
+
+  @Override
+  public boolean removeProperties(Namespace namespace, Set<String> properties) {
+    client.removeParameters(toDatasetReference(namespace), properties);
+    return true;

Review Comment:
   This isn't really adhering to the API behavior because it's returning true if the properties are actually removed.  It appears that HiveCatalog has this same behavior, but JDBCCatalog does it correctly.



-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org

For queries about this service, please contact Infrastructure at:
users@infra.apache.org


---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For additional commands, e-mail: issues-help@iceberg.apache.org


Re: [PR] Catalog: Add BigQuery Metastore Catalog Support [iceberg]

Posted by "talatuyarer (via GitHub)" <gi...@apache.org>.
talatuyarer commented on code in PR #12808:
URL: https://github.com/apache/iceberg/pull/12808#discussion_r2051079976


##########
bigquery/src/main/java/org/apache/iceberg/gcp/bigquery/BigQueryMetaStoreClientImpl.java:
##########
@@ -0,0 +1,620 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.iceberg.gcp.bigquery;
+
+import com.google.api.client.googleapis.javanet.GoogleNetHttpTransport;
+import com.google.api.client.http.HttpHeaders;
+import com.google.api.client.http.HttpResponse;
+import com.google.api.client.http.HttpResponseException;
+import com.google.api.client.http.HttpStatusCodes;
+import com.google.api.client.json.gson.GsonFactory;
+import com.google.api.client.util.Data;
+import com.google.api.services.bigquery.Bigquery;
+import com.google.api.services.bigquery.BigqueryScopes;
+import com.google.api.services.bigquery.model.Dataset;
+import com.google.api.services.bigquery.model.DatasetList;
+import com.google.api.services.bigquery.model.DatasetList.Datasets;
+import com.google.api.services.bigquery.model.DatasetReference;
+import com.google.api.services.bigquery.model.ExternalCatalogDatasetOptions;
+import com.google.api.services.bigquery.model.ExternalCatalogTableOptions;
+import com.google.api.services.bigquery.model.Table;
+import com.google.api.services.bigquery.model.TableList;
+import com.google.api.services.bigquery.model.TableList.Tables;
+import com.google.api.services.bigquery.model.TableReference;
+import com.google.api.services.bigquery.model.TableSchema;
+import com.google.auth.http.HttpCredentialsAdapter;
+import com.google.auth.oauth2.GoogleCredentials;
+import com.google.cloud.BaseServiceException;
+import com.google.cloud.ExceptionHandler;
+import com.google.cloud.bigquery.BigQueryErrorMessages;
+import com.google.cloud.bigquery.BigQueryOptions;
+import com.google.cloud.bigquery.BigQueryRetryConfig;
+import com.google.cloud.bigquery.BigQueryRetryHelper;
+import java.io.IOException;
+import java.nio.charset.StandardCharsets;
+import java.security.GeneralSecurityException;
+import java.util.List;
+import java.util.Locale;
+import java.util.Map;
+import java.util.Set;
+import java.util.concurrent.Callable;
+import java.util.stream.Collectors;
+import java.util.stream.Stream;
+import org.apache.iceberg.BaseMetastoreTableOperations;
+import org.apache.iceberg.exceptions.AlreadyExistsException;
+import org.apache.iceberg.exceptions.BadRequestException;
+import org.apache.iceberg.exceptions.ForbiddenException;
+import org.apache.iceberg.exceptions.NoSuchIcebergTableException;
+import org.apache.iceberg.exceptions.NoSuchNamespaceException;
+import org.apache.iceberg.exceptions.NoSuchTableException;
+import org.apache.iceberg.exceptions.NotAuthorizedException;
+import org.apache.iceberg.exceptions.RuntimeIOException;
+import org.apache.iceberg.exceptions.ServiceFailureException;
+import org.apache.iceberg.exceptions.ServiceUnavailableException;
+import org.apache.iceberg.exceptions.ValidationException;
+import org.apache.iceberg.relocated.com.google.common.base.Preconditions;
+import org.apache.iceberg.relocated.com.google.common.collect.Lists;
+import org.apache.iceberg.relocated.com.google.common.collect.Maps;
+
+/** A client of Google Bigquery Metastore functions over the BigQuery service. */
+public final class BigQueryMetaStoreClientImpl implements BigQueryMetaStoreClient {
+
+  public static final ExceptionHandler.Interceptor EXCEPTION_HANDLER_INTERCEPTOR =
+      new ExceptionHandler.Interceptor() {
+
+        private static final long serialVersionUID = -8429573486870467828L;
+
+        @Override
+        public RetryResult afterEval(Exception exception, RetryResult retryResult) {
+          return ExceptionHandler.Interceptor.RetryResult.CONTINUE_EVALUATION;
+        }
+
+        @Override
+        public RetryResult beforeEval(Exception exception) {
+          if (exception instanceof BaseServiceException) {
+            boolean retriable = ((BaseServiceException) exception).isRetryable();
+            return retriable
+                ? ExceptionHandler.Interceptor.RetryResult.RETRY
+                : ExceptionHandler.Interceptor.RetryResult.CONTINUE_EVALUATION;
+          }
+          return ExceptionHandler.Interceptor.RetryResult.CONTINUE_EVALUATION;
+        }
+      };
+
+  private final Bigquery client;
+  private final BigQueryOptions bigqueryOptions;
+
+  // Retry config with error messages and regex for rate limit exceeded errors.
+  private static final BigQueryRetryConfig DEFAULT_RETRY_CONFIG =
+      BigQueryRetryConfig.newBuilder()
+          .retryOnMessage(BigQueryErrorMessages.RATE_LIMIT_EXCEEDED_MSG)
+          .retryOnMessage(BigQueryErrorMessages.JOB_RATE_LIMIT_EXCEEDED_MSG)
+          .retryOnRegEx(BigQueryErrorMessages.RetryRegExPatterns.RATE_LIMIT_EXCEEDED_REGEX)
+          .build();
+
+  public static final ExceptionHandler BIGQUERY_EXCEPTION_HANDLER =
+      ExceptionHandler.newBuilder()
+          .abortOn(RuntimeException.class)
+          .retryOn(java.net.ConnectException.class) // retry on Connection Exception
+          .retryOn(java.net.UnknownHostException.class) // retry on UnknownHostException
+          .addInterceptors(EXCEPTION_HANDLER_INTERCEPTOR)
+          .build();
+
+  /** Constructs a client of the Google BigQuery service. */
+  public BigQueryMetaStoreClientImpl(BigQueryOptions options)
+      throws IOException, GeneralSecurityException {
+    // Initialize client that will be used to send requests. This client only needs to be created
+    // once, and can be reused for multiple requests
+    HttpCredentialsAdapter httpCredentialsAdapter =
+        new HttpCredentialsAdapter(
+            GoogleCredentials.getApplicationDefault().createScoped(BigqueryScopes.all()));
+    this.client =
+        new Bigquery.Builder(
+                GoogleNetHttpTransport.newTrustedTransport(),
+                GsonFactory.getDefaultInstance(),
+                httpRequest -> {
+                  httpCredentialsAdapter.initialize(httpRequest);
+                  // Instead of throwing exceptions, analyze the HttpResponse object and inspect its
+                  // status code. This will allow BigQuery API errors to be converted into Iceberg
+                  // exceptions.
+                  httpRequest.setThrowExceptionOnExecuteError(false);
+                })
+            .setApplicationName("BigQuery Metastore Iceberg Catalog Plugin")
+            .build();
+    this.bigqueryOptions = options;
+  }
+
+  @Override
+  public Dataset createDataset(Dataset dataset) {
+    // TODO (b/399885863): Ensure Dataset creation is idempotent when handling retries.

Review Comment:
   it is internal Jira ticket id. Removed  most of places. 



-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org

For queries about this service, please contact Infrastructure at:
users@infra.apache.org


---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For additional commands, e-mail: issues-help@iceberg.apache.org


Re: [PR] Catalog: Add BigQuery Metastore Catalog Support [iceberg]

Posted by "talatuyarer (via GitHub)" <gi...@apache.org>.
talatuyarer commented on code in PR #12808:
URL: https://github.com/apache/iceberg/pull/12808#discussion_r2054476006


##########
bigquery/src/main/java/org/apache/iceberg/gcp/bigquery/BigQueryMetastoreCatalog.java:
##########
@@ -0,0 +1,402 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.iceberg.gcp.bigquery;
+
+import com.google.api.services.bigquery.model.Dataset;
+import com.google.api.services.bigquery.model.DatasetList.Datasets;
+import com.google.api.services.bigquery.model.DatasetReference;
+import com.google.api.services.bigquery.model.ExternalCatalogDatasetOptions;
+import com.google.api.services.bigquery.model.TableReference;
+import com.google.cloud.ServiceOptions;
+import com.google.cloud.bigquery.BigQueryOptions;
+import java.io.IOException;
+import java.security.GeneralSecurityException;
+import java.util.List;
+import java.util.Locale;
+import java.util.Map;
+import java.util.Objects;
+import java.util.Set;
+import org.apache.hadoop.conf.Configurable;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.iceberg.BaseMetastoreCatalog;
+import org.apache.iceberg.CatalogProperties;
+import org.apache.iceberg.CatalogUtil;
+import org.apache.iceberg.TableMetadata;
+import org.apache.iceberg.TableOperations;
+import org.apache.iceberg.catalog.Namespace;
+import org.apache.iceberg.catalog.SupportsNamespaces;
+import org.apache.iceberg.catalog.TableIdentifier;
+import org.apache.iceberg.exceptions.AlreadyExistsException;
+import org.apache.iceberg.exceptions.NoSuchNamespaceException;
+import org.apache.iceberg.exceptions.NoSuchTableException;
+import org.apache.iceberg.exceptions.ServiceFailureException;
+import org.apache.iceberg.exceptions.ValidationException;
+import org.apache.iceberg.io.FileIO;
+import org.apache.iceberg.relocated.com.google.common.annotations.VisibleForTesting;
+import org.apache.iceberg.relocated.com.google.common.base.Preconditions;
+import org.apache.iceberg.relocated.com.google.common.base.Strings;
+import org.apache.iceberg.relocated.com.google.common.collect.ImmutableList;
+import org.apache.iceberg.relocated.com.google.common.collect.ImmutableMap;
+import org.apache.iceberg.relocated.com.google.common.collect.Maps;
+import org.apache.iceberg.util.LocationUtil;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/** Iceberg Bigquery Metastore Catalog implementation. */
+public final class BigQueryMetastoreCatalog extends BaseMetastoreCatalog
+    implements SupportsNamespaces, Configurable {
+
+  // User provided properties.
+  public static final String PROPERTIES_KEY_GCP_PROJECT = "gcp-project";
+  public static final String PROPERTIES_KEY_GCP_LOCATION = "gcp-location";
+  public static final String PROPERTIES_KEY_FILTER_UNSUPPORTED_TABLES = "filter-unsupported-tables";
+  public static final String PROPERTIES_KEY_TESTING_ENABLED = "testing-enabled";

Review Comment:
   testing-enabled is for FakeClient. When we call CatalogUtils.loadCatalog I pass that argument to use FakeClient. 



-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org

For queries about this service, please contact Infrastructure at:
users@infra.apache.org


---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For additional commands, e-mail: issues-help@iceberg.apache.org


Re: [PR] Catalog: Add BigQuery Metastore Catalog Support [iceberg]

Posted by "talatuyarer (via GitHub)" <gi...@apache.org>.
talatuyarer commented on code in PR #12808:
URL: https://github.com/apache/iceberg/pull/12808#discussion_r2064334939


##########
bigquery/src/test/java/org/apache/iceberg/gcp/bigquery/BigQueryMetastoreTestUtils.java:
##########
@@ -0,0 +1,87 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.iceberg.gcp.bigquery;
+
+import static org.apache.iceberg.types.Types.NestedField.required;
+import static org.assertj.core.api.Assertions.assertThat;
+
+import com.google.api.services.bigquery.model.ExternalCatalogTableOptions;
+import com.google.api.services.bigquery.model.StorageDescriptor;
+import com.google.api.services.bigquery.model.Table;
+import com.google.api.services.bigquery.model.TableReference;
+import java.io.File;
+import java.io.IOException;
+import java.util.Collections;
+import java.util.Optional;
+import org.apache.commons.io.FileUtils;
+import org.apache.commons.io.filefilter.TrueFileFilter;
+import org.apache.iceberg.Schema;
+import org.apache.iceberg.catalog.TableIdentifier;
+import org.apache.iceberg.types.Types;
+
+/** Test utility methods for BigQuery Metastore Iceberg catalog. */
+public final class BigQueryMetastoreTestUtils {

Review Comment:
   Ok I removed this class and move the test under Test Class



-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org

For queries about this service, please contact Infrastructure at:
users@infra.apache.org


---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For additional commands, e-mail: issues-help@iceberg.apache.org


Re: [PR] Catalog: Add BigQuery Metastore Catalog Support [iceberg]

Posted by "talatuyarer (via GitHub)" <gi...@apache.org>.
talatuyarer commented on code in PR #12808:
URL: https://github.com/apache/iceberg/pull/12808#discussion_r2059760489


##########
bigquery/src/test/java/org/apache/iceberg/gcp/bigquery/TestBigQueryCatalog.java:
##########
@@ -0,0 +1,673 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.iceberg.gcp.bigquery;
+
+import static org.apache.iceberg.CatalogUtil.ICEBERG_CATALOG_TYPE;
+import static org.apache.iceberg.CatalogUtil.ICEBERG_CATALOG_TYPE_BIGQUERY;
+import static org.apache.iceberg.gcp.bigquery.BigQueryMetastoreCatalog.PROJECT_ID;
+import static org.apache.iceberg.gcp.bigquery.BigQueryMetastoreCatalog.TESTING_ENABLED;
+import static org.apache.iceberg.types.Types.NestedField.optional;
+import static org.assertj.core.api.Assertions.assertThat;
+import static org.assertj.core.api.Assertions.assertThatThrownBy;
+
+import java.io.File;
+import java.util.List;
+import java.util.Map;
+import org.apache.iceberg.CatalogProperties;
+import org.apache.iceberg.CatalogUtil;
+import org.apache.iceberg.PartitionSpec;
+import org.apache.iceberg.Schema;
+import org.apache.iceberg.SortOrder;
+import org.apache.iceberg.Table;
+import org.apache.iceberg.TableUtil;
+import org.apache.iceberg.Transaction;
+import org.apache.iceberg.catalog.CatalogTests;
+import org.apache.iceberg.catalog.Namespace;
+import org.apache.iceberg.catalog.TableIdentifier;
+import org.apache.iceberg.exceptions.NoSuchNamespaceException;
+import org.apache.iceberg.exceptions.NoSuchTableException;
+import org.apache.iceberg.expressions.Expressions;
+import org.apache.iceberg.relocated.com.google.common.collect.ImmutableMap;
+import org.apache.iceberg.types.Types;
+import org.junit.jupiter.api.AfterEach;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Disabled;
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.io.TempDir;
+import org.junit.jupiter.params.ParameterizedTest;
+import org.junit.jupiter.params.provider.ValueSource;
+
+public class TestBigQueryCatalog extends CatalogTests<BigQueryMetastoreCatalog> {
+
+  private static final String GCP_PROJECT = "project-id";
+  private static final String CATALOG_ID = "catalog-name";
+
+  // another schema that is not the same
+  protected static final Schema OTHER_SCHEMA_OPTIONAL =
+      new Schema(optional(1, "some_id", Types.IntegerType.get()));
+
+  // Schema passed to create tables
+  protected static final Schema SCHEMA_OPTIONAL =
+      new Schema(
+          optional(3, "id", Types.IntegerType.get(), "unique ID 🤪"),
+          optional(4, "data", Types.StringType.get()));
+
+  // This is the actual schema for the table, with column IDs reassigned
+  protected static final Schema REPLACE_SCHEMA_OPTIONAL =
+      new Schema(
+          optional(2, "id", Types.IntegerType.get(), "unique ID 🤪"),
+          optional(3, "data", Types.StringType.get()));
+
+  protected static final PartitionSpec REPLACE_SPEC_OPTIONAL =
+      PartitionSpec.builderFor(REPLACE_SCHEMA_OPTIONAL).bucket("id", 16).withSpecId(1).build();
+
+  protected static final SortOrder REPLACE_WRITE_ORDER_OPTIONAL =
+      SortOrder.builderFor(REPLACE_SCHEMA_OPTIONAL)
+          .asc(Expressions.bucket("id", 16))
+          .asc("id")
+          .build();
+
+  @TempDir private File tempFolder;
+  private BigQueryMetastoreCatalog catalog;
+  private String warehouseLocation;
+
+  @BeforeEach
+  public void before() throws Exception {
+    catalog = initCatalog(CATALOG_ID, ImmutableMap.of());
+  }
+
+  @AfterEach
+  public void after() throws Exception {
+    // Drop all tables in all datasets
+    List<Namespace> namespaces = catalog().listNamespaces(Namespace.empty());
+    for (Namespace namespace : namespaces) {
+      List<TableIdentifier> tables = catalog().listTables(namespace);
+      for (TableIdentifier table : tables) {
+        try {
+          catalog().dropTable(table, true); // drop with purge
+        } catch (NoSuchTableException e) {
+          // Table might be dropped by a previous iteration, ignore
+        }
+      }
+
+      // Drop all datasets (except any initial ones)
+      try {
+        catalog().dropNamespace(namespace);
+      } catch (NoSuchNamespaceException e) {
+        // Namespace might be dropped by a previous iteration, ignore
+      }
+    }
+  }
+
+  @Override
+  protected boolean requiresNamespaceCreate() {
+    return true;
+  }
+
+  @Override
+  protected boolean supportsNamesWithSlashes() {
+    return false;
+  }
+
+  @Override
+  protected boolean supportsNamesWithDot() {
+    return false;
+  }
+
+  @Override
+  public BigQueryMetastoreCatalog catalog() {
+    return catalog;
+  }
+
+  @Override
+  protected BigQueryMetastoreCatalog initCatalog(
+      String catalogName, Map<String, String> additionalProperties) {
+
+    warehouseLocation = tempFolder.toPath().resolve("hive-warehouse").toString();
+
+    Map<String, String> properties =
+        Map.of(
+            ICEBERG_CATALOG_TYPE,
+            ICEBERG_CATALOG_TYPE_BIGQUERY,
+            PROJECT_ID,
+            GCP_PROJECT,
+            CatalogProperties.WAREHOUSE_LOCATION,
+            warehouseLocation,
+            TESTING_ENABLED,
+            "true",
+            CatalogProperties.TABLE_DEFAULT_PREFIX + "default-key1",
+            "catalog-default-key1",
+            CatalogProperties.TABLE_DEFAULT_PREFIX + "default-key2",
+            "catalog-default-key2",
+            CatalogProperties.TABLE_DEFAULT_PREFIX + "override-key3",
+            "catalog-default-key3",
+            CatalogProperties.TABLE_OVERRIDE_PREFIX + "override-key3",
+            "catalog-override-key3",
+            CatalogProperties.TABLE_OVERRIDE_PREFIX + "override-key4",
+            "catalog-override-key4");
+
+    BigQueryMetastoreCatalog tmpCatalog =
+        (BigQueryMetastoreCatalog)
+            CatalogUtil.buildIcebergCatalog(
+                catalogName,
+                ImmutableMap.<String, String>builder()
+                    .putAll(properties)
+                    .putAll(additionalProperties)
+                    .build(),
+                null);
+
+    return tmpCatalog;
+  }
+
+  @Test
+  public void testRenameTable() {
+    if (requiresNamespaceCreate()) {
+      catalog.createNamespace(NS);
+    }
+
+    assertThat(catalog.tableExists(TABLE))
+        .as("Source table should not exist before create")
+        .isFalse();
+
+    catalog.buildTable(TABLE, SCHEMA).create();
+    assertThat(catalog.tableExists(TABLE)).as("Table should exist after create").isTrue();
+
+    assertThat(catalog.tableExists(RENAMED_TABLE))
+        .as("Destination table should not exist before rename")
+        .isFalse();
+
+    assertThatThrownBy(() -> catalog.renameTable(TABLE, RENAMED_TABLE))
+        .isInstanceOf(UnsupportedOperationException.class)
+        .hasMessageContaining("Table rename operation is unsupported");
+
+    catalog.dropTable(TABLE);
+  }
+
+  @Disabled(
+      "BigQuery does not allow adding a REQUIRED column to an existing table schema. "
+          + "When this operation is attempted, an exception gets thrown.")
+  @Test
+  public void testOverrideTablePropertiesReplaceTransaction() {}

Review Comment:
   Sounds good to me. I can remove all optional field tests now. What about V3 spec tests such as createTableTransaction or testCreateTableWithDefaultColumnValue ? Bigquery does not support some V3 features. I will disable them ? 



##########
bigquery/src/test/java/org/apache/iceberg/gcp/bigquery/TestBigQueryCatalog.java:
##########
@@ -0,0 +1,673 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.iceberg.gcp.bigquery;
+
+import static org.apache.iceberg.CatalogUtil.ICEBERG_CATALOG_TYPE;
+import static org.apache.iceberg.CatalogUtil.ICEBERG_CATALOG_TYPE_BIGQUERY;
+import static org.apache.iceberg.gcp.bigquery.BigQueryMetastoreCatalog.PROJECT_ID;
+import static org.apache.iceberg.gcp.bigquery.BigQueryMetastoreCatalog.TESTING_ENABLED;
+import static org.apache.iceberg.types.Types.NestedField.optional;
+import static org.assertj.core.api.Assertions.assertThat;
+import static org.assertj.core.api.Assertions.assertThatThrownBy;
+
+import java.io.File;
+import java.util.List;
+import java.util.Map;
+import org.apache.iceberg.CatalogProperties;
+import org.apache.iceberg.CatalogUtil;
+import org.apache.iceberg.PartitionSpec;
+import org.apache.iceberg.Schema;
+import org.apache.iceberg.SortOrder;
+import org.apache.iceberg.Table;
+import org.apache.iceberg.TableUtil;
+import org.apache.iceberg.Transaction;
+import org.apache.iceberg.catalog.CatalogTests;
+import org.apache.iceberg.catalog.Namespace;
+import org.apache.iceberg.catalog.TableIdentifier;
+import org.apache.iceberg.exceptions.NoSuchNamespaceException;
+import org.apache.iceberg.exceptions.NoSuchTableException;
+import org.apache.iceberg.expressions.Expressions;
+import org.apache.iceberg.relocated.com.google.common.collect.ImmutableMap;
+import org.apache.iceberg.types.Types;
+import org.junit.jupiter.api.AfterEach;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Disabled;
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.io.TempDir;
+import org.junit.jupiter.params.ParameterizedTest;
+import org.junit.jupiter.params.provider.ValueSource;
+
+public class TestBigQueryCatalog extends CatalogTests<BigQueryMetastoreCatalog> {
+
+  private static final String GCP_PROJECT = "project-id";
+  private static final String CATALOG_ID = "catalog-name";
+
+  // another schema that is not the same
+  protected static final Schema OTHER_SCHEMA_OPTIONAL =
+      new Schema(optional(1, "some_id", Types.IntegerType.get()));
+
+  // Schema passed to create tables
+  protected static final Schema SCHEMA_OPTIONAL =
+      new Schema(
+          optional(3, "id", Types.IntegerType.get(), "unique ID 🤪"),
+          optional(4, "data", Types.StringType.get()));
+
+  // This is the actual schema for the table, with column IDs reassigned
+  protected static final Schema REPLACE_SCHEMA_OPTIONAL =
+      new Schema(
+          optional(2, "id", Types.IntegerType.get(), "unique ID 🤪"),
+          optional(3, "data", Types.StringType.get()));
+
+  protected static final PartitionSpec REPLACE_SPEC_OPTIONAL =
+      PartitionSpec.builderFor(REPLACE_SCHEMA_OPTIONAL).bucket("id", 16).withSpecId(1).build();
+
+  protected static final SortOrder REPLACE_WRITE_ORDER_OPTIONAL =
+      SortOrder.builderFor(REPLACE_SCHEMA_OPTIONAL)
+          .asc(Expressions.bucket("id", 16))
+          .asc("id")
+          .build();
+
+  @TempDir private File tempFolder;
+  private BigQueryMetastoreCatalog catalog;
+  private String warehouseLocation;
+
+  @BeforeEach
+  public void before() throws Exception {
+    catalog = initCatalog(CATALOG_ID, ImmutableMap.of());
+  }
+
+  @AfterEach
+  public void after() throws Exception {
+    // Drop all tables in all datasets
+    List<Namespace> namespaces = catalog().listNamespaces(Namespace.empty());
+    for (Namespace namespace : namespaces) {
+      List<TableIdentifier> tables = catalog().listTables(namespace);
+      for (TableIdentifier table : tables) {
+        try {
+          catalog().dropTable(table, true); // drop with purge
+        } catch (NoSuchTableException e) {
+          // Table might be dropped by a previous iteration, ignore
+        }
+      }
+
+      // Drop all datasets (except any initial ones)
+      try {
+        catalog().dropNamespace(namespace);
+      } catch (NoSuchNamespaceException e) {
+        // Namespace might be dropped by a previous iteration, ignore
+      }
+    }
+  }
+
+  @Override
+  protected boolean requiresNamespaceCreate() {
+    return true;
+  }
+
+  @Override
+  protected boolean supportsNamesWithSlashes() {
+    return false;
+  }
+
+  @Override
+  protected boolean supportsNamesWithDot() {
+    return false;
+  }
+
+  @Override
+  public BigQueryMetastoreCatalog catalog() {
+    return catalog;
+  }
+
+  @Override
+  protected BigQueryMetastoreCatalog initCatalog(
+      String catalogName, Map<String, String> additionalProperties) {
+
+    warehouseLocation = tempFolder.toPath().resolve("hive-warehouse").toString();
+
+    Map<String, String> properties =
+        Map.of(
+            ICEBERG_CATALOG_TYPE,
+            ICEBERG_CATALOG_TYPE_BIGQUERY,
+            PROJECT_ID,
+            GCP_PROJECT,
+            CatalogProperties.WAREHOUSE_LOCATION,
+            warehouseLocation,
+            TESTING_ENABLED,
+            "true",
+            CatalogProperties.TABLE_DEFAULT_PREFIX + "default-key1",
+            "catalog-default-key1",
+            CatalogProperties.TABLE_DEFAULT_PREFIX + "default-key2",
+            "catalog-default-key2",
+            CatalogProperties.TABLE_DEFAULT_PREFIX + "override-key3",
+            "catalog-default-key3",
+            CatalogProperties.TABLE_OVERRIDE_PREFIX + "override-key3",
+            "catalog-override-key3",
+            CatalogProperties.TABLE_OVERRIDE_PREFIX + "override-key4",
+            "catalog-override-key4");
+
+    BigQueryMetastoreCatalog tmpCatalog =
+        (BigQueryMetastoreCatalog)
+            CatalogUtil.buildIcebergCatalog(
+                catalogName,
+                ImmutableMap.<String, String>builder()
+                    .putAll(properties)
+                    .putAll(additionalProperties)
+                    .build(),
+                null);
+
+    return tmpCatalog;
+  }
+
+  @Test
+  public void testRenameTable() {
+    if (requiresNamespaceCreate()) {
+      catalog.createNamespace(NS);
+    }
+
+    assertThat(catalog.tableExists(TABLE))
+        .as("Source table should not exist before create")
+        .isFalse();
+
+    catalog.buildTable(TABLE, SCHEMA).create();
+    assertThat(catalog.tableExists(TABLE)).as("Table should exist after create").isTrue();
+
+    assertThat(catalog.tableExists(RENAMED_TABLE))
+        .as("Destination table should not exist before rename")
+        .isFalse();
+
+    assertThatThrownBy(() -> catalog.renameTable(TABLE, RENAMED_TABLE))
+        .isInstanceOf(UnsupportedOperationException.class)
+        .hasMessageContaining("Table rename operation is unsupported");
+
+    catalog.dropTable(TABLE);
+  }
+
+  @Disabled(
+      "BigQuery does not allow adding a REQUIRED column to an existing table schema. "
+          + "When this operation is attempted, an exception gets thrown.")
+  @Test
+  public void testOverrideTablePropertiesReplaceTransaction() {}

Review Comment:
   Done



-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org

For queries about this service, please contact Infrastructure at:
users@infra.apache.org


---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For additional commands, e-mail: issues-help@iceberg.apache.org


Re: [PR] Catalog: Add BigQuery Metastore Catalog Support [iceberg]

Posted by "talatuyarer (via GitHub)" <gi...@apache.org>.
talatuyarer commented on code in PR #12808:
URL: https://github.com/apache/iceberg/pull/12808#discussion_r2059697375


##########
bigquery/src/main/java/org/apache/iceberg/gcp/bigquery/BigQueryMetastoreClient.java:
##########
@@ -0,0 +1,121 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.iceberg.gcp.bigquery;
+
+import com.google.api.services.bigquery.model.Dataset;
+import com.google.api.services.bigquery.model.DatasetList.Datasets;
+import com.google.api.services.bigquery.model.DatasetReference;
+import com.google.api.services.bigquery.model.Table;
+import com.google.api.services.bigquery.model.TableList.Tables;
+import com.google.api.services.bigquery.model.TableReference;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+
+/**
+ * A client of Google BigQuery Metastore functions over the BigQuery service. Uses the Google
+ * BigQuery API.
+ */
+public interface BigQueryMetastoreClient {
+
+  /**
+   * Creates and returns a new dataset.
+   *
+   * @param dataset the dataset to create
+   */
+  Dataset createDataset(Dataset dataset);
+
+  /**
+   * Returns a dataset.
+   *
+   * @param datasetReference full dataset reference
+   */
+  Dataset getDataset(DatasetReference datasetReference);
+
+  /**
+   * Deletes a dataset.
+   *
+   * @param datasetReference full dataset reference
+   */
+  void deleteDataset(DatasetReference datasetReference);
+
+  /**
+   * Updates parameters of a dataset or adds them if did not exist, leaving already-existing ones
+   * intact.
+   *
+   * @param datasetReference full dataset reference
+   * @param parameters metadata parameters to add
+   * @return dataset after patch
+   */
+  Dataset setDatasetParameters(DatasetReference datasetReference, Map<String, String> parameters);
+
+  /**
+   * Removes given set of keys of parameters of a dataset. Ignores keys that do not exist already.
+   *
+   * @param datasetReference full dataset reference
+   * @param parameters metadata parameter keys to remove
+   * @return dataset after patch
+   */
+  Dataset removeDatasetParameters(DatasetReference datasetReference, Set<String> parameters);
+
+  /**
+   * Lists datasets under a given project
+   *
+   * @param projectId the identifier of the project to list datasets under
+   */
+  List<Datasets> listDatasets(String projectId);
+
+  /**
+   * Creates and returns a new table.
+   *
+   * @param table body of the table to create
+   */
+  Table createTable(Table table);
+
+  /**
+   * Returns a table.
+   *
+   * @param tableReference full table reference
+   */
+  Table getTable(TableReference tableReference);
+
+  /**
+   * Updates the catalog table options of an Iceberg table and returns the updated table.
+   *
+   * @param tableReference full table reference
+   * @param table to patch
+   */
+  Table patchTable(TableReference tableReference, Table table);

Review Comment:
   Yes It is update I changed the name



-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org

For queries about this service, please contact Infrastructure at:
users@infra.apache.org


---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For additional commands, e-mail: issues-help@iceberg.apache.org


Re: [PR] Catalog: Add BigQuery Metastore Catalog Support [iceberg]

Posted by "nastra (via GitHub)" <gi...@apache.org>.
nastra commented on code in PR #12808:
URL: https://github.com/apache/iceberg/pull/12808#discussion_r2065983322


##########
bigquery/src/main/java/org/apache/iceberg/gcp/bigquery/BigQueryMetastoreCatalog.java:
##########
@@ -0,0 +1,394 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.iceberg.gcp.bigquery;
+
+import com.google.api.services.bigquery.model.Dataset;
+import com.google.api.services.bigquery.model.DatasetList.Datasets;
+import com.google.api.services.bigquery.model.DatasetReference;
+import com.google.api.services.bigquery.model.ExternalCatalogDatasetOptions;
+import com.google.api.services.bigquery.model.TableReference;
+import com.google.cloud.ServiceOptions;
+import com.google.cloud.bigquery.BigQueryOptions;
+import java.io.IOException;
+import java.security.GeneralSecurityException;
+import java.util.List;
+import java.util.Locale;
+import java.util.Map;
+import java.util.Objects;
+import java.util.Set;
+import org.apache.hadoop.conf.Configurable;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.iceberg.BaseMetastoreCatalog;
+import org.apache.iceberg.CatalogProperties;
+import org.apache.iceberg.CatalogUtil;
+import org.apache.iceberg.TableMetadata;
+import org.apache.iceberg.TableOperations;
+import org.apache.iceberg.catalog.Namespace;
+import org.apache.iceberg.catalog.SupportsNamespaces;
+import org.apache.iceberg.catalog.TableIdentifier;
+import org.apache.iceberg.exceptions.AlreadyExistsException;
+import org.apache.iceberg.exceptions.NoSuchNamespaceException;
+import org.apache.iceberg.exceptions.NoSuchTableException;
+import org.apache.iceberg.exceptions.ServiceFailureException;
+import org.apache.iceberg.exceptions.ValidationException;
+import org.apache.iceberg.io.FileIO;
+import org.apache.iceberg.relocated.com.google.common.annotations.VisibleForTesting;
+import org.apache.iceberg.relocated.com.google.common.base.Preconditions;
+import org.apache.iceberg.relocated.com.google.common.base.Strings;
+import org.apache.iceberg.relocated.com.google.common.collect.ImmutableList;
+import org.apache.iceberg.relocated.com.google.common.collect.ImmutableMap;
+import org.apache.iceberg.relocated.com.google.common.collect.Maps;
+import org.apache.iceberg.util.LocationUtil;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/** Iceberg Bigquery Metastore Catalog implementation. */
+public class BigQueryMetastoreCatalog extends BaseMetastoreCatalog
+    implements SupportsNamespaces, Configurable {
+
+  // User provided properties.
+  public static final String PROJECT_ID = "gcp-project";
+  public static final String GCP_LOCATION = "gcp-location";
+  public static final String LIST_ALL_TABLES = "list-all-tables";
+
+  public static final String HIVE_METASTORE_WAREHOUSE_DIR = "hive.metastore.warehouse.dir";
+
+  private static final Logger LOG = LoggerFactory.getLogger(BigQueryMetastoreCatalog.class);
+
+  private static final String DEFAULT_GCP_LOCATION = "us";
+
+  private String catalogName;
+  private Map<String, String> catalogProperties;
+  private FileIO fileIO;
+  private Configuration conf;
+  private String projectId;
+  private String location;
+  private BigQueryMetastoreClient client;
+  private boolean listAllTables;
+
+  // Must have a no-arg constructor to be dynamically loaded
+  // initialize(String name, Map<String, String> properties) will be called to complete
+  // initialization
+  public BigQueryMetastoreCatalog() {}
+
+  @Override
+  public void initialize(String inputName, Map<String, String> properties) {
+    Preconditions.checkArgument(
+        properties.containsKey(PROJECT_ID), "GCP project must be specified");
+
+    projectId = properties.get(PROJECT_ID);
+    location = properties.getOrDefault(GCP_LOCATION, DEFAULT_GCP_LOCATION);
+
+    BigQueryOptions options =
+        BigQueryOptions.newBuilder()
+            .setProjectId(projectId)
+            .setLocation(location)
+            .setRetrySettings(ServiceOptions.getDefaultRetrySettings())
+            .build();
+
+    try {
+      client = new BigQueryMetastoreClientImpl(options);
+    } catch (IOException e) {
+      throw new ServiceFailureException(e, "Creating BigQuery client failed");
+    } catch (GeneralSecurityException e) {
+      throw new ValidationException(e, "Creating BigQuery client failed due to a security issue");
+    }
+
+    initialize(inputName, properties, projectId, location, client);
+  }
+
+  @VisibleForTesting
+  void initialize(
+      String inputName,
+      Map<String, String> properties,
+      String initialProjectId,
+      String initialLocation,
+      BigQueryMetastoreClient bigQueryMetaStoreClient) {
+    Preconditions.checkArgument(bigQueryMetaStoreClient != null, "BigQuery client can not be null");
+    this.catalogName = inputName;
+    this.catalogProperties = ImmutableMap.copyOf(properties);
+    this.projectId = initialProjectId;
+    this.location = initialLocation;
+    this.client = bigQueryMetaStoreClient;
+
+    if (this.conf == null) {
+      LOG.warn("No configuration was set, using the default environment Configuration");
+      this.conf = new Configuration();
+    }
+
+    LOG.info("Using BigQuery Metastore Iceberg Catalog: {}", inputName);
+
+    if (properties.containsKey(CatalogProperties.WAREHOUSE_LOCATION)) {
+      this.conf.set(
+          HIVE_METASTORE_WAREHOUSE_DIR,
+          LocationUtil.stripTrailingSlash(properties.get(CatalogProperties.WAREHOUSE_LOCATION)));
+    }
+
+    String fileIoImpl =
+        properties.getOrDefault(
+            CatalogProperties.FILE_IO_IMPL, "org.apache.iceberg.hadoop.HadoopFileIO");
+    this.fileIO = CatalogUtil.loadFileIO(fileIoImpl, properties, conf);
+
+    this.listAllTables = Boolean.parseBoolean(properties.getOrDefault(LIST_ALL_TABLES, "true"));
+  }
+
+  @Override
+  protected TableOperations newTableOps(TableIdentifier identifier) {
+    return new BigQueryTableOperations(client, fileIO, toTableReference(identifier), conf);
+  }
+
+  @Override
+  protected String defaultWarehouseLocation(TableIdentifier identifier) {
+    String locationUri = null;
+    DatasetReference datasetReference = toDatasetReference(identifier.namespace());
+    Dataset dataset = client.load(datasetReference);
+    if (dataset != null && dataset.getExternalCatalogDatasetOptions() != null) {
+      locationUri = dataset.getExternalCatalogDatasetOptions().getDefaultStorageLocationUri();
+    }
+
+    return String.format(
+        "%s/%s",
+        Strings.isNullOrEmpty(locationUri)
+            ? createDefaultStorageLocationUri(datasetReference.getDatasetId())
+            : LocationUtil.stripTrailingSlash(locationUri),
+        identifier.name());
+  }
+
+  @Override
+  public List<TableIdentifier> listTables(Namespace namespace) {
+    validateNamespace(namespace);
+
+    return client.list(toDatasetReference(namespace), listAllTables).stream()
+        .map(
+            table -> TableIdentifier.of(namespace.level(0), table.getTableReference().getTableId()))
+        .collect(ImmutableList.toImmutableList());
+  }
+
+  @Override
+  public boolean dropTable(TableIdentifier identifier, boolean purge) {
+    try {
+      TableOperations ops = newTableOps(identifier);
+      TableMetadata lastMetadata = ops.current();
+
+      client.delete(toTableReference(identifier));
+
+      if (purge && lastMetadata != null) {
+        CatalogUtil.dropTableData(ops.io(), lastMetadata);
+      }
+    } catch (NoSuchTableException e) {
+      return false;
+    }
+
+    return true;
+  }
+
+  @Override
+  public void renameTable(TableIdentifier from, TableIdentifier to) {
+    validateNamespace(from.namespace());
+    validateNamespace(to.namespace());
+
+    if (!namespaceExists(to.namespace())) {
+      throw new NoSuchNamespaceException("Namespace does not exist: %s", to.namespace());
+    }
+
+    if (!namespaceExists(from.namespace())) {
+      throw new NoSuchNamespaceException("Namespace does not exist: %s", from.namespace());
+    }
+
+    // Check if the source table exists
+    if (!tableExists(from)) {
+      throw new NoSuchTableException("Table does not exist: %s", from.name());
+    }
+
+    if (!from.namespace().equals(to.namespace())) {
+      throw new ValidationException("New table name must be in the same namespace");
+    }
+
+    // Check if the destination table already exists
+    if (tableExists(to)) {
+      throw new AlreadyExistsException("Table already exists: %s", to.name());
+    }
+
+    // TODO: Enable once supported by BigQuery API.
+    throw new UnsupportedOperationException("Table rename operation is unsupported.");
+  }
+
+  @Override
+  public void createNamespace(Namespace namespace, Map<String, String> metadata) {
+    Dataset builder = new Dataset();
+    DatasetReference datasetReference = toDatasetReference(namespace);
+    builder.setLocation(this.location);
+    builder.setDatasetReference(datasetReference);
+    builder.setExternalCatalogDatasetOptions(
+        BigQueryMetastoreUtils.createExternalCatalogDatasetOptions(
+            createDefaultStorageLocationUri(datasetReference.getDatasetId()), metadata));
+
+    client.create(builder);
+  }
+
+  @Override
+  public List<Namespace> listNamespaces() {
+    try {
+      return listNamespaces(Namespace.empty());
+    } catch (NoSuchNamespaceException e) {
+      return ImmutableList.of();
+    }
+  }
+
+  /**
+   * Since this catalog only supports one-level namespaces, it always returns an empty list unless
+   * passed an empty namespace to list all namespaces within the catalog.
+   */
+  @Override
+  public List<Namespace> listNamespaces(Namespace namespace) {
+    List<Datasets> allDatasets = client.list(projectId);
+
+    ImmutableList<Namespace> namespaces =
+        allDatasets.stream()
+            .map(BigQueryMetastoreCatalog::toNamespace)
+            .collect(ImmutableList.toImmutableList());
+    if (namespaces.isEmpty()) {
+      throw new NoSuchNamespaceException("Namespace does not exist: %s", namespace);
+    }
+
+    return namespaces;
+  }
+
+  @Override
+  public boolean dropNamespace(Namespace namespace) {
+    try {
+      client.delete(toDatasetReference(namespace));
+      // We don't delete the data folder for safety, which aligns with Hive Metastore's default
+      // behavior.
+      // We can support database or catalog level config controlling file deletion in the future.
+      return true;
+    } catch (NoSuchNamespaceException e) {
+      return false;
+    }
+  }
+
+  @Override
+  public boolean setProperties(Namespace namespace, Map<String, String> properties) {
+    Dataset dataset = client.load(toDatasetReference(namespace));
+
+    ExternalCatalogDatasetOptions existingOptions = dataset.getExternalCatalogDatasetOptions();
+    Map<String, String> existingParameters =
+        existingOptions != null ? existingOptions.getParameters() : null;
+
+    Map<String, String> newParameters = Maps.newHashMap();
+    if (existingParameters != null) {
+      newParameters.putAll(existingParameters);
+    }
+
+    newParameters.putAll(properties);
+
+    if (Objects.equals(existingParameters, newParameters)) {
+      // No change in parameters detected
+      return false;
+    }
+
+    client.setParameters(toDatasetReference(namespace), properties);
+    return true;
+  }
+
+  @Override
+  public boolean removeProperties(Namespace namespace, Set<String> properties) {
+    client.removeParameters(toDatasetReference(namespace), properties);
+    return true;
+  }
+
+  @SuppressWarnings("FormatStringAnnotation")
+  @Override
+  public Map<String, String> loadNamespaceMetadata(Namespace namespace) {
+    try {
+      return toMetadata(client.load(toDatasetReference(namespace)));
+    } catch (IllegalArgumentException e) {
+      throw new NoSuchNamespaceException(e.getMessage());
+    }
+  }
+
+  @Override
+  public String name() {
+    return catalogName;
+  }
+
+  @Override
+  protected Map<String, String> properties() {
+    return catalogProperties == null ? ImmutableMap.of() : catalogProperties;
+  }
+
+  @Override
+  public void setConf(Configuration conf) {
+    this.conf = new Configuration(conf);
+  }
+
+  @Override
+  public Configuration getConf() {
+    return this.conf;
+  }
+
+  private String createDefaultStorageLocationUri(String dbId) {
+    String warehouseLocation = conf.get(HIVE_METASTORE_WAREHOUSE_DIR);
+    Preconditions.checkNotNull(warehouseLocation, "Data warehouse location is not set");
+    return String.format("%s/%s.db", LocationUtil.stripTrailingSlash(warehouseLocation), dbId);
+  }
+
+  private static Namespace toNamespace(Datasets dataset) {

Review Comment:
   it should work with this diff:
   
   ```
   --- a/bigquery/src/main/java/org/apache/iceberg/gcp/bigquery/BigQueryMetastoreCatalog.java
   +++ b/bigquery/src/main/java/org/apache/iceberg/gcp/bigquery/BigQueryMetastoreCatalog.java
   @@ -269,7 +269,7 @@ public class BigQueryMetastoreCatalog extends BaseMetastoreCatalog
   
        ImmutableList<Namespace> namespaces =
            allDatasets.stream()
   -            .map(BigQueryMetastoreCatalog::toNamespace)
   +            .map(this::toNamespace)
                .collect(ImmutableList.toImmutableList());
        if (namespaces.isEmpty()) {
          throw new NoSuchNamespaceException("Namespace does not exist: %s", namespace);
   @@ -359,7 +359,7 @@ public class BigQueryMetastoreCatalog extends BaseMetastoreCatalog
        return String.format("%s/%s.db", LocationUtil.stripTrailingSlash(warehouseLocation), dbId);
      }
   
   -  private static Namespace toNamespace(Datasets dataset) {
   +  private Namespace toNamespace(Datasets dataset) {
        return Namespace.of(dataset.getDatasetReference().getDatasetId());
      }
   
   ```



-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org

For queries about this service, please contact Infrastructure at:
users@infra.apache.org


---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For additional commands, e-mail: issues-help@iceberg.apache.org


Re: [PR] Catalog: Add BigQuery Metastore Catalog Support [iceberg]

Posted by "talatuyarer (via GitHub)" <gi...@apache.org>.
talatuyarer commented on code in PR #12808:
URL: https://github.com/apache/iceberg/pull/12808#discussion_r2067701790


##########
bigquery/src/test/java/org/apache/iceberg/gcp/bigquery/BigQueryTableOperationsTest.java:
##########
@@ -0,0 +1,274 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.iceberg.gcp.bigquery;
+
+import static org.apache.iceberg.gcp.bigquery.BigQueryMetastoreCatalog.PROJECT_ID;
+import static org.apache.iceberg.types.Types.NestedField.required;
+import static org.assertj.core.api.Assertions.assertThat;
+import static org.assertj.core.api.Assertions.assertThatThrownBy;
+import static org.mockito.Mockito.any;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.reset;
+import static org.mockito.Mockito.times;
+import static org.mockito.Mockito.verify;
+import static org.mockito.Mockito.when;
+
+import com.google.api.services.bigquery.model.Dataset;
+import com.google.api.services.bigquery.model.DatasetReference;
+import com.google.api.services.bigquery.model.ExternalCatalogDatasetOptions;
+import com.google.api.services.bigquery.model.ExternalCatalogTableOptions;
+import com.google.api.services.bigquery.model.StorageDescriptor;
+import com.google.api.services.bigquery.model.Table;
+import com.google.api.services.bigquery.model.TableReference;
+import java.io.File;
+import java.io.IOException;
+import java.util.Collections;
+import java.util.Optional;
+import org.apache.commons.io.FileUtils;
+import org.apache.commons.io.filefilter.TrueFileFilter;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.iceberg.CatalogProperties;
+import org.apache.iceberg.PartitionSpec;
+import org.apache.iceberg.Schema;
+import org.apache.iceberg.catalog.TableIdentifier;
+import org.apache.iceberg.exceptions.CommitFailedException;
+import org.apache.iceberg.exceptions.NoSuchTableException;
+import org.apache.iceberg.exceptions.ValidationException;
+import org.apache.iceberg.relocated.com.google.common.collect.ImmutableMap;
+import org.apache.iceberg.types.Types;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.io.TempDir;
+import org.mockito.ArgumentCaptor;
+
+public class BigQueryTableOperationsTest {
+
+  @TempDir private File tempFolder;
+  public static final String METADATA_LOCATION_PROP = "metadata_location";
+  private static final String GCP_PROJECT = "my-project";
+  private static final String GCP_REGION = "us";
+  private static final String DATASET_ID = "db";
+  private static final String TABLE_ID = "tbl";
+  private static final TableIdentifier SPARK_TABLE_ID = TableIdentifier.of(DATASET_ID, TABLE_ID);
+
+  private static final TableReference TABLE_REFERENCE =
+      new TableReference().setProjectId(GCP_PROJECT).setDatasetId(DATASET_ID).setTableId(TABLE_ID);
+
+  private final BigQueryMetastoreClient bigQueryMetaStoreClient =
+      mock(BigQueryMetastoreClient.class);
+
+  private BigQueryMetastoreCatalog bigQueryMetastoreCatalog;
+  private BigQueryTableOperations tableOps;
+
+  @BeforeEach
+  public void before() {
+    this.bigQueryMetastoreCatalog = new BigQueryMetastoreCatalog();
+    this.bigQueryMetastoreCatalog.setConf(new Configuration());
+    String warehouseLocation = tempFolder.toPath().resolve("hive-warehouse").toString();
+
+    bigQueryMetastoreCatalog.initialize(
+        "CATALOG_ID",
+        ImmutableMap.of(
+            PROJECT_ID,
+            GCP_PROJECT,
+            CatalogProperties.WAREHOUSE_LOCATION,
+            warehouseLocation,
+            CatalogProperties.FILE_IO_IMPL,
+            "org.apache.iceberg.hadoop.HadoopFileIO"),
+        GCP_PROJECT,
+        GCP_REGION,
+        bigQueryMetaStoreClient);
+    this.tableOps = (BigQueryTableOperations) bigQueryMetastoreCatalog.newTableOps(SPARK_TABLE_ID);
+  }
+
+  @Test
+  public void testFetchLatestMetadataFromBigQuery() throws Exception {
+    Table createdTable = createTestTable();
+    reset(bigQueryMetaStoreClient);
+    when(bigQueryMetaStoreClient.load(TABLE_REFERENCE)).thenReturn(createdTable);
+
+    tableOps.refresh();
+    assertThat(
+            createdTable
+                .getExternalCatalogTableOptions()
+                .getParameters()
+                .getOrDefault(METADATA_LOCATION_PROP, ""))
+        .isEqualTo(tableOps.currentMetadataLocation());
+
+    reset(bigQueryMetaStoreClient);
+    when(bigQueryMetaStoreClient.load(TABLE_REFERENCE))
+        .thenThrow(new NoSuchTableException("error message getTable"));
+    // Refresh fails when table is not found but metadata already presents.
+    assertThatThrownBy(() -> tableOps.refresh())
+        .isInstanceOf(NoSuchTableException.class)
+        .hasMessageContaining("error message getTable");
+  }
+
+  @Test
+  public void testFailForNonIcebergTable() {
+    when(bigQueryMetaStoreClient.load(TABLE_REFERENCE))
+        .thenReturn(new Table().setTableReference(TABLE_REFERENCE));
+
+    assertThatThrownBy(() -> tableOps.refresh())
+        .isInstanceOf(ValidationException.class)
+        .hasMessageContaining("metadata location not found");
+  }
+
+  @Test
+  public void testNoOpWhenMetadataAndTableNotFound() {
+    when(bigQueryMetaStoreClient.load(TABLE_REFERENCE))
+        .thenThrow(new NoSuchTableException("error message getTable"));
+    // Table not found won't cause errors when the metadata is null.
+    assertThat(tableOps.currentMetadataLocation()).isNull();
+    tableOps.refresh();
+  }
+
+  @Test
+  public void testTableNameAsExpected() {
+    assertThat(tableOps.tableName()).isEqualTo("db.tbl");
+  }
+
+  @Test
+  public void testUseEtagForUpdateTable() throws Exception {
+    Table tableWithEtag = createTestTable().setEtag("etag");
+    reset(bigQueryMetaStoreClient);
+    when(bigQueryMetaStoreClient.load(TABLE_REFERENCE)).thenReturn(tableWithEtag, tableWithEtag);
+
+    org.apache.iceberg.Table loadedTable = bigQueryMetastoreCatalog.loadTable(SPARK_TABLE_ID);
+
+    when(bigQueryMetaStoreClient.update(any(), any())).thenReturn(tableWithEtag);
+    loadedTable.updateSchema().addColumn("n", Types.IntegerType.get()).commit();
+
+    ArgumentCaptor<TableReference> tableReferenceArgumentCaptor =
+        ArgumentCaptor.forClass(TableReference.class);
+    ArgumentCaptor<Table> tableArgumentCaptor = ArgumentCaptor.forClass(Table.class);
+    verify(bigQueryMetaStoreClient, times(1))
+        .update(tableReferenceArgumentCaptor.capture(), tableArgumentCaptor.capture());
+    assertThat(tableReferenceArgumentCaptor.getValue()).isEqualTo(TABLE_REFERENCE);
+    assertThat(tableArgumentCaptor.getValue().getEtag()).isEqualTo("etag");
+  }
+
+  @Test
+  public void testFailWhenEtagMismatch() throws Exception {
+    Table tableWithEtag = createTestTable().setEtag("etag");
+    reset(bigQueryMetaStoreClient);
+    when(bigQueryMetaStoreClient.load(TABLE_REFERENCE)).thenReturn(tableWithEtag, tableWithEtag);
+
+    org.apache.iceberg.Table loadedTable = bigQueryMetastoreCatalog.loadTable(SPARK_TABLE_ID);
+
+    when(bigQueryMetaStoreClient.update(any(), any()))
+        .thenThrow(new ValidationException("error message etag mismatch"));
+    assertThatThrownBy(
+            () -> loadedTable.updateSchema().addColumn("n", Types.IntegerType.get()).commit())
+        .isInstanceOf(CommitFailedException.class)
+        .hasMessageContaining(
+            "Updating table failed due to conflict updates (etag mismatch). Retry the update");
+  }
+
+  @Test
+  public void testFailWhenMetadataLocationDiff() throws Exception {
+    Table tableWithEtag = createTestTable().setEtag("etag");
+    Table tableWithNewMetadata =
+        new Table()
+            .setEtag("etag")
+            .setExternalCatalogTableOptions(
+                new ExternalCatalogTableOptions()
+                    .setParameters(ImmutableMap.of(METADATA_LOCATION_PROP, "a/new/location")));
+
+    reset(bigQueryMetaStoreClient);
+    // Two invocations, for loadTable and commit.
+    when(bigQueryMetaStoreClient.load(TABLE_REFERENCE))
+        .thenReturn(tableWithEtag, tableWithNewMetadata);
+
+    org.apache.iceberg.Table loadedTable = bigQueryMetastoreCatalog.loadTable(SPARK_TABLE_ID);
+
+    when(bigQueryMetaStoreClient.update(any(), any())).thenReturn(tableWithEtag);
+    assertThatThrownBy(
+            () -> loadedTable.updateSchema().addColumn("n", Types.IntegerType.get()).commit())
+        .isInstanceOf(CommitFailedException.class)
+        .hasMessageContaining("is not same as the current table metadata location");
+  }
+
+  @Test
+  public void testCreateTableCommitSucceeds() throws Exception {
+    var testTable = createTestTable();
+    when(bigQueryMetaStoreClient.create(any(Table.class))).thenReturn(testTable);
+    when(bigQueryMetaStoreClient.load(
+            new DatasetReference().setProjectId(GCP_PROJECT).setDatasetId(DATASET_ID)))
+        .thenReturn(
+            new Dataset()
+                .setExternalCatalogDatasetOptions(
+                    new ExternalCatalogDatasetOptions()
+                        .setDefaultStorageLocationUri("build/db_folder")));
+
+    Schema schema = testSchema();
+    bigQueryMetastoreCatalog.createTable(SPARK_TABLE_ID, schema, PartitionSpec.unpartitioned());

Review Comment:
   I added assertion part too. 



-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org

For queries about this service, please contact Infrastructure at:
users@infra.apache.org


---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For additional commands, e-mail: issues-help@iceberg.apache.org


Re: [PR] Catalog: Add BigQuery Metastore Catalog Support [iceberg]

Posted by "nastra (via GitHub)" <gi...@apache.org>.
nastra commented on code in PR #12808:
URL: https://github.com/apache/iceberg/pull/12808#discussion_r2067912871


##########
bigquery/src/main/java/org/apache/iceberg/gcp/bigquery/BigQueryMetastoreClientImpl.java:
##########
@@ -0,0 +1,594 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.iceberg.gcp.bigquery;
+
+import com.google.api.client.googleapis.javanet.GoogleNetHttpTransport;
+import com.google.api.client.http.HttpHeaders;
+import com.google.api.client.http.HttpResponse;
+import com.google.api.client.http.HttpResponseException;
+import com.google.api.client.http.HttpStatusCodes;
+import com.google.api.client.json.gson.GsonFactory;
+import com.google.api.client.util.Data;
+import com.google.api.services.bigquery.Bigquery;
+import com.google.api.services.bigquery.BigqueryScopes;
+import com.google.api.services.bigquery.model.Dataset;
+import com.google.api.services.bigquery.model.DatasetList;
+import com.google.api.services.bigquery.model.DatasetList.Datasets;
+import com.google.api.services.bigquery.model.DatasetReference;
+import com.google.api.services.bigquery.model.ExternalCatalogDatasetOptions;
+import com.google.api.services.bigquery.model.ExternalCatalogTableOptions;
+import com.google.api.services.bigquery.model.Table;
+import com.google.api.services.bigquery.model.TableList;
+import com.google.api.services.bigquery.model.TableList.Tables;
+import com.google.api.services.bigquery.model.TableReference;
+import com.google.api.services.bigquery.model.TableSchema;
+import com.google.auth.http.HttpCredentialsAdapter;
+import com.google.auth.oauth2.GoogleCredentials;
+import com.google.cloud.BaseServiceException;
+import com.google.cloud.ExceptionHandler;
+import com.google.cloud.bigquery.BigQueryErrorMessages;
+import com.google.cloud.bigquery.BigQueryOptions;
+import com.google.cloud.bigquery.BigQueryRetryConfig;
+import com.google.cloud.bigquery.BigQueryRetryHelper;
+import java.io.IOException;
+import java.nio.charset.StandardCharsets;
+import java.security.GeneralSecurityException;
+import java.util.List;
+import java.util.Locale;
+import java.util.Map;
+import java.util.Set;
+import java.util.stream.Collectors;
+import java.util.stream.Stream;
+import org.apache.iceberg.BaseMetastoreTableOperations;
+import org.apache.iceberg.exceptions.AlreadyExistsException;
+import org.apache.iceberg.exceptions.BadRequestException;
+import org.apache.iceberg.exceptions.ForbiddenException;
+import org.apache.iceberg.exceptions.NoSuchIcebergTableException;
+import org.apache.iceberg.exceptions.NoSuchNamespaceException;
+import org.apache.iceberg.exceptions.NoSuchTableException;
+import org.apache.iceberg.exceptions.NotAuthorizedException;
+import org.apache.iceberg.exceptions.RuntimeIOException;
+import org.apache.iceberg.exceptions.ServiceFailureException;
+import org.apache.iceberg.exceptions.ServiceUnavailableException;
+import org.apache.iceberg.exceptions.ValidationException;
+import org.apache.iceberg.relocated.com.google.common.base.Preconditions;
+import org.apache.iceberg.relocated.com.google.common.collect.Lists;
+import org.apache.iceberg.relocated.com.google.common.collect.Maps;
+
+/** A client of Google Bigquery Metastore functions over the BigQuery service. */
+public final class BigQueryMetastoreClientImpl implements BigQueryMetastoreClient {
+
+  public static final ExceptionHandler.Interceptor EXCEPTION_HANDLER_INTERCEPTOR =
+      new ExceptionHandler.Interceptor() {
+
+        private static final long serialVersionUID = -8429573486870467828L;
+
+        @Override
+        public RetryResult afterEval(Exception exception, RetryResult retryResult) {
+          return ExceptionHandler.Interceptor.RetryResult.CONTINUE_EVALUATION;
+        }
+
+        @Override
+        public RetryResult beforeEval(Exception exception) {
+          if (exception instanceof BaseServiceException) {
+            boolean retriable = ((BaseServiceException) exception).isRetryable();
+            return retriable
+                ? ExceptionHandler.Interceptor.RetryResult.RETRY
+                : ExceptionHandler.Interceptor.RetryResult.CONTINUE_EVALUATION;
+          }
+
+          return ExceptionHandler.Interceptor.RetryResult.CONTINUE_EVALUATION;
+        }
+      };
+
+  private final Bigquery client;
+  private final BigQueryOptions bigqueryOptions;
+
+  // Retry config with error messages and regex for rate limit exceeded errors.
+  private static final BigQueryRetryConfig DEFAULT_RETRY_CONFIG =
+      BigQueryRetryConfig.newBuilder()
+          .retryOnMessage(BigQueryErrorMessages.RATE_LIMIT_EXCEEDED_MSG)
+          .retryOnMessage(BigQueryErrorMessages.JOB_RATE_LIMIT_EXCEEDED_MSG)
+          .retryOnRegEx(BigQueryErrorMessages.RetryRegExPatterns.RATE_LIMIT_EXCEEDED_REGEX)
+          .build();
+
+  public static final ExceptionHandler BIGQUERY_EXCEPTION_HANDLER =
+      ExceptionHandler.newBuilder()
+          .abortOn(RuntimeException.class)
+          .retryOn(
+              java.net.ConnectException
+                  .class) // Retry on connection failures due to transient network issues.
+          .retryOn(
+              java.net.UnknownHostException
+                  .class) // Retry to recover from temporary DNS resolution failures.
+          .addInterceptors(EXCEPTION_HANDLER_INTERCEPTOR)
+          .build();
+
+  /** Constructs a client of the Google BigQuery service. */
+  public BigQueryMetastoreClientImpl(BigQueryOptions options)
+      throws IOException, GeneralSecurityException {
+    // Initialize client that will be used to send requests. This client only needs to be created
+    // once, and can be reused for multiple requests
+    HttpCredentialsAdapter httpCredentialsAdapter =
+        new HttpCredentialsAdapter(
+            GoogleCredentials.getApplicationDefault().createScoped(BigqueryScopes.all()));
+    this.client =
+        new Bigquery.Builder(
+                GoogleNetHttpTransport.newTrustedTransport(),
+                GsonFactory.getDefaultInstance(),
+                httpRequest -> {
+                  httpCredentialsAdapter.initialize(httpRequest);
+                  // Instead of throwing exceptions, analyze the HttpResponse object and inspect its
+                  // status code. This will allow BigQuery API errors to be converted into Iceberg
+                  // exceptions.
+                  httpRequest.setThrowExceptionOnExecuteError(false);
+                })
+            .setApplicationName("BigQuery Metastore Iceberg Catalog Plugin")
+            .build();
+    this.bigqueryOptions = options;
+  }
+
+  @Override
+  public Dataset create(Dataset dataset) {
+    Dataset response = null;
+    try {
+      response =
+          BigQueryRetryHelper.runWithRetries(
+              () -> internalCreate(dataset),
+              bigqueryOptions.getRetrySettings(),
+              BIGQUERY_EXCEPTION_HANDLER,
+              bigqueryOptions.getClock(),
+              DEFAULT_RETRY_CONFIG);
+    } catch (BigQueryRetryHelper.BigQueryRetryHelperException e) {
+      handleBigQueryRetryException(e);
+    }
+    return response;
+  }
+
+  @SuppressWarnings("FormatStringAnnotation")
+  private Dataset internalCreate(Dataset dataset) {
+    try {
+      HttpResponse response =
+          client
+              .datasets()
+              .insert(dataset.getDatasetReference().getProjectId(), dataset)
+              .executeUnparsed();
+      return convertExceptionIfUnsuccessful(response).parseAs(Dataset.class);
+    } catch (IOException e) {
+      throw new RuntimeIOException(e);
+    } catch (AlreadyExistsException e) {
+      throw new AlreadyExistsException("Namespace already exists: " + dataset.getId());
+    }
+  }
+
+  @Override
+  @SuppressWarnings("FormatStringAnnotation")
+  public Dataset load(DatasetReference datasetReference) {
+    try {
+      HttpResponse response =
+          client
+              .datasets()
+              .get(datasetReference.getProjectId(), datasetReference.getDatasetId())
+              .executeUnparsed();
+      if (response.getStatusCode() == HttpStatusCodes.STATUS_CODE_NOT_FOUND) {
+        throw new NoSuchNamespaceException(
+            "Namespace does not exist: " + datasetReference.getDatasetId());
+      }
+
+      return convertExceptionIfUnsuccessful(response).parseAs(Dataset.class);
+    } catch (IOException e) {
+      throw new RuntimeIOException(e);
+    }
+  }
+
+  @Override
+  @SuppressWarnings("FormatStringAnnotation")
+  public void delete(DatasetReference datasetReference) {
+    try {
+      HttpResponse response =
+          client
+              .datasets()
+              .delete(datasetReference.getProjectId(), datasetReference.getDatasetId())
+              .executeUnparsed();
+      if (response.getStatusCode() == HttpStatusCodes.STATUS_CODE_NOT_FOUND) {
+        throw new NoSuchNamespaceException(
+            "Namespace does not exist: " + datasetReference.getDatasetId());
+      }
+
+      convertExceptionIfUnsuccessful(response);
+    } catch (IOException e) {
+      throw new RuntimeIOException(e);
+    }
+  }
+
+  @Override
+  public Dataset setParameters(DatasetReference datasetReference, Map<String, String> parameters) {
+    Dataset dataset = load(datasetReference);
+    ExternalCatalogDatasetOptions externalCatalogDatasetOptions =
+        dataset.getExternalCatalogDatasetOptions() == null
+            ? new ExternalCatalogDatasetOptions()
+            : dataset.getExternalCatalogDatasetOptions();
+    Map<String, String> finalParameters =
+        externalCatalogDatasetOptions.getParameters() == null
+            ? Maps.newHashMap()
+            : externalCatalogDatasetOptions.getParameters();
+    finalParameters.putAll(parameters);
+
+    dataset.setExternalCatalogDatasetOptions(
+        externalCatalogDatasetOptions.setParameters(finalParameters));
+
+    return internalUpdate(dataset);
+  }
+
+  @Override
+  public Dataset removeParameters(DatasetReference datasetReference, Set<String> parameters) {
+    Dataset dataset = load(datasetReference);
+    ExternalCatalogDatasetOptions externalCatalogDatasetOptions =
+        dataset.getExternalCatalogDatasetOptions() == null
+            ? new ExternalCatalogDatasetOptions()
+            : dataset.getExternalCatalogDatasetOptions();
+    Map<String, String> finalParameters =
+        externalCatalogDatasetOptions.getParameters() == null
+            ? Maps.newHashMap()
+            : externalCatalogDatasetOptions.getParameters();
+    parameters.forEach(finalParameters::remove);
+
+    dataset.setExternalCatalogDatasetOptions(
+        externalCatalogDatasetOptions.setParameters(finalParameters));
+
+    return internalUpdate(dataset);
+  }
+
+  @Override
+  public List<Datasets> list(String projectId) {
+    try {
+      String nextPageToken = null;
+      List<Datasets> datasets = Lists.newArrayList();
+      do {
+        HttpResponse pageResponse =
+            client.datasets().list(projectId).setPageToken(nextPageToken).executeUnparsed();
+        DatasetList result =
+            convertExceptionIfUnsuccessful(pageResponse).parseAs(DatasetList.class);
+        nextPageToken = result.getNextPageToken();
+        if (result.getDatasets() != null) {
+          datasets.addAll(result.getDatasets());
+        }
+
+      } while (nextPageToken != null && !nextPageToken.isEmpty());
+      return datasets;
+    } catch (IOException e) {
+      throw new RuntimeIOException(e);
+    }
+  }
+
+  @Override
+  public Table create(Table table) {
+    // Ensure it is an Iceberg table supported by the BigQuery metastore catalog.
+    validateTable(table);
+    // TODO: Ensure table creation is idempotent when handling retries.
+    Table response = null;
+    try {
+      response =
+          BigQueryRetryHelper.runWithRetries(
+              () -> internalCreate(table),
+              bigqueryOptions.getRetrySettings(),
+              BIGQUERY_EXCEPTION_HANDLER,
+              bigqueryOptions.getClock(),
+              DEFAULT_RETRY_CONFIG);
+    } catch (BigQueryRetryHelper.BigQueryRetryHelperException e) {
+      handleBigQueryRetryException(e);
+    }
+
+    return response;
+  }
+
+  @SuppressWarnings("FormatStringAnnotation")

Review Comment:
   this can be removed



-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org

For queries about this service, please contact Infrastructure at:
users@infra.apache.org


---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For additional commands, e-mail: issues-help@iceberg.apache.org


Re: [PR] Catalog: Add BigQuery Metastore Catalog Support [iceberg]

Posted by "danielcweeks (via GitHub)" <gi...@apache.org>.
danielcweeks commented on code in PR #12808:
URL: https://github.com/apache/iceberg/pull/12808#discussion_r2072085325


##########
bigquery/src/main/java/org/apache/iceberg/gcp/bigquery/BigQueryMetastoreUtils.java:
##########
@@ -0,0 +1,95 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.iceberg.gcp.bigquery;
+
+import com.google.api.services.bigquery.model.ExternalCatalogDatasetOptions;
+import com.google.api.services.bigquery.model.ExternalCatalogTableOptions;
+import com.google.api.services.bigquery.model.SerDeInfo;
+import com.google.api.services.bigquery.model.StorageDescriptor;
+import java.util.Map;
+
+/** Shared utilities for BigQuery Metastore specific functions and constants. */
+public final class BigQueryMetastoreUtils {

Review Comment:
   I don't think this class needs to be public



-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org

For queries about this service, please contact Infrastructure at:
users@infra.apache.org


---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For additional commands, e-mail: issues-help@iceberg.apache.org


Re: [PR] Catalog: Add BigQuery Metastore Catalog Support [iceberg]

Posted by "danielcweeks (via GitHub)" <gi...@apache.org>.
danielcweeks commented on code in PR #12808:
URL: https://github.com/apache/iceberg/pull/12808#discussion_r2072148291


##########
bigquery/src/main/java/org/apache/iceberg/gcp/bigquery/BigQueryMetastoreCatalog.java:
##########
@@ -0,0 +1,366 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.iceberg.gcp.bigquery;
+
+import com.google.api.services.bigquery.model.Dataset;
+import com.google.api.services.bigquery.model.DatasetList.Datasets;
+import com.google.api.services.bigquery.model.DatasetReference;
+import com.google.api.services.bigquery.model.ExternalCatalogDatasetOptions;
+import com.google.api.services.bigquery.model.TableReference;
+import com.google.cloud.ServiceOptions;
+import com.google.cloud.bigquery.BigQueryOptions;
+import java.io.IOException;
+import java.io.UncheckedIOException;
+import java.security.GeneralSecurityException;
+import java.util.List;
+import java.util.Locale;
+import java.util.Map;
+import java.util.Objects;
+import java.util.Set;
+import org.apache.hadoop.conf.Configurable;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.iceberg.BaseMetastoreCatalog;
+import org.apache.iceberg.CatalogProperties;
+import org.apache.iceberg.CatalogUtil;
+import org.apache.iceberg.TableMetadata;
+import org.apache.iceberg.TableOperations;
+import org.apache.iceberg.catalog.Namespace;
+import org.apache.iceberg.catalog.SupportsNamespaces;
+import org.apache.iceberg.catalog.TableIdentifier;
+import org.apache.iceberg.exceptions.NoSuchNamespaceException;
+import org.apache.iceberg.exceptions.NoSuchTableException;
+import org.apache.iceberg.io.FileIO;
+import org.apache.iceberg.relocated.com.google.common.annotations.VisibleForTesting;
+import org.apache.iceberg.relocated.com.google.common.base.Preconditions;
+import org.apache.iceberg.relocated.com.google.common.base.Strings;
+import org.apache.iceberg.relocated.com.google.common.collect.ImmutableList;
+import org.apache.iceberg.relocated.com.google.common.collect.ImmutableMap;
+import org.apache.iceberg.relocated.com.google.common.collect.Maps;
+import org.apache.iceberg.util.LocationUtil;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/** Iceberg Bigquery Metastore Catalog implementation. */
+public class BigQueryMetastoreCatalog extends BaseMetastoreCatalog
+    implements SupportsNamespaces, Configurable {
+
+  // User provided properties.
+  public static final String PROJECT_ID = "gcp.bigquery.project-id";
+  public static final String GCP_LOCATION = "gcp.bigquery.location";
+  public static final String LIST_ALL_TABLES = "gcp.bigquery.list-all-tables";
+
+  private static final Logger LOG = LoggerFactory.getLogger(BigQueryMetastoreCatalog.class);
+
+  private static final String DEFAULT_GCP_LOCATION = "us";
+
+  private String catalogName;
+  private Map<String, String> catalogProperties;
+  private FileIO fileIO;
+  private Configuration conf;
+  private String projectId;
+  private String projectLocation;
+  private BigQueryMetastoreClient client;
+  private boolean listAllTables;
+  private String warehouseLocation;
+
+  public BigQueryMetastoreCatalog() {}
+
+  @Override
+  public void initialize(String name, Map<String, String> properties) {
+    Preconditions.checkArgument(
+        properties.containsKey(PROJECT_ID),
+        "Invalid GCP project: %s must be specified",
+        PROJECT_ID);
+
+    this.projectId = properties.get(PROJECT_ID);
+    this.projectLocation = properties.getOrDefault(GCP_LOCATION, DEFAULT_GCP_LOCATION);
+
+    BigQueryOptions options =
+        BigQueryOptions.newBuilder()
+            .setProjectId(projectId)
+            .setLocation(projectLocation)
+            .setRetrySettings(ServiceOptions.getDefaultRetrySettings())
+            .build();
+
+    try {
+      client = new BigQueryMetastoreClientImpl(options);
+    } catch (IOException e) {
+      throw new UncheckedIOException("Creating BigQuery client failed", e);
+    } catch (GeneralSecurityException e) {
+      throw new RuntimeException("Creating BigQuery client failed due to a security issue", e);
+    }
+
+    initialize(name, properties, projectId, projectLocation, client);
+  }
+
+  @VisibleForTesting
+  void initialize(
+      String name,
+      Map<String, String> properties,
+      String initialProjectId,
+      String initialLocation,
+      BigQueryMetastoreClient bigQueryMetaStoreClient) {
+    Preconditions.checkArgument(bigQueryMetaStoreClient != null, "Invalid BigQuery client: null");
+    this.catalogName = name;
+    this.catalogProperties = ImmutableMap.copyOf(properties);
+    this.projectId = initialProjectId;
+    this.projectLocation = initialLocation;
+    this.client = bigQueryMetaStoreClient;
+
+    if (this.conf == null) {
+      LOG.warn("No configuration was set, using the default environment Configuration");
+      this.conf = new Configuration();
+    }
+
+    LOG.info("Using BigQuery Metastore Iceberg Catalog: {}", name);
+
+    if (properties.containsKey(CatalogProperties.WAREHOUSE_LOCATION)) {
+      this.warehouseLocation =
+          LocationUtil.stripTrailingSlash(properties.get(CatalogProperties.WAREHOUSE_LOCATION));
+    }
+
+    this.fileIO =
+        CatalogUtil.loadFileIO(
+            properties.getOrDefault(
+                CatalogProperties.FILE_IO_IMPL, "org.apache.iceberg.io.ResolvingFileIO"),
+            properties,
+            conf);
+
+    this.listAllTables = Boolean.parseBoolean(properties.getOrDefault(LIST_ALL_TABLES, "true"));
+  }
+
+  @Override
+  protected TableOperations newTableOps(TableIdentifier identifier) {
+    return new BigQueryTableOperations(client, fileIO, toTableReference(identifier), conf);
+  }
+
+  @Override
+  protected String defaultWarehouseLocation(TableIdentifier identifier) {
+    String locationUri = null;
+    DatasetReference datasetReference = toDatasetReference(identifier.namespace());
+    Dataset dataset = client.load(datasetReference);
+    if (dataset != null && dataset.getExternalCatalogDatasetOptions() != null) {
+      locationUri = dataset.getExternalCatalogDatasetOptions().getDefaultStorageLocationUri();
+    }
+
+    return String.format(
+        "%s/%s",
+        Strings.isNullOrEmpty(locationUri)
+            ? createDefaultStorageLocationUri(datasetReference.getDatasetId())
+            : LocationUtil.stripTrailingSlash(locationUri),
+        identifier.name());
+  }
+
+  @Override
+  public List<TableIdentifier> listTables(Namespace namespace) {
+    validateNamespace(namespace);
+
+    return client.list(toDatasetReference(namespace), listAllTables).stream()
+        .map(
+            table -> TableIdentifier.of(namespace.level(0), table.getTableReference().getTableId()))
+        .collect(ImmutableList.toImmutableList());
+  }
+
+  @Override
+  public boolean dropTable(TableIdentifier identifier, boolean purge) {
+    try {
+      TableOperations ops = newTableOps(identifier);
+      TableMetadata lastMetadata = ops.current();
+
+      client.delete(toTableReference(identifier));
+
+      if (purge && lastMetadata != null) {
+        CatalogUtil.dropTableData(ops.io(), lastMetadata);
+      }
+    } catch (NoSuchTableException e) {
+      return false;
+    }
+
+    return true;
+  }
+
+  @Override
+  public void renameTable(TableIdentifier from, TableIdentifier to) {
+    // TODO: Enable once supported by BigQuery API.
+    throw new UnsupportedOperationException("Table rename operation is unsupported.");
+  }
+
+  @Override
+  public void createNamespace(Namespace namespace, Map<String, String> metadata) {
+    Dataset builder = new Dataset();
+    DatasetReference datasetReference = toDatasetReference(namespace);
+    builder.setLocation(this.projectLocation);
+    builder.setDatasetReference(datasetReference);
+    builder.setExternalCatalogDatasetOptions(
+        BigQueryMetastoreUtils.createExternalCatalogDatasetOptions(
+            createDefaultStorageLocationUri(datasetReference.getDatasetId()), metadata));
+
+    client.create(builder);
+  }
+
+  @Override
+  public List<Namespace> listNamespaces() {
+    try {
+      return listNamespaces(Namespace.empty());
+    } catch (NoSuchNamespaceException e) {
+      return ImmutableList.of();
+    }
+  }
+
+  /**
+   * Since this catalog only supports one-level namespaces, it always returns an empty list unless
+   * passed an empty namespace to list all namespaces within the catalog.
+   */
+  @Override
+  public List<Namespace> listNamespaces(Namespace namespace) {
+    List<Datasets> allDatasets = client.list(projectId);
+
+    ImmutableList<Namespace> namespaces =
+        allDatasets.stream().map(this::toNamespace).collect(ImmutableList.toImmutableList());
+    if (namespaces.isEmpty()) {
+      throw new NoSuchNamespaceException("Namespace does not exist: %s", namespace);
+    }
+
+    return namespaces;
+  }
+
+  @Override
+  public boolean dropNamespace(Namespace namespace) {
+    try {
+      client.delete(toDatasetReference(namespace));
+      // We don't delete the data folder for safety, which aligns with Hive Metastore's default
+      // behavior.
+      // We can support database or catalog level config controlling file deletion in the future.
+      return true;
+    } catch (NoSuchNamespaceException e) {
+      return false;
+    }
+  }
+
+  @Override
+  public boolean setProperties(Namespace namespace, Map<String, String> properties) {
+    Dataset dataset = client.load(toDatasetReference(namespace));
+
+    ExternalCatalogDatasetOptions existingOptions = dataset.getExternalCatalogDatasetOptions();
+    Map<String, String> existingParameters =
+        existingOptions != null ? existingOptions.getParameters() : null;
+
+    Map<String, String> newParameters = Maps.newHashMap();
+    if (existingParameters != null) {
+      newParameters.putAll(existingParameters);
+    }
+
+    newParameters.putAll(properties);
+
+    if (Objects.equals(existingParameters, newParameters)) {
+      // No change in parameters detected
+      return false;
+    }
+
+    client.setParameters(toDatasetReference(namespace), properties);
+    return true;
+  }
+
+  @Override
+  public boolean removeProperties(Namespace namespace, Set<String> properties) {
+    client.removeParameters(toDatasetReference(namespace), properties);
+    return true;
+  }
+
+  @Override
+  public Map<String, String> loadNamespaceMetadata(Namespace namespace) {
+    try {
+      return toMetadata(client.load(toDatasetReference(namespace)));
+    } catch (IllegalArgumentException e) {
+      throw new NoSuchNamespaceException("%s", e.getMessage());
+    }
+  }
+
+  @Override
+  public String name() {
+    return catalogName;
+  }
+
+  @Override
+  protected Map<String, String> properties() {
+    return catalogProperties == null ? ImmutableMap.of() : catalogProperties;
+  }
+
+  @Override
+  public void setConf(Configuration conf) {
+    this.conf = new Configuration(conf);
+  }
+
+  @Override
+  public Configuration getConf() {
+    return this.conf;

Review Comment:
   nit: convention is to only use `this` with assignment.



-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org

For queries about this service, please contact Infrastructure at:
users@infra.apache.org


---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For additional commands, e-mail: issues-help@iceberg.apache.org


Re: [PR] Catalog: Add BigQuery Metastore Catalog Support [iceberg]

Posted by "nastra (via GitHub)" <gi...@apache.org>.
nastra commented on code in PR #12808:
URL: https://github.com/apache/iceberg/pull/12808#discussion_r2066013954


##########
bigquery/src/main/java/org/apache/iceberg/gcp/bigquery/BigQueryMetastoreClientImpl.java:
##########
@@ -0,0 +1,594 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.iceberg.gcp.bigquery;
+
+import com.google.api.client.googleapis.javanet.GoogleNetHttpTransport;
+import com.google.api.client.http.HttpHeaders;
+import com.google.api.client.http.HttpResponse;
+import com.google.api.client.http.HttpResponseException;
+import com.google.api.client.http.HttpStatusCodes;
+import com.google.api.client.json.gson.GsonFactory;
+import com.google.api.client.util.Data;
+import com.google.api.services.bigquery.Bigquery;
+import com.google.api.services.bigquery.BigqueryScopes;
+import com.google.api.services.bigquery.model.Dataset;
+import com.google.api.services.bigquery.model.DatasetList;
+import com.google.api.services.bigquery.model.DatasetList.Datasets;
+import com.google.api.services.bigquery.model.DatasetReference;
+import com.google.api.services.bigquery.model.ExternalCatalogDatasetOptions;
+import com.google.api.services.bigquery.model.ExternalCatalogTableOptions;
+import com.google.api.services.bigquery.model.Table;
+import com.google.api.services.bigquery.model.TableList;
+import com.google.api.services.bigquery.model.TableList.Tables;
+import com.google.api.services.bigquery.model.TableReference;
+import com.google.api.services.bigquery.model.TableSchema;
+import com.google.auth.http.HttpCredentialsAdapter;
+import com.google.auth.oauth2.GoogleCredentials;
+import com.google.cloud.BaseServiceException;
+import com.google.cloud.ExceptionHandler;
+import com.google.cloud.bigquery.BigQueryErrorMessages;
+import com.google.cloud.bigquery.BigQueryOptions;
+import com.google.cloud.bigquery.BigQueryRetryConfig;
+import com.google.cloud.bigquery.BigQueryRetryHelper;
+import java.io.IOException;
+import java.nio.charset.StandardCharsets;
+import java.security.GeneralSecurityException;
+import java.util.List;
+import java.util.Locale;
+import java.util.Map;
+import java.util.Set;
+import java.util.stream.Collectors;
+import java.util.stream.Stream;
+import org.apache.iceberg.BaseMetastoreTableOperations;
+import org.apache.iceberg.exceptions.AlreadyExistsException;
+import org.apache.iceberg.exceptions.BadRequestException;
+import org.apache.iceberg.exceptions.ForbiddenException;
+import org.apache.iceberg.exceptions.NoSuchIcebergTableException;
+import org.apache.iceberg.exceptions.NoSuchNamespaceException;
+import org.apache.iceberg.exceptions.NoSuchTableException;
+import org.apache.iceberg.exceptions.NotAuthorizedException;
+import org.apache.iceberg.exceptions.RuntimeIOException;
+import org.apache.iceberg.exceptions.ServiceFailureException;
+import org.apache.iceberg.exceptions.ServiceUnavailableException;
+import org.apache.iceberg.exceptions.ValidationException;
+import org.apache.iceberg.relocated.com.google.common.base.Preconditions;
+import org.apache.iceberg.relocated.com.google.common.collect.Lists;
+import org.apache.iceberg.relocated.com.google.common.collect.Maps;
+
+/** A client of Google Bigquery Metastore functions over the BigQuery service. */
+public final class BigQueryMetastoreClientImpl implements BigQueryMetastoreClient {
+
+  public static final ExceptionHandler.Interceptor EXCEPTION_HANDLER_INTERCEPTOR =
+      new ExceptionHandler.Interceptor() {
+
+        private static final long serialVersionUID = -8429573486870467828L;
+
+        @Override
+        public RetryResult afterEval(Exception exception, RetryResult retryResult) {
+          return ExceptionHandler.Interceptor.RetryResult.CONTINUE_EVALUATION;
+        }
+
+        @Override
+        public RetryResult beforeEval(Exception exception) {
+          if (exception instanceof BaseServiceException) {
+            boolean retriable = ((BaseServiceException) exception).isRetryable();
+            return retriable
+                ? ExceptionHandler.Interceptor.RetryResult.RETRY
+                : ExceptionHandler.Interceptor.RetryResult.CONTINUE_EVALUATION;
+          }
+
+          return ExceptionHandler.Interceptor.RetryResult.CONTINUE_EVALUATION;
+        }
+      };
+
+  private final Bigquery client;
+  private final BigQueryOptions bigqueryOptions;
+
+  // Retry config with error messages and regex for rate limit exceeded errors.
+  private static final BigQueryRetryConfig DEFAULT_RETRY_CONFIG =
+      BigQueryRetryConfig.newBuilder()
+          .retryOnMessage(BigQueryErrorMessages.RATE_LIMIT_EXCEEDED_MSG)
+          .retryOnMessage(BigQueryErrorMessages.JOB_RATE_LIMIT_EXCEEDED_MSG)
+          .retryOnRegEx(BigQueryErrorMessages.RetryRegExPatterns.RATE_LIMIT_EXCEEDED_REGEX)
+          .build();
+
+  public static final ExceptionHandler BIGQUERY_EXCEPTION_HANDLER =
+      ExceptionHandler.newBuilder()
+          .abortOn(RuntimeException.class)
+          .retryOn(
+              java.net.ConnectException
+                  .class) // Retry on connection failures due to transient network issues.
+          .retryOn(
+              java.net.UnknownHostException
+                  .class) // Retry to recover from temporary DNS resolution failures.
+          .addInterceptors(EXCEPTION_HANDLER_INTERCEPTOR)
+          .build();
+
+  /** Constructs a client of the Google BigQuery service. */
+  public BigQueryMetastoreClientImpl(BigQueryOptions options)
+      throws IOException, GeneralSecurityException {
+    // Initialize client that will be used to send requests. This client only needs to be created
+    // once, and can be reused for multiple requests
+    HttpCredentialsAdapter httpCredentialsAdapter =
+        new HttpCredentialsAdapter(
+            GoogleCredentials.getApplicationDefault().createScoped(BigqueryScopes.all()));
+    this.client =
+        new Bigquery.Builder(
+                GoogleNetHttpTransport.newTrustedTransport(),
+                GsonFactory.getDefaultInstance(),
+                httpRequest -> {
+                  httpCredentialsAdapter.initialize(httpRequest);
+                  // Instead of throwing exceptions, analyze the HttpResponse object and inspect its
+                  // status code. This will allow BigQuery API errors to be converted into Iceberg
+                  // exceptions.
+                  httpRequest.setThrowExceptionOnExecuteError(false);
+                })
+            .setApplicationName("BigQuery Metastore Iceberg Catalog Plugin")
+            .build();
+    this.bigqueryOptions = options;
+  }
+
+  @Override
+  public Dataset create(Dataset dataset) {
+    Dataset response = null;
+    try {
+      response =
+          BigQueryRetryHelper.runWithRetries(
+              () -> internalCreate(dataset),
+              bigqueryOptions.getRetrySettings(),
+              BIGQUERY_EXCEPTION_HANDLER,
+              bigqueryOptions.getClock(),
+              DEFAULT_RETRY_CONFIG);
+    } catch (BigQueryRetryHelper.BigQueryRetryHelperException e) {
+      handleBigQueryRetryException(e);
+    }
+    return response;
+  }
+
+  @SuppressWarnings("FormatStringAnnotation")
+  private Dataset internalCreate(Dataset dataset) {
+    try {
+      HttpResponse response =
+          client
+              .datasets()
+              .insert(dataset.getDatasetReference().getProjectId(), dataset)
+              .executeUnparsed();
+      return convertExceptionIfUnsuccessful(response).parseAs(Dataset.class);
+    } catch (IOException e) {
+      throw new RuntimeIOException(e);
+    } catch (AlreadyExistsException e) {
+      throw new AlreadyExistsException("Namespace already exists: " + dataset.getId());
+    }
+  }
+
+  @Override
+  @SuppressWarnings("FormatStringAnnotation")
+  public Dataset load(DatasetReference datasetReference) {
+    try {
+      HttpResponse response =
+          client
+              .datasets()
+              .get(datasetReference.getProjectId(), datasetReference.getDatasetId())
+              .executeUnparsed();
+      if (response.getStatusCode() == HttpStatusCodes.STATUS_CODE_NOT_FOUND) {
+        throw new NoSuchNamespaceException(
+            "Namespace does not exist: " + datasetReference.getDatasetId());
+      }
+
+      return convertExceptionIfUnsuccessful(response).parseAs(Dataset.class);
+    } catch (IOException e) {
+      throw new RuntimeIOException(e);
+    }
+  }
+
+  @Override
+  @SuppressWarnings("FormatStringAnnotation")
+  public void delete(DatasetReference datasetReference) {
+    try {
+      HttpResponse response =
+          client
+              .datasets()
+              .delete(datasetReference.getProjectId(), datasetReference.getDatasetId())
+              .executeUnparsed();
+      if (response.getStatusCode() == HttpStatusCodes.STATUS_CODE_NOT_FOUND) {
+        throw new NoSuchNamespaceException(
+            "Namespace does not exist: " + datasetReference.getDatasetId());
+      }
+
+      convertExceptionIfUnsuccessful(response);
+    } catch (IOException e) {
+      throw new RuntimeIOException(e);
+    }
+  }
+
+  @Override
+  public Dataset setParameters(DatasetReference datasetReference, Map<String, String> parameters) {
+    Dataset dataset = load(datasetReference);
+    ExternalCatalogDatasetOptions externalCatalogDatasetOptions =
+        dataset.getExternalCatalogDatasetOptions() == null
+            ? new ExternalCatalogDatasetOptions()
+            : dataset.getExternalCatalogDatasetOptions();
+    Map<String, String> finalParameters =
+        externalCatalogDatasetOptions.getParameters() == null
+            ? Maps.newHashMap()
+            : externalCatalogDatasetOptions.getParameters();
+    finalParameters.putAll(parameters);
+
+    dataset.setExternalCatalogDatasetOptions(
+        externalCatalogDatasetOptions.setParameters(finalParameters));
+
+    return internalUpdate(dataset);
+  }
+
+  @Override
+  public Dataset removeParameters(DatasetReference datasetReference, Set<String> parameters) {
+    Dataset dataset = load(datasetReference);
+    ExternalCatalogDatasetOptions externalCatalogDatasetOptions =
+        dataset.getExternalCatalogDatasetOptions() == null
+            ? new ExternalCatalogDatasetOptions()
+            : dataset.getExternalCatalogDatasetOptions();
+    Map<String, String> finalParameters =
+        externalCatalogDatasetOptions.getParameters() == null
+            ? Maps.newHashMap()
+            : externalCatalogDatasetOptions.getParameters();
+    parameters.forEach(finalParameters::remove);
+
+    dataset.setExternalCatalogDatasetOptions(
+        externalCatalogDatasetOptions.setParameters(finalParameters));
+
+    return internalUpdate(dataset);
+  }
+
+  @Override
+  public List<Datasets> list(String projectId) {
+    try {
+      String nextPageToken = null;
+      List<Datasets> datasets = Lists.newArrayList();
+      do {
+        HttpResponse pageResponse =
+            client.datasets().list(projectId).setPageToken(nextPageToken).executeUnparsed();
+        DatasetList result =
+            convertExceptionIfUnsuccessful(pageResponse).parseAs(DatasetList.class);
+        nextPageToken = result.getNextPageToken();
+        if (result.getDatasets() != null) {
+          datasets.addAll(result.getDatasets());
+        }
+
+      } while (nextPageToken != null && !nextPageToken.isEmpty());
+      return datasets;
+    } catch (IOException e) {
+      throw new RuntimeIOException(e);
+    }
+  }
+
+  @Override
+  public Table create(Table table) {
+    // Ensure it is an Iceberg table supported by the BigQuery metastore catalog.
+    validateTable(table);
+    // TODO: Ensure table creation is idempotent when handling retries.
+    Table response = null;
+    try {
+      response =
+          BigQueryRetryHelper.runWithRetries(
+              () -> internalCreate(table),
+              bigqueryOptions.getRetrySettings(),
+              BIGQUERY_EXCEPTION_HANDLER,
+              bigqueryOptions.getClock(),
+              DEFAULT_RETRY_CONFIG);
+    } catch (BigQueryRetryHelper.BigQueryRetryHelperException e) {
+      handleBigQueryRetryException(e);
+    }
+
+    return response;
+  }
+
+  @SuppressWarnings("FormatStringAnnotation")
+  private Table internalCreate(Table table) {
+    try {
+      HttpResponse response =
+          client
+              .tables()
+              .insert(
+                  Preconditions.checkNotNull(table.getTableReference()).getProjectId(),
+                  Preconditions.checkNotNull(table.getTableReference()).getDatasetId(),
+                  table)
+              .executeUnparsed();
+      return convertExceptionIfUnsuccessful(response).parseAs(Table.class);
+    } catch (IOException e) {
+      throw new RuntimeIOException(e);
+    } catch (AlreadyExistsException e) {
+      throw new AlreadyExistsException(String.format("Table %s already exists", table), e);
+    }
+  }
+
+  @Override
+  @SuppressWarnings("FormatStringAnnotation")
+  public Table load(TableReference tableReference) {
+    try {
+      HttpResponse response =
+          client
+              .tables()
+              .get(
+                  tableReference.getProjectId(),
+                  tableReference.getDatasetId(),
+                  tableReference.getTableId())
+              .executeUnparsed();
+      if (response.getStatusCode() == HttpStatusCodes.STATUS_CODE_NOT_FOUND) {
+        throw new NoSuchTableException(response.getStatusMessage());
+      }
+
+      return validateTable(convertExceptionIfUnsuccessful(response).parseAs(Table.class));
+    } catch (IOException e) {
+      throw new RuntimeIOException(e);
+    }
+  }
+
+  @Override
+  @SuppressWarnings("FormatStringAnnotation")
+  public Table update(TableReference tableReference, Table table) {
+    // Ensure it is an Iceberg table supported by the BQ metastore catalog.
+    validateTable(table);
+
+    ExternalCatalogTableOptions newExternalCatalogTableOptions =
+        new ExternalCatalogTableOptions()
+            .setStorageDescriptor(table.getExternalCatalogTableOptions().getStorageDescriptor())
+            .setConnectionId(table.getExternalCatalogTableOptions().getConnectionId())
+            .setParameters(table.getExternalCatalogTableOptions().getParameters());
+    Table updatedTable =
+        new Table()
+            .setExternalCatalogTableOptions(newExternalCatalogTableOptions)
+            // Must set the schema as null for using schema auto-detect.
+            .setSchema(Data.nullOf(TableSchema.class));
+
+    Table response = null;
+    try {
+      response =
+          BigQueryRetryHelper.runWithRetries(
+              () -> internalUpdate(tableReference, updatedTable, table.getEtag()),
+              bigqueryOptions.getRetrySettings(),
+              BIGQUERY_EXCEPTION_HANDLER,
+              bigqueryOptions.getClock(),
+              DEFAULT_RETRY_CONFIG);
+    } catch (BigQueryRetryHelper.BigQueryRetryHelperException e) {
+      handleBigQueryRetryException(e);
+    }
+
+    return response;
+  }
+
+  @SuppressWarnings("FormatStringAnnotation")

Review Comment:
   same as above. please also update alll the other places in this class



-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org

For queries about this service, please contact Infrastructure at:
users@infra.apache.org


---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For additional commands, e-mail: issues-help@iceberg.apache.org


Re: [PR] Catalog: Add BigQuery Metastore Catalog Support [iceberg]

Posted by "danielcweeks (via GitHub)" <gi...@apache.org>.
danielcweeks commented on code in PR #12808:
URL: https://github.com/apache/iceberg/pull/12808#discussion_r2070909585


##########
bigquery/src/main/java/org/apache/iceberg/gcp/bigquery/BigQueryMetastoreCatalog.java:
##########
@@ -0,0 +1,370 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.iceberg.gcp.bigquery;
+
+import com.google.api.services.bigquery.model.Dataset;
+import com.google.api.services.bigquery.model.DatasetList.Datasets;
+import com.google.api.services.bigquery.model.DatasetReference;
+import com.google.api.services.bigquery.model.ExternalCatalogDatasetOptions;
+import com.google.api.services.bigquery.model.TableReference;
+import com.google.cloud.ServiceOptions;
+import com.google.cloud.bigquery.BigQueryOptions;
+import java.io.IOException;
+import java.security.GeneralSecurityException;
+import java.util.List;
+import java.util.Locale;
+import java.util.Map;
+import java.util.Objects;
+import java.util.Set;
+import org.apache.hadoop.conf.Configurable;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.iceberg.BaseMetastoreCatalog;
+import org.apache.iceberg.CatalogProperties;
+import org.apache.iceberg.CatalogUtil;
+import org.apache.iceberg.TableMetadata;
+import org.apache.iceberg.TableOperations;
+import org.apache.iceberg.catalog.Namespace;
+import org.apache.iceberg.catalog.SupportsNamespaces;
+import org.apache.iceberg.catalog.TableIdentifier;
+import org.apache.iceberg.exceptions.NoSuchNamespaceException;
+import org.apache.iceberg.exceptions.NoSuchTableException;
+import org.apache.iceberg.exceptions.ServiceFailureException;
+import org.apache.iceberg.exceptions.ValidationException;
+import org.apache.iceberg.io.FileIO;
+import org.apache.iceberg.relocated.com.google.common.annotations.VisibleForTesting;
+import org.apache.iceberg.relocated.com.google.common.base.Preconditions;
+import org.apache.iceberg.relocated.com.google.common.base.Strings;
+import org.apache.iceberg.relocated.com.google.common.collect.ImmutableList;
+import org.apache.iceberg.relocated.com.google.common.collect.ImmutableMap;
+import org.apache.iceberg.relocated.com.google.common.collect.Maps;
+import org.apache.iceberg.util.LocationUtil;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/** Iceberg Bigquery Metastore Catalog implementation. */
+public class BigQueryMetastoreCatalog extends BaseMetastoreCatalog
+    implements SupportsNamespaces, Configurable {
+
+  // User provided properties.
+  public static final String PROJECT_ID = "gcp.bigquery.project-id";
+  public static final String GCP_LOCATION = "gcp.bigquery.location";
+  public static final String LIST_ALL_TABLES = "gcp.bigquery.list-all-tables";
+
+  private static final Logger LOG = LoggerFactory.getLogger(BigQueryMetastoreCatalog.class);
+
+  private static final String DEFAULT_GCP_LOCATION = "us";
+
+  private String catalogName;
+  private Map<String, String> catalogProperties;
+  private FileIO fileIO;
+  private Configuration conf;
+  private String projectId;
+  private String projectLocation;
+  private BigQueryMetastoreClient client;
+  private boolean listAllTables;
+  private String warehouseLocation;
+
+  // Must have a no-arg constructor to be dynamically loaded
+  // initialize(String name, Map<String, String> properties) will be called to complete
+  // initialization
+  public BigQueryMetastoreCatalog() {}
+
+  @Override
+  public void initialize(String name, Map<String, String> properties) {
+    Preconditions.checkArgument(
+        properties.containsKey(PROJECT_ID),
+        "Invalid GCP project: %s must be specified",
+        PROJECT_ID);
+
+    this.projectId = properties.get(PROJECT_ID);
+    this.projectLocation = properties.getOrDefault(GCP_LOCATION, DEFAULT_GCP_LOCATION);
+
+    BigQueryOptions options =
+        BigQueryOptions.newBuilder()
+            .setProjectId(projectId)
+            .setLocation(projectLocation)
+            .setRetrySettings(ServiceOptions.getDefaultRetrySettings())
+            .build();
+
+    try {
+      client = new BigQueryMetastoreClientImpl(options);
+    } catch (IOException e) {
+      throw new ServiceFailureException(e, "Creating BigQuery client failed");
+    } catch (GeneralSecurityException e) {
+      throw new ValidationException(e, "Creating BigQuery client failed due to a security issue");

Review Comment:
   This is also not the right exception to throw, if this a required exception we probably need to find a better fit (or just wrap in RuntimeException).



-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org

For queries about this service, please contact Infrastructure at:
users@infra.apache.org


---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For additional commands, e-mail: issues-help@iceberg.apache.org


Re: [PR] Catalog: Add BigQuery Metastore Catalog Support [iceberg]

Posted by "talatuyarer (via GitHub)" <gi...@apache.org>.
talatuyarer commented on code in PR #12808:
URL: https://github.com/apache/iceberg/pull/12808#discussion_r2057079908


##########
bigquery/src/main/java/org/apache/iceberg/gcp/bigquery/BigQueryMetastoreCatalog.java:
##########
@@ -0,0 +1,400 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.iceberg.gcp.bigquery;
+
+import com.google.api.services.bigquery.model.Dataset;
+import com.google.api.services.bigquery.model.DatasetList.Datasets;
+import com.google.api.services.bigquery.model.DatasetReference;
+import com.google.api.services.bigquery.model.ExternalCatalogDatasetOptions;
+import com.google.api.services.bigquery.model.TableReference;
+import com.google.cloud.ServiceOptions;
+import com.google.cloud.bigquery.BigQueryOptions;
+import java.io.IOException;
+import java.security.GeneralSecurityException;
+import java.util.List;
+import java.util.Locale;
+import java.util.Map;
+import java.util.Objects;
+import java.util.Set;
+import org.apache.hadoop.conf.Configurable;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.iceberg.BaseMetastoreCatalog;
+import org.apache.iceberg.CatalogProperties;
+import org.apache.iceberg.CatalogUtil;
+import org.apache.iceberg.TableMetadata;
+import org.apache.iceberg.TableOperations;
+import org.apache.iceberg.catalog.Namespace;
+import org.apache.iceberg.catalog.SupportsNamespaces;
+import org.apache.iceberg.catalog.TableIdentifier;
+import org.apache.iceberg.exceptions.AlreadyExistsException;
+import org.apache.iceberg.exceptions.NoSuchNamespaceException;
+import org.apache.iceberg.exceptions.NoSuchTableException;
+import org.apache.iceberg.exceptions.ServiceFailureException;
+import org.apache.iceberg.exceptions.ValidationException;
+import org.apache.iceberg.io.FileIO;
+import org.apache.iceberg.relocated.com.google.common.annotations.VisibleForTesting;
+import org.apache.iceberg.relocated.com.google.common.base.Preconditions;
+import org.apache.iceberg.relocated.com.google.common.base.Strings;
+import org.apache.iceberg.relocated.com.google.common.collect.ImmutableList;
+import org.apache.iceberg.relocated.com.google.common.collect.ImmutableMap;
+import org.apache.iceberg.relocated.com.google.common.collect.Maps;
+import org.apache.iceberg.util.LocationUtil;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/** Iceberg Bigquery Metastore Catalog implementation. */
+public final class BigQueryMetastoreCatalog extends BaseMetastoreCatalog
+    implements SupportsNamespaces, Configurable {
+
+  // User provided properties.
+  public static final String PROJECT_ID = "gcp-project";
+  public static final String GCP_LOCATION = "gcp-location";
+  public static final String FILTER_UNSUPPORTED_TABLES = "filter-unsupported-tables";
+  public static final String TESTING_ENABLED = "testing-enabled";
+
+  public static final String HIVE_METASTORE_WAREHOUSE_DIR = "hive.metastore.warehouse.dir";
+
+  private static final Logger LOG = LoggerFactory.getLogger(BigQueryMetastoreCatalog.class);
+
+  private static final String DEFAULT_GCP_LOCATION = "us";
+
+  private String catalogName;
+  private Map<String, String> catalogProperties;
+  private FileIO fileIO;
+  private Configuration conf;
+  private String projectId;
+  private String location;
+  private BigQueryMetastoreClient client;
+  private boolean filterUnsupportedTables;
+
+  // Must have a no-arg constructor to be dynamically loaded
+  // initialize(String name, Map<String, String> properties) will be called to complete
+  // initialization
+  public BigQueryMetastoreCatalog() {}
+
+  @Override
+  public void initialize(String inputName, Map<String, String> properties) {
+    Preconditions.checkArgument(
+        properties.containsKey(PROJECT_ID), "GCP project must be specified");
+
+    projectId = properties.get(PROJECT_ID);
+    location = properties.getOrDefault(GCP_LOCATION, DEFAULT_GCP_LOCATION);
+    boolean testingEnabled =
+        Boolean.parseBoolean(properties.getOrDefault(TESTING_ENABLED, "false"));
+
+    BigQueryOptions options =
+        BigQueryOptions.newBuilder()
+            .setProjectId(projectId)
+            .setLocation(location)
+            .setRetrySettings(ServiceOptions.getDefaultRetrySettings())
+            .build();
+
+    try {
+      if (testingEnabled) {
+        client = new FakeBigQueryMetastoreClient(options);
+
+      } else {
+        client = new BigQueryMetastoreClientImpl(options);
+      }
+    } catch (IOException e) {
+      throw new ServiceFailureException(e, "Creating BigQuery client failed");
+    } catch (GeneralSecurityException e) {
+      throw new ValidationException(e, "Creating BigQuery client failed due to a security issue");
+    }
+    initialize(inputName, properties, projectId, location, client);
+  }
+
+  @VisibleForTesting
+  void initialize(
+      String inputName,
+      Map<String, String> properties,
+      String initialProjectId,
+      String initialLocation,
+      BigQueryMetastoreClient bigQueryMetaStoreClient) {
+    Preconditions.checkArgument(bigQueryMetaStoreClient != null, "BigQuery client can not be null");
+    this.catalogName = inputName;
+    this.catalogProperties = ImmutableMap.copyOf(properties);
+    this.projectId = initialProjectId;
+    this.location = initialLocation;
+    this.client = bigQueryMetaStoreClient;
+
+    if (this.conf == null) {
+      LOG.warn("No configuration was set, using the default environment Configuration");
+      this.conf = new Configuration();
+    }
+
+    LOG.info("Using BigQuery Metastore Iceberg Catalog: {}", inputName);
+
+    if (properties.containsKey(CatalogProperties.WAREHOUSE_LOCATION)) {
+      // Iceberg always removes trailing slash to avoid paths like "<folder>//data" in file systems
+      // like s3.
+      this.conf.set(
+          HIVE_METASTORE_WAREHOUSE_DIR,
+          LocationUtil.stripTrailingSlash(properties.get(CatalogProperties.WAREHOUSE_LOCATION)));
+    }
+
+    String fileIoImpl =
+        properties.getOrDefault(
+            CatalogProperties.FILE_IO_IMPL, "org.apache.iceberg.hadoop.HadoopFileIO");
+    this.fileIO = CatalogUtil.loadFileIO(fileIoImpl, properties, conf);
+
+    this.filterUnsupportedTables =
+        Boolean.parseBoolean(properties.getOrDefault(FILTER_UNSUPPORTED_TABLES, "false"));
+  }
+
+  @Override
+  protected TableOperations newTableOps(TableIdentifier identifier) {
+    return new BigQueryTableOperations(
+        client,
+        fileIO,
+        projectId,
+        // Sometimes extensions have the namespace contain the table name too, so we are forced to
+        // allow invalid namespace and just take the first part here like other catalog
+        // implementations do.
+        identifier.namespace().level(0),
+        identifier.name(),
+        conf);
+  }
+
+  @Override
+  protected String defaultWarehouseLocation(TableIdentifier identifier) {
+    String locationUri = null;
+    DatasetReference datasetReference = toDatasetReference(identifier.namespace());
+    Dataset dataset = client.getDataset(datasetReference);
+    if (dataset != null && dataset.getExternalCatalogDatasetOptions() != null) {
+      locationUri = dataset.getExternalCatalogDatasetOptions().getDefaultStorageLocationUri();
+    }
+    return String.format(
+        "%s/%s",
+        Strings.isNullOrEmpty(locationUri)
+            ? datasetReference.getDatasetId()
+            : LocationUtil.stripTrailingSlash(locationUri),
+        identifier.name());
+  }
+
+  @Override
+  public List<TableIdentifier> listTables(Namespace namespace) {
+    validateNamespace(namespace);
+
+    return client.listTables(toDatasetReference(namespace), filterUnsupportedTables).stream()
+        .map(
+            table -> TableIdentifier.of(namespace.level(0), table.getTableReference().getTableId()))
+        .collect(ImmutableList.toImmutableList());
+  }
+
+  @Override
+  public boolean dropTable(TableIdentifier identifier, boolean purge) {
+    try {
+      TableOperations ops = newTableOps(identifier);
+      TableMetadata lastMetadata = ops.current();
+
+      client.deleteTable(toTableReference(identifier));
+
+      if (purge && lastMetadata != null) {
+        CatalogUtil.dropTableData(ops.io(), lastMetadata);
+      }
+    } catch (NoSuchTableException e) { // Not catching a NoSuchIcebergTableException on purpose
+      return false; // The documentation says just return false in this case
+    }
+    return true;
+  }
+
+  @Override
+  public void renameTable(TableIdentifier from, TableIdentifier to) {
+    validateNamespace(from.namespace());
+    validateNamespace(to.namespace());
+
+    try {
+      client.getDataset(toDatasetReference(to.namespace()));
+      client.getDataset(toDatasetReference(from.namespace()));
+    } catch (NoSuchNamespaceException e) {
+      throw e;
+    }
+
+    TableReference fromTableRef = toTableReference(from);
+    TableReference toTableRef = toTableReference(to);
+
+    // Check if the source table exists
+    try {
+      client.getTable(fromTableRef);
+    } catch (NoSuchTableException e) {
+      throw new NoSuchTableException("Table does not exist: %s", from.name());
+    }
+
+    // Check if the destination table already exists
+    try {
+      client.getTable(toTableRef);
+      // If getTable succeeds, the destination table exists
+      throw new AlreadyExistsException("Table already exists: %s", to.name());
+    } catch (NoSuchTableException e) {
+      // Destination table does not exist, proceed with throwing exception
+    }
+
+    if (!from.namespace().equals(to.namespace())) {
+      throw new ValidationException("New table name must be in the same namespace");
+    }
+
+    // TODO(b/354981675): Enable once supported by the API.
+    throw new UnsupportedOperationException("Table rename operation is unsupported.");
+  }
+
+  @Override
+  public void createNamespace(Namespace namespace, Map<String, String> metadata) {
+    Dataset builder = new Dataset();
+    DatasetReference datasetReference = toDatasetReference(namespace);
+    builder.setLocation(this.location);
+    builder.setDatasetReference(datasetReference);
+    builder.setExternalCatalogDatasetOptions(
+        BigQueryMetastoreUtils.createExternalCatalogDatasetOptions(
+            datasetReference.getDatasetId(), metadata));
+
+    client.createDataset(builder);
+  }
+
+  /**
+   * Since this catalog only supports one-level namespaces, it always returns an empty list unless
+   * passed an empty namespace to list all namespaces within the catalog.
+   */
+  @Override
+  public List<Namespace> listNamespaces(Namespace namespace) {
+    if (namespace.levels().length != 0) {
+      // BQMS does not support namespaces under database or tables, returns empty.
+      // It is called when dropping a namespace to make sure it's empty (listTables is called as
+      // well), returns empty to unblock deletion.
+      return ImmutableList.of();
+    }
+    return client.listDatasets(projectId).stream()
+        .map(BigQueryMetastoreCatalog::getNamespace)
+        .collect(ImmutableList.toImmutableList());
+  }
+
+  @Override
+  public boolean dropNamespace(Namespace namespace) {
+    try {
+      client.deleteDataset(toDatasetReference(namespace));
+      // We don't delete the data folder for safety, which aligns with Hive Metastore's default
+      // behavior.
+      // We can support database or catalog level config controlling file deletion in the future.
+      return true;
+    } catch (NoSuchNamespaceException e) {
+      return false;
+    }
+  }
+
+  @Override
+  public boolean setProperties(Namespace namespace, Map<String, String> properties) {
+    Dataset dataset = client.getDataset(toDatasetReference(namespace));
+
+    ExternalCatalogDatasetOptions existingOptions = dataset.getExternalCatalogDatasetOptions();
+    Map<String, String> existingParameters =
+        existingOptions != null ? existingOptions.getParameters() : null;
+
+    Map<String, String> newParameters = Maps.newHashMap();
+    if (existingParameters != null) {
+      newParameters.putAll(existingParameters);
+    }
+    newParameters.putAll(properties);
+
+    if (Objects.equals(existingParameters, newParameters)) {
+      // No change in parameters detected
+      return false;
+    }
+
+    client.setDatasetParameters(toDatasetReference(namespace), properties);
+    return true;
+  }
+
+  @Override
+  public boolean removeProperties(Namespace namespace, Set<String> properties) {
+    client.removeDatasetParameters(toDatasetReference(namespace), properties);
+    return true;
+  }
+
+  @SuppressWarnings("FormatStringAnnotation")
+  @Override
+  public Map<String, String> loadNamespaceMetadata(Namespace namespace) {
+    try {
+      return getMetadata(client.getDataset(toDatasetReference(namespace)));
+    } catch (IllegalArgumentException e) {
+      throw new NoSuchNamespaceException(e.getMessage());
+    }
+  }
+
+  @Override
+  public String name() {
+    return catalogName;
+  }
+
+  @Override
+  protected Map<String, String> properties() {
+    return catalogProperties == null ? ImmutableMap.of() : catalogProperties;
+  }
+
+  @Override
+  public void setConf(Configuration conf) {
+    this.conf = new Configuration(conf);
+  }
+
+  @Override
+  public Configuration getConf() {
+    return this.conf;
+  }
+
+  private static Namespace getNamespace(Datasets dataset) {
+    return Namespace.of(dataset.getDatasetReference().getDatasetId());
+  }
+
+  private DatasetReference toDatasetReference(Namespace namespace) {
+    validateNamespace(namespace);
+    return new DatasetReference().setProjectId(projectId).setDatasetId(namespace.level(0));
+  }
+
+  private TableReference toTableReference(TableIdentifier tableIdentifier) {
+    DatasetReference datasetReference = toDatasetReference(tableIdentifier.namespace());
+    return new TableReference()
+        .setProjectId(datasetReference.getProjectId())
+        .setDatasetId(datasetReference.getDatasetId())
+        .setTableId(tableIdentifier.name());
+  }
+
+  private static Map<String, String> getMetadata(Dataset dataset) {
+    ExternalCatalogDatasetOptions options = dataset.getExternalCatalogDatasetOptions();
+    Map<String, String> metadata = Maps.newHashMap();
+    if (options != null) {
+      if (options.getParameters() != null) {
+        metadata.putAll(options.getParameters());
+      }
+      if (!Strings.isNullOrEmpty(options.getDefaultStorageLocationUri())) {
+        metadata.put("location", options.getDefaultStorageLocationUri());

Review Comment:
   No, The storage location URI is equivalent to hive metastore's database locationUri in the dataset. 



-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org

For queries about this service, please contact Infrastructure at:
users@infra.apache.org


---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For additional commands, e-mail: issues-help@iceberg.apache.org


Re: [PR] Catalog: Add BigQuery Metastore Catalog Support [iceberg]

Posted by "ebyhr (via GitHub)" <gi...@apache.org>.
ebyhr commented on code in PR #12808:
URL: https://github.com/apache/iceberg/pull/12808#discussion_r2059717294


##########
bigquery/src/test/java/org/apache/iceberg/gcp/bigquery/TestBigQueryCatalog.java:
##########
@@ -0,0 +1,673 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.iceberg.gcp.bigquery;
+
+import static org.apache.iceberg.CatalogUtil.ICEBERG_CATALOG_TYPE;
+import static org.apache.iceberg.CatalogUtil.ICEBERG_CATALOG_TYPE_BIGQUERY;
+import static org.apache.iceberg.gcp.bigquery.BigQueryMetastoreCatalog.PROJECT_ID;
+import static org.apache.iceberg.gcp.bigquery.BigQueryMetastoreCatalog.TESTING_ENABLED;
+import static org.apache.iceberg.types.Types.NestedField.optional;
+import static org.assertj.core.api.Assertions.assertThat;
+import static org.assertj.core.api.Assertions.assertThatThrownBy;
+
+import java.io.File;
+import java.util.List;
+import java.util.Map;
+import org.apache.iceberg.CatalogProperties;
+import org.apache.iceberg.CatalogUtil;
+import org.apache.iceberg.PartitionSpec;
+import org.apache.iceberg.Schema;
+import org.apache.iceberg.SortOrder;
+import org.apache.iceberg.Table;
+import org.apache.iceberg.TableUtil;
+import org.apache.iceberg.Transaction;
+import org.apache.iceberg.catalog.CatalogTests;
+import org.apache.iceberg.catalog.Namespace;
+import org.apache.iceberg.catalog.TableIdentifier;
+import org.apache.iceberg.exceptions.NoSuchNamespaceException;
+import org.apache.iceberg.exceptions.NoSuchTableException;
+import org.apache.iceberg.expressions.Expressions;
+import org.apache.iceberg.relocated.com.google.common.collect.ImmutableMap;
+import org.apache.iceberg.types.Types;
+import org.junit.jupiter.api.AfterEach;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Disabled;
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.io.TempDir;
+import org.junit.jupiter.params.ParameterizedTest;
+import org.junit.jupiter.params.provider.ValueSource;
+
+public class TestBigQueryCatalog extends CatalogTests<BigQueryMetastoreCatalog> {
+
+  private static final String GCP_PROJECT = "project-id";
+  private static final String CATALOG_ID = "catalog-name";
+
+  // another schema that is not the same
+  protected static final Schema OTHER_SCHEMA_OPTIONAL =
+      new Schema(optional(1, "some_id", Types.IntegerType.get()));
+
+  // Schema passed to create tables
+  protected static final Schema SCHEMA_OPTIONAL =
+      new Schema(
+          optional(3, "id", Types.IntegerType.get(), "unique ID 🤪"),
+          optional(4, "data", Types.StringType.get()));
+
+  // This is the actual schema for the table, with column IDs reassigned
+  protected static final Schema REPLACE_SCHEMA_OPTIONAL =
+      new Schema(
+          optional(2, "id", Types.IntegerType.get(), "unique ID 🤪"),
+          optional(3, "data", Types.StringType.get()));
+
+  protected static final PartitionSpec REPLACE_SPEC_OPTIONAL =
+      PartitionSpec.builderFor(REPLACE_SCHEMA_OPTIONAL).bucket("id", 16).withSpecId(1).build();
+
+  protected static final SortOrder REPLACE_WRITE_ORDER_OPTIONAL =
+      SortOrder.builderFor(REPLACE_SCHEMA_OPTIONAL)
+          .asc(Expressions.bucket("id", 16))
+          .asc("id")
+          .build();
+
+  @TempDir private File tempFolder;
+  private BigQueryMetastoreCatalog catalog;
+  private String warehouseLocation;
+
+  @BeforeEach
+  public void before() throws Exception {
+    catalog = initCatalog(CATALOG_ID, ImmutableMap.of());
+  }
+
+  @AfterEach
+  public void after() throws Exception {
+    // Drop all tables in all datasets
+    List<Namespace> namespaces = catalog().listNamespaces(Namespace.empty());
+    for (Namespace namespace : namespaces) {
+      List<TableIdentifier> tables = catalog().listTables(namespace);
+      for (TableIdentifier table : tables) {
+        try {
+          catalog().dropTable(table, true); // drop with purge

Review Comment:
   > drop with purge
   
   I would remove this comment. It's obvious from the code. 



##########
bigquery/src/test/java/org/apache/iceberg/gcp/bigquery/TestBigQueryCatalog.java:
##########
@@ -0,0 +1,673 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.iceberg.gcp.bigquery;
+
+import static org.apache.iceberg.CatalogUtil.ICEBERG_CATALOG_TYPE;
+import static org.apache.iceberg.CatalogUtil.ICEBERG_CATALOG_TYPE_BIGQUERY;
+import static org.apache.iceberg.gcp.bigquery.BigQueryMetastoreCatalog.PROJECT_ID;
+import static org.apache.iceberg.gcp.bigquery.BigQueryMetastoreCatalog.TESTING_ENABLED;
+import static org.apache.iceberg.types.Types.NestedField.optional;
+import static org.assertj.core.api.Assertions.assertThat;
+import static org.assertj.core.api.Assertions.assertThatThrownBy;
+
+import java.io.File;
+import java.util.List;
+import java.util.Map;
+import org.apache.iceberg.CatalogProperties;
+import org.apache.iceberg.CatalogUtil;
+import org.apache.iceberg.PartitionSpec;
+import org.apache.iceberg.Schema;
+import org.apache.iceberg.SortOrder;
+import org.apache.iceberg.Table;
+import org.apache.iceberg.TableUtil;
+import org.apache.iceberg.Transaction;
+import org.apache.iceberg.catalog.CatalogTests;
+import org.apache.iceberg.catalog.Namespace;
+import org.apache.iceberg.catalog.TableIdentifier;
+import org.apache.iceberg.exceptions.NoSuchNamespaceException;
+import org.apache.iceberg.exceptions.NoSuchTableException;
+import org.apache.iceberg.expressions.Expressions;
+import org.apache.iceberg.relocated.com.google.common.collect.ImmutableMap;
+import org.apache.iceberg.types.Types;
+import org.junit.jupiter.api.AfterEach;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Disabled;
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.io.TempDir;
+import org.junit.jupiter.params.ParameterizedTest;
+import org.junit.jupiter.params.provider.ValueSource;
+
+public class TestBigQueryCatalog extends CatalogTests<BigQueryMetastoreCatalog> {
+
+  private static final String GCP_PROJECT = "project-id";
+  private static final String CATALOG_ID = "catalog-name";
+
+  // another schema that is not the same
+  protected static final Schema OTHER_SCHEMA_OPTIONAL =
+      new Schema(optional(1, "some_id", Types.IntegerType.get()));
+
+  // Schema passed to create tables
+  protected static final Schema SCHEMA_OPTIONAL =
+      new Schema(
+          optional(3, "id", Types.IntegerType.get(), "unique ID 🤪"),
+          optional(4, "data", Types.StringType.get()));
+
+  // This is the actual schema for the table, with column IDs reassigned
+  protected static final Schema REPLACE_SCHEMA_OPTIONAL =
+      new Schema(
+          optional(2, "id", Types.IntegerType.get(), "unique ID 🤪"),
+          optional(3, "data", Types.StringType.get()));
+
+  protected static final PartitionSpec REPLACE_SPEC_OPTIONAL =
+      PartitionSpec.builderFor(REPLACE_SCHEMA_OPTIONAL).bucket("id", 16).withSpecId(1).build();
+
+  protected static final SortOrder REPLACE_WRITE_ORDER_OPTIONAL =

Review Comment:
   These constants can be private. 



-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org

For queries about this service, please contact Infrastructure at:
users@infra.apache.org


---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For additional commands, e-mail: issues-help@iceberg.apache.org


Re: [PR] Catalog: Add BigQuery Metastore Catalog Support [iceberg]

Posted by "danielcweeks (via GitHub)" <gi...@apache.org>.
danielcweeks commented on code in PR #12808:
URL: https://github.com/apache/iceberg/pull/12808#discussion_r2080088898


##########
bigquery/src/main/java/org/apache/iceberg/gcp/bigquery/BigQueryMetastoreCatalog.java:
##########
@@ -0,0 +1,366 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.iceberg.gcp.bigquery;
+
+import com.google.api.services.bigquery.model.Dataset;
+import com.google.api.services.bigquery.model.DatasetList.Datasets;
+import com.google.api.services.bigquery.model.DatasetReference;
+import com.google.api.services.bigquery.model.ExternalCatalogDatasetOptions;
+import com.google.api.services.bigquery.model.TableReference;
+import com.google.cloud.ServiceOptions;
+import com.google.cloud.bigquery.BigQueryOptions;
+import java.io.IOException;
+import java.io.UncheckedIOException;
+import java.security.GeneralSecurityException;
+import java.util.List;
+import java.util.Locale;
+import java.util.Map;
+import java.util.Objects;
+import java.util.Set;
+import org.apache.hadoop.conf.Configurable;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.iceberg.BaseMetastoreCatalog;
+import org.apache.iceberg.CatalogProperties;
+import org.apache.iceberg.CatalogUtil;
+import org.apache.iceberg.TableMetadata;
+import org.apache.iceberg.TableOperations;
+import org.apache.iceberg.catalog.Namespace;
+import org.apache.iceberg.catalog.SupportsNamespaces;
+import org.apache.iceberg.catalog.TableIdentifier;
+import org.apache.iceberg.exceptions.NoSuchNamespaceException;
+import org.apache.iceberg.exceptions.NoSuchTableException;
+import org.apache.iceberg.io.FileIO;
+import org.apache.iceberg.relocated.com.google.common.annotations.VisibleForTesting;
+import org.apache.iceberg.relocated.com.google.common.base.Preconditions;
+import org.apache.iceberg.relocated.com.google.common.base.Strings;
+import org.apache.iceberg.relocated.com.google.common.collect.ImmutableList;
+import org.apache.iceberg.relocated.com.google.common.collect.ImmutableMap;
+import org.apache.iceberg.relocated.com.google.common.collect.Maps;
+import org.apache.iceberg.util.LocationUtil;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/** Iceberg Bigquery Metastore Catalog implementation. */
+public class BigQueryMetastoreCatalog extends BaseMetastoreCatalog
+    implements SupportsNamespaces, Configurable {
+
+  // User provided properties.
+  public static final String PROJECT_ID = "gcp.bigquery.project-id";
+  public static final String GCP_LOCATION = "gcp.bigquery.location";
+  public static final String LIST_ALL_TABLES = "gcp.bigquery.list-all-tables";
+
+  private static final Logger LOG = LoggerFactory.getLogger(BigQueryMetastoreCatalog.class);
+
+  private static final String DEFAULT_GCP_LOCATION = "us";
+
+  private String catalogName;
+  private Map<String, String> catalogProperties;
+  private FileIO fileIO;
+  private Configuration conf;
+  private String projectId;
+  private String projectLocation;
+  private BigQueryMetastoreClient client;
+  private boolean listAllTables;
+  private String warehouseLocation;
+
+  public BigQueryMetastoreCatalog() {}
+
+  @Override
+  public void initialize(String name, Map<String, String> properties) {
+    Preconditions.checkArgument(
+        properties.containsKey(PROJECT_ID),
+        "Invalid GCP project: %s must be specified",
+        PROJECT_ID);
+
+    this.projectId = properties.get(PROJECT_ID);
+    this.projectLocation = properties.getOrDefault(GCP_LOCATION, DEFAULT_GCP_LOCATION);
+
+    BigQueryOptions options =
+        BigQueryOptions.newBuilder()
+            .setProjectId(projectId)
+            .setLocation(projectLocation)
+            .setRetrySettings(ServiceOptions.getDefaultRetrySettings())
+            .build();
+
+    try {
+      client = new BigQueryMetastoreClientImpl(options);
+    } catch (IOException e) {
+      throw new UncheckedIOException("Creating BigQuery client failed", e);
+    } catch (GeneralSecurityException e) {
+      throw new RuntimeException("Creating BigQuery client failed due to a security issue", e);
+    }
+
+    initialize(name, properties, projectId, projectLocation, client);
+  }
+
+  @VisibleForTesting
+  void initialize(
+      String name,
+      Map<String, String> properties,
+      String initialProjectId,
+      String initialLocation,
+      BigQueryMetastoreClient bigQueryMetaStoreClient) {
+    Preconditions.checkArgument(bigQueryMetaStoreClient != null, "Invalid BigQuery client: null");
+    this.catalogName = name;
+    this.catalogProperties = ImmutableMap.copyOf(properties);
+    this.projectId = initialProjectId;
+    this.projectLocation = initialLocation;
+    this.client = bigQueryMetaStoreClient;
+
+    if (this.conf == null) {
+      LOG.warn("No configuration was set, using the default environment Configuration");
+      this.conf = new Configuration();
+    }
+
+    LOG.info("Using BigQuery Metastore Iceberg Catalog: {}", name);
+
+    if (properties.containsKey(CatalogProperties.WAREHOUSE_LOCATION)) {
+      this.warehouseLocation =
+          LocationUtil.stripTrailingSlash(properties.get(CatalogProperties.WAREHOUSE_LOCATION));
+    }
+
+    this.fileIO =
+        CatalogUtil.loadFileIO(
+            properties.getOrDefault(
+                CatalogProperties.FILE_IO_IMPL, "org.apache.iceberg.io.ResolvingFileIO"),
+            properties,
+            conf);
+
+    this.listAllTables = Boolean.parseBoolean(properties.getOrDefault(LIST_ALL_TABLES, "true"));
+  }
+
+  @Override
+  protected TableOperations newTableOps(TableIdentifier identifier) {
+    return new BigQueryTableOperations(client, fileIO, toTableReference(identifier), conf);
+  }
+
+  @Override
+  protected String defaultWarehouseLocation(TableIdentifier identifier) {
+    String locationUri = null;
+    DatasetReference datasetReference = toDatasetReference(identifier.namespace());
+    Dataset dataset = client.load(datasetReference);
+    if (dataset != null && dataset.getExternalCatalogDatasetOptions() != null) {
+      locationUri = dataset.getExternalCatalogDatasetOptions().getDefaultStorageLocationUri();
+    }
+
+    return String.format(
+        "%s/%s",
+        Strings.isNullOrEmpty(locationUri)
+            ? createDefaultStorageLocationUri(datasetReference.getDatasetId())
+            : LocationUtil.stripTrailingSlash(locationUri),
+        identifier.name());
+  }
+
+  @Override
+  public List<TableIdentifier> listTables(Namespace namespace) {
+    validateNamespace(namespace);
+
+    return client.list(toDatasetReference(namespace), listAllTables).stream()
+        .map(
+            table -> TableIdentifier.of(namespace.level(0), table.getTableReference().getTableId()))
+        .collect(ImmutableList.toImmutableList());
+  }
+
+  @Override
+  public boolean dropTable(TableIdentifier identifier, boolean purge) {
+    try {
+      TableOperations ops = newTableOps(identifier);
+      TableMetadata lastMetadata = ops.current();
+
+      client.delete(toTableReference(identifier));
+
+      if (purge && lastMetadata != null) {
+        CatalogUtil.dropTableData(ops.io(), lastMetadata);
+      }
+    } catch (NoSuchTableException e) {
+      return false;
+    }
+
+    return true;
+  }
+
+  @Override
+  public void renameTable(TableIdentifier from, TableIdentifier to) {
+    // TODO: Enable once supported by BigQuery API.
+    throw new UnsupportedOperationException("Table rename operation is unsupported.");
+  }
+
+  @Override
+  public void createNamespace(Namespace namespace, Map<String, String> metadata) {
+    Dataset builder = new Dataset();
+    DatasetReference datasetReference = toDatasetReference(namespace);
+    builder.setLocation(this.projectLocation);
+    builder.setDatasetReference(datasetReference);
+    builder.setExternalCatalogDatasetOptions(
+        BigQueryMetastoreUtils.createExternalCatalogDatasetOptions(
+            createDefaultStorageLocationUri(datasetReference.getDatasetId()), metadata));
+
+    client.create(builder);
+  }
+
+  @Override
+  public List<Namespace> listNamespaces() {
+    try {
+      return listNamespaces(Namespace.empty());
+    } catch (NoSuchNamespaceException e) {
+      return ImmutableList.of();
+    }
+  }
+
+  /**
+   * Since this catalog only supports one-level namespaces, it always returns an empty list unless
+   * passed an empty namespace to list all namespaces within the catalog.
+   */
+  @Override
+  public List<Namespace> listNamespaces(Namespace namespace) {
+    List<Datasets> allDatasets = client.list(projectId);
+
+    ImmutableList<Namespace> namespaces =
+        allDatasets.stream().map(this::toNamespace).collect(ImmutableList.toImmutableList());
+    if (namespaces.isEmpty()) {
+      throw new NoSuchNamespaceException("Namespace does not exist: %s", namespace);
+    }
+
+    return namespaces;
+  }
+
+  @Override
+  public boolean dropNamespace(Namespace namespace) {
+    try {
+      client.delete(toDatasetReference(namespace));

Review Comment:
   The test I'm concerned about is this one [testDropNonEmptyNamespace](https://github.com/apache/iceberg/blob/7f3f450bbddf55bb383ff1409d6d0ca4557c9ffc/core/src/test/java/org/apache/iceberg/catalog/CatalogTests.java#L401)
   
   But this highlights and issue we have here with how we're testing.  The client in this case is throwing the expected exception, but that won't happen with the real client.  So, in effect we're producing the results for the test, not testing the behavior.



-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org

For queries about this service, please contact Infrastructure at:
users@infra.apache.org


---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For additional commands, e-mail: issues-help@iceberg.apache.org


Re: [PR] Catalog: Add BigQuery Metastore Catalog Support [iceberg]

Posted by "nastra (via GitHub)" <gi...@apache.org>.
nastra commented on code in PR #12808:
URL: https://github.com/apache/iceberg/pull/12808#discussion_r2055541694


##########
bigquery/src/test/java/org/apache/iceberg/gcp/bigquery/TestBigQueryCatalog.java:
##########
@@ -0,0 +1,673 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.iceberg.gcp.bigquery;
+
+import static org.apache.iceberg.CatalogUtil.ICEBERG_CATALOG_TYPE;
+import static org.apache.iceberg.CatalogUtil.ICEBERG_CATALOG_TYPE_BIGQUERY;
+import static org.apache.iceberg.gcp.bigquery.BigQueryMetastoreCatalog.PROJECT_ID;
+import static org.apache.iceberg.gcp.bigquery.BigQueryMetastoreCatalog.TESTING_ENABLED;
+import static org.apache.iceberg.types.Types.NestedField.optional;
+import static org.assertj.core.api.Assertions.assertThat;
+import static org.assertj.core.api.Assertions.assertThatThrownBy;
+
+import java.io.File;
+import java.util.List;
+import java.util.Map;
+import org.apache.iceberg.CatalogProperties;
+import org.apache.iceberg.CatalogUtil;
+import org.apache.iceberg.PartitionSpec;
+import org.apache.iceberg.Schema;
+import org.apache.iceberg.SortOrder;
+import org.apache.iceberg.Table;
+import org.apache.iceberg.TableUtil;
+import org.apache.iceberg.Transaction;
+import org.apache.iceberg.catalog.CatalogTests;
+import org.apache.iceberg.catalog.Namespace;
+import org.apache.iceberg.catalog.TableIdentifier;
+import org.apache.iceberg.exceptions.NoSuchNamespaceException;
+import org.apache.iceberg.exceptions.NoSuchTableException;
+import org.apache.iceberg.expressions.Expressions;
+import org.apache.iceberg.relocated.com.google.common.collect.ImmutableMap;
+import org.apache.iceberg.types.Types;
+import org.junit.jupiter.api.AfterEach;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Disabled;
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.io.TempDir;
+import org.junit.jupiter.params.ParameterizedTest;
+import org.junit.jupiter.params.provider.ValueSource;
+
+public class TestBigQueryCatalog extends CatalogTests<BigQueryMetastoreCatalog> {
+
+  private static final String GCP_PROJECT = "project-id";
+  private static final String CATALOG_ID = "catalog-name";
+
+  // another schema that is not the same
+  protected static final Schema OTHER_SCHEMA_OPTIONAL =
+      new Schema(optional(1, "some_id", Types.IntegerType.get()));
+
+  // Schema passed to create tables
+  protected static final Schema SCHEMA_OPTIONAL =
+      new Schema(
+          optional(3, "id", Types.IntegerType.get(), "unique ID 🤪"),
+          optional(4, "data", Types.StringType.get()));
+
+  // This is the actual schema for the table, with column IDs reassigned
+  protected static final Schema REPLACE_SCHEMA_OPTIONAL =
+      new Schema(
+          optional(2, "id", Types.IntegerType.get(), "unique ID 🤪"),
+          optional(3, "data", Types.StringType.get()));
+
+  protected static final PartitionSpec REPLACE_SPEC_OPTIONAL =
+      PartitionSpec.builderFor(REPLACE_SCHEMA_OPTIONAL).bucket("id", 16).withSpecId(1).build();
+
+  protected static final SortOrder REPLACE_WRITE_ORDER_OPTIONAL =
+      SortOrder.builderFor(REPLACE_SCHEMA_OPTIONAL)
+          .asc(Expressions.bucket("id", 16))
+          .asc("id")
+          .build();
+
+  @TempDir private File tempFolder;
+  private BigQueryMetastoreCatalog catalog;
+  private BigQueryMetastoreClient fakeBigQueryMetastoreClient;
+  private String warehouseLocation;
+
+  @BeforeEach
+  public void before() throws Exception {
+    catalog = initCatalog(CATALOG_ID, ImmutableMap.of());
+  }
+
+  @AfterEach
+  public void after() throws Exception {
+    // Drop all tables in all datasets
+    List<Namespace> namespaces = catalog().listNamespaces(Namespace.empty());
+    for (Namespace namespace : namespaces) {
+      List<TableIdentifier> tables = catalog().listTables(namespace);
+      for (TableIdentifier table : tables) {
+        try {
+          catalog().dropTable(table, true); // drop with purge
+        } catch (NoSuchTableException e) {
+          // Table might be dropped by a previous iteration, ignore
+        }
+      }
+
+      // Drop all datasets (except any initial ones)
+      try {
+        catalog().dropNamespace(namespace);
+      } catch (NoSuchNamespaceException e) {
+        // Namespace might be dropped by a previous iteration, ignore
+      }
+    }
+  }
+
+  @Override
+  protected boolean requiresNamespaceCreate() {
+    return true;
+  }
+
+  @Override
+  protected boolean supportsNamesWithSlashes() {
+    return false;
+  }
+
+  @Override
+  protected boolean supportsNamesWithDot() {
+    return false;
+  }
+
+  @Override
+  public BigQueryMetastoreCatalog catalog() {
+    return catalog;
+  }
+
+  @Override
+  protected BigQueryMetastoreCatalog initCatalog(
+      String catalogName, Map<String, String> additionalProperties) {
+
+    warehouseLocation = tempFolder.toPath().resolve("hive-warehouse").toString();
+
+    Map<String, String> properties =
+        Map.of(
+            ICEBERG_CATALOG_TYPE,
+            ICEBERG_CATALOG_TYPE_BIGQUERY,
+            PROJECT_ID,
+            GCP_PROJECT,
+            CatalogProperties.WAREHOUSE_LOCATION,
+            warehouseLocation,
+            TESTING_ENABLED,
+            "true",
+            CatalogProperties.TABLE_DEFAULT_PREFIX + "default-key1",
+            "catalog-default-key1",
+            CatalogProperties.TABLE_DEFAULT_PREFIX + "default-key2",
+            "catalog-default-key2",
+            CatalogProperties.TABLE_DEFAULT_PREFIX + "override-key3",
+            "catalog-default-key3",
+            CatalogProperties.TABLE_OVERRIDE_PREFIX + "override-key3",
+            "catalog-override-key3",
+            CatalogProperties.TABLE_OVERRIDE_PREFIX + "override-key4",
+            "catalog-override-key4");
+
+    BigQueryMetastoreCatalog tmpCatalog =
+        (BigQueryMetastoreCatalog)
+            CatalogUtil.buildIcebergCatalog(
+                catalogName,
+                ImmutableMap.<String, String>builder()
+                    .putAll(properties)
+                    .putAll(additionalProperties)
+                    .build(),
+                null);
+
+    return tmpCatalog;
+  }
+
+  @Test
+  public void testRenameTable() {
+    if (requiresNamespaceCreate()) {
+      catalog.createNamespace(NS);
+    }
+
+    assertThat(catalog.tableExists(TABLE))
+        .as("Source table should not exist before create")
+        .isFalse();
+
+    catalog.buildTable(TABLE, SCHEMA).create();
+    assertThat(catalog.tableExists(TABLE)).as("Table should exist after create").isTrue();
+
+    assertThat(catalog.tableExists(RENAMED_TABLE))
+        .as("Destination table should not exist before rename")
+        .isFalse();
+
+    assertThatThrownBy(() -> catalog.renameTable(TABLE, RENAMED_TABLE))
+        .isInstanceOf(UnsupportedOperationException.class)
+        .hasMessageContaining("Table rename operation is unsupported");
+
+    catalog.dropTable(TABLE);
+  }
+
+  @Disabled(
+      "BigQuery does not allow adding a REQUIRED column to an existing table schema. "
+          + "When this operation is attempted, an exception gets thrown.")
+  @Test
+  public void testOverrideTablePropertiesReplaceTransaction() {}
+
+  @Test
+  public void testConcurrentReplaceTransactionSchema() {
+
+    if (requiresNamespaceCreate()) {
+      catalog.createNamespace(NS);
+    }
+
+    Transaction transaction = catalog.buildTable(TABLE, OTHER_SCHEMA_OPTIONAL).createTransaction();
+    transaction.newFastAppend().appendFile(FILE_A).commit();
+    transaction.commitTransaction();
+
+    Table original = catalog.loadTable(TABLE);
+    assertFiles(original, FILE_A);
+
+    Transaction secondReplace =
+        catalog.buildTable(TABLE, OTHER_SCHEMA_OPTIONAL).replaceTransaction();
+    secondReplace.newFastAppend().appendFile(FILE_C).commit();
+
+    Transaction firstReplace = catalog.buildTable(TABLE, SCHEMA_OPTIONAL).replaceTransaction();
+    firstReplace.newFastAppend().appendFile(FILE_B).commit();
+    firstReplace.commitTransaction();
+
+    Table afterFirstReplace = catalog.loadTable(TABLE);
+    assertThat(afterFirstReplace.schema().asStruct())
+        .as("Table schema should match the new schema")
+        .isEqualTo(REPLACE_SCHEMA_OPTIONAL.asStruct());
+    assertUUIDsMatch(original, afterFirstReplace);
+    assertFiles(afterFirstReplace, FILE_B);
+
+    secondReplace.commitTransaction();
+
+    Table afterSecondReplace = catalog.loadTable(TABLE);
+    assertThat(afterSecondReplace.schema().asStruct())
+        .as("Table schema should match the original schema")
+        .isEqualTo(original.schema().asStruct());
+    assertUUIDsMatch(original, afterSecondReplace);
+    assertFiles(afterSecondReplace, FILE_C);
+  }
+
+  @Test
+  public void testCompleteCreateTransaction() {
+    if (requiresNamespaceCreate()) {
+      catalog.createNamespace(NS);
+    }
+
+    String tableLocation = warehouseLocation + "/tmp/ns/table";
+
+    Map<String, String> properties =
+        ImmutableMap.of("user", "someone", "created-at", "2022-02-25T00:38:19");
+    Transaction create =
+        catalog
+            .buildTable(TABLE, SCHEMA)
+            .withLocation(tableLocation)
+            .withPartitionSpec(SPEC)
+            .withSortOrder(WRITE_ORDER)
+            .withProperties(properties)
+            .createTransaction();
+
+    assertThat(catalog.tableExists(TABLE))
+        .as("Table should not exist after createTransaction")
+        .isFalse();
+
+    create.newFastAppend().appendFile(FILE_A).commit();
+
+    assertThat(catalog.tableExists(TABLE))
+        .as("Table should not exist after append commit")
+        .isFalse();
+
+    create.commitTransaction();
+
+    assertThat(catalog.tableExists(TABLE)).as("Table should exist after append commit").isTrue();
+
+    Table table = catalog.loadTable(TABLE);
+    assertThat(table.schema().asStruct())
+        .as("Table schema should match the new schema")
+        .isEqualTo(TABLE_SCHEMA.asStruct());
+    assertThat(table.spec().fields())
+        .as("Table should have create partition spec")
+        .isEqualTo(TABLE_SPEC.fields());
+    assertThat(table.sortOrder())
+        .as("Table should have create sort order")
+        .isEqualTo(TABLE_WRITE_ORDER);
+    assertThat(table.properties().entrySet())
+        .as("Table properties should be a superset of the requested properties")
+        .containsAll(properties.entrySet());
+    if (!overridesRequestedLocation()) {
+      assertThat(table.location())
+          .as("Table location should match requested")
+          .isEqualTo(tableLocation);
+    }
+    assertFiles(table, FILE_A);
+    assertFilesPartitionSpec(table);
+    assertPreviousMetadataFileCount(table, 0);
+  }
+
+  @Disabled(
+      "BigQuery does not allow adding a REQUIRED column to an existing table schema. "
+          + "When this operation is attempted, an exception gets thrown.")
+  @Test
+  public void testCreateOrReplaceReplaceTransactionReplace() {}
+
+  // TODO: Remove this test when BigQuery supports required field change
+  @Test
+  public void testCreateOrReplaceReplaceTransactionReplaceWithOptionalFields() {
+
+    if (requiresNamespaceCreate()) {
+      catalog.createNamespace(NS);
+    }
+
+    Table original = catalog.buildTable(TABLE, OTHER_SCHEMA_OPTIONAL).create();
+
+    assertThat(catalog.tableExists(TABLE))
+        .as("Table should exist before replaceTransaction")
+        .isTrue();
+
+    Transaction createOrReplace =
+        catalog.buildTable(TABLE, SCHEMA_OPTIONAL).createOrReplaceTransaction();
+
+    assertThat(catalog.tableExists(TABLE))
+        .as("Table should still exist after replaceTransaction")
+        .isTrue();
+
+    createOrReplace.newFastAppend().appendFile(FILE_A).commit();
+
+    // validate table has not changed
+    Table table = catalog.loadTable(TABLE);
+
+    assertThat(table.schema().asStruct())
+        .as("Table schema should match concurrent create")
+        .isEqualTo(OTHER_SCHEMA_OPTIONAL.asStruct());
+
+    assertUUIDsMatch(original, table);
+    assertNoFiles(table);
+
+    createOrReplace.commitTransaction();
+
+    // validate the table after replace
+    assertThat(catalog.tableExists(TABLE)).as("Table should exist after append commit").isTrue();
+    table.refresh(); // refresh should work with UUID validation
+
+    Table loaded = catalog.loadTable(TABLE);
+
+    assertThat(loaded.schema().asStruct())
+        .as("Table schema should match the new schema")
+        .isEqualTo(REPLACE_SCHEMA_OPTIONAL.asStruct());
+    assertUUIDsMatch(original, loaded);
+    assertFiles(loaded, FILE_A);
+    assertPreviousMetadataFileCount(loaded, 1);
+  }
+
+  @Disabled(
+      "BigQuery does not allow adding a REQUIRED column to an existing table schema. "
+          + "When this operation is attempted, an exception gets thrown.")
+  @Test
+  public void testConcurrentReplaceTransactionSchema2() {}
+
+  // TODO: Remove this test when BigQuery supports required field change
+  @Test
+  public void testConcurrentReplaceTransactionSchema2WithOptionalFields() {

Review Comment:
   I don't think any of these tests that use optional fields are needed here. The goal here is to verify all of the functionality at the catalog level



-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org

For queries about this service, please contact Infrastructure at:
users@infra.apache.org


---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For additional commands, e-mail: issues-help@iceberg.apache.org


Re: [PR] Catalog: Add BigQuery Metastore Catalog Support [iceberg]

Posted by "nastra (via GitHub)" <gi...@apache.org>.
nastra commented on code in PR #12808:
URL: https://github.com/apache/iceberg/pull/12808#discussion_r2055546190


##########
bigquery/src/test/java/org/apache/iceberg/gcp/bigquery/BigQueryTableOperationsTest.java:
##########
@@ -0,0 +1,228 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.iceberg.gcp.bigquery;
+
+import static org.apache.iceberg.gcp.bigquery.BigQueryMetastoreCatalog.PROJECT_ID;
+import static org.assertj.core.api.Assertions.assertThat;
+import static org.assertj.core.api.Assertions.assertThatThrownBy;
+import static org.mockito.Mockito.any;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.reset;
+import static org.mockito.Mockito.times;
+import static org.mockito.Mockito.verify;
+import static org.mockito.Mockito.when;
+
+import com.google.api.services.bigquery.model.Dataset;
+import com.google.api.services.bigquery.model.DatasetReference;
+import com.google.api.services.bigquery.model.ExternalCatalogDatasetOptions;
+import com.google.api.services.bigquery.model.ExternalCatalogTableOptions;
+import com.google.api.services.bigquery.model.Table;
+import com.google.api.services.bigquery.model.TableReference;
+import java.io.File;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.iceberg.CatalogProperties;
+import org.apache.iceberg.PartitionSpec;
+import org.apache.iceberg.Schema;
+import org.apache.iceberg.catalog.TableIdentifier;
+import org.apache.iceberg.exceptions.CommitFailedException;
+import org.apache.iceberg.exceptions.NoSuchTableException;
+import org.apache.iceberg.exceptions.ValidationException;
+import org.apache.iceberg.relocated.com.google.common.collect.ImmutableMap;
+import org.apache.iceberg.types.Types;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.io.TempDir;
+import org.mockito.ArgumentCaptor;
+
+public class BigQueryTableOperationsTest {
+
+  @TempDir private File tempFolder;
+
+  private static final String GCP_PROJECT = "my-project";
+  private static final String GCP_REGION = "us";
+  private static final String DATASET_ID = "db";
+  private static final String TABLE_ID = "tbl";
+  private static final TableIdentifier SPARK_TABLE_ID = TableIdentifier.of(DATASET_ID, TABLE_ID);
+
+  private static final TableReference TABLE_REFERENCE =
+      new TableReference().setProjectId(GCP_PROJECT).setDatasetId(DATASET_ID).setTableId(TABLE_ID);
+
+  private final BigQueryMetastoreClient bigQueryMetaStoreClient =
+      mock(BigQueryMetastoreClient.class);
+
+  private BigQueryMetastoreCatalog bigQueryMetastoreCatalog;
+  private BigQueryTableOperations tableOps;
+
+  @BeforeEach
+  public void before() {
+    this.bigQueryMetastoreCatalog = new BigQueryMetastoreCatalog();
+    this.bigQueryMetastoreCatalog.setConf(new Configuration());
+    String warehouseLocation = tempFolder.toPath().resolve("hive-warehouse").toString();
+
+    bigQueryMetastoreCatalog.initialize(
+        "CATALOG_ID",
+        /* properties= */ ImmutableMap.of(
+            PROJECT_ID,
+            GCP_PROJECT,
+            CatalogProperties.WAREHOUSE_LOCATION,
+            warehouseLocation,
+            CatalogProperties.FILE_IO_IMPL,
+            "org.apache.iceberg.hadoop.HadoopFileIO"),
+        GCP_PROJECT,
+        GCP_REGION,
+        bigQueryMetaStoreClient);
+    this.tableOps = (BigQueryTableOperations) bigQueryMetastoreCatalog.newTableOps(SPARK_TABLE_ID);
+  }
+
+  @Test
+  public void testDoFresh_fetchLatestMetadataFromBigQuery() throws Exception {

Review Comment:
   method names should follow Java conventions and not contain _ in their names



-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org

For queries about this service, please contact Infrastructure at:
users@infra.apache.org


---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For additional commands, e-mail: issues-help@iceberg.apache.org


Re: [PR] Catalog: Add BigQuery Metastore Catalog Support [iceberg]

Posted by "nastra (via GitHub)" <gi...@apache.org>.
nastra commented on code in PR #12808:
URL: https://github.com/apache/iceberg/pull/12808#discussion_r2055655435


##########
bigquery/src/main/java/org/apache/iceberg/gcp/bigquery/BigQueryMetastoreCatalog.java:
##########
@@ -0,0 +1,400 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.iceberg.gcp.bigquery;
+
+import com.google.api.services.bigquery.model.Dataset;
+import com.google.api.services.bigquery.model.DatasetList.Datasets;
+import com.google.api.services.bigquery.model.DatasetReference;
+import com.google.api.services.bigquery.model.ExternalCatalogDatasetOptions;
+import com.google.api.services.bigquery.model.TableReference;
+import com.google.cloud.ServiceOptions;
+import com.google.cloud.bigquery.BigQueryOptions;
+import java.io.IOException;
+import java.security.GeneralSecurityException;
+import java.util.List;
+import java.util.Locale;
+import java.util.Map;
+import java.util.Objects;
+import java.util.Set;
+import org.apache.hadoop.conf.Configurable;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.iceberg.BaseMetastoreCatalog;
+import org.apache.iceberg.CatalogProperties;
+import org.apache.iceberg.CatalogUtil;
+import org.apache.iceberg.TableMetadata;
+import org.apache.iceberg.TableOperations;
+import org.apache.iceberg.catalog.Namespace;
+import org.apache.iceberg.catalog.SupportsNamespaces;
+import org.apache.iceberg.catalog.TableIdentifier;
+import org.apache.iceberg.exceptions.AlreadyExistsException;
+import org.apache.iceberg.exceptions.NoSuchNamespaceException;
+import org.apache.iceberg.exceptions.NoSuchTableException;
+import org.apache.iceberg.exceptions.ServiceFailureException;
+import org.apache.iceberg.exceptions.ValidationException;
+import org.apache.iceberg.io.FileIO;
+import org.apache.iceberg.relocated.com.google.common.annotations.VisibleForTesting;
+import org.apache.iceberg.relocated.com.google.common.base.Preconditions;
+import org.apache.iceberg.relocated.com.google.common.base.Strings;
+import org.apache.iceberg.relocated.com.google.common.collect.ImmutableList;
+import org.apache.iceberg.relocated.com.google.common.collect.ImmutableMap;
+import org.apache.iceberg.relocated.com.google.common.collect.Maps;
+import org.apache.iceberg.util.LocationUtil;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/** Iceberg Bigquery Metastore Catalog implementation. */
+public final class BigQueryMetastoreCatalog extends BaseMetastoreCatalog
+    implements SupportsNamespaces, Configurable {
+
+  // User provided properties.
+  public static final String PROJECT_ID = "gcp-project";
+  public static final String GCP_LOCATION = "gcp-location";
+  public static final String FILTER_UNSUPPORTED_TABLES = "filter-unsupported-tables";
+  public static final String TESTING_ENABLED = "testing-enabled";
+
+  public static final String HIVE_METASTORE_WAREHOUSE_DIR = "hive.metastore.warehouse.dir";
+
+  private static final Logger LOG = LoggerFactory.getLogger(BigQueryMetastoreCatalog.class);
+
+  private static final String DEFAULT_GCP_LOCATION = "us";
+
+  private String catalogName;
+  private Map<String, String> catalogProperties;
+  private FileIO fileIO;
+  private Configuration conf;
+  private String projectId;
+  private String location;
+  private BigQueryMetastoreClient client;
+  private boolean filterUnsupportedTables;
+
+  // Must have a no-arg constructor to be dynamically loaded
+  // initialize(String name, Map<String, String> properties) will be called to complete
+  // initialization
+  public BigQueryMetastoreCatalog() {}
+
+  @Override
+  public void initialize(String inputName, Map<String, String> properties) {
+    Preconditions.checkArgument(
+        properties.containsKey(PROJECT_ID), "GCP project must be specified");
+
+    projectId = properties.get(PROJECT_ID);
+    location = properties.getOrDefault(GCP_LOCATION, DEFAULT_GCP_LOCATION);
+    boolean testingEnabled =
+        Boolean.parseBoolean(properties.getOrDefault(TESTING_ENABLED, "false"));
+
+    BigQueryOptions options =
+        BigQueryOptions.newBuilder()
+            .setProjectId(projectId)
+            .setLocation(location)
+            .setRetrySettings(ServiceOptions.getDefaultRetrySettings())
+            .build();
+
+    try {
+      if (testingEnabled) {
+        client = new FakeBigQueryMetastoreClient(options);
+
+      } else {
+        client = new BigQueryMetastoreClientImpl(options);
+      }
+    } catch (IOException e) {
+      throw new ServiceFailureException(e, "Creating BigQuery client failed");
+    } catch (GeneralSecurityException e) {
+      throw new ValidationException(e, "Creating BigQuery client failed due to a security issue");
+    }
+    initialize(inputName, properties, projectId, location, client);
+  }
+
+  @VisibleForTesting
+  void initialize(
+      String inputName,
+      Map<String, String> properties,
+      String initialProjectId,
+      String initialLocation,
+      BigQueryMetastoreClient bigQueryMetaStoreClient) {
+    Preconditions.checkArgument(bigQueryMetaStoreClient != null, "BigQuery client can not be null");
+    this.catalogName = inputName;
+    this.catalogProperties = ImmutableMap.copyOf(properties);
+    this.projectId = initialProjectId;
+    this.location = initialLocation;
+    this.client = bigQueryMetaStoreClient;
+
+    if (this.conf == null) {
+      LOG.warn("No configuration was set, using the default environment Configuration");
+      this.conf = new Configuration();
+    }
+
+    LOG.info("Using BigQuery Metastore Iceberg Catalog: {}", inputName);
+
+    if (properties.containsKey(CatalogProperties.WAREHOUSE_LOCATION)) {
+      // Iceberg always removes trailing slash to avoid paths like "<folder>//data" in file systems
+      // like s3.
+      this.conf.set(
+          HIVE_METASTORE_WAREHOUSE_DIR,
+          LocationUtil.stripTrailingSlash(properties.get(CatalogProperties.WAREHOUSE_LOCATION)));
+    }
+
+    String fileIoImpl =
+        properties.getOrDefault(
+            CatalogProperties.FILE_IO_IMPL, "org.apache.iceberg.hadoop.HadoopFileIO");
+    this.fileIO = CatalogUtil.loadFileIO(fileIoImpl, properties, conf);
+
+    this.filterUnsupportedTables =
+        Boolean.parseBoolean(properties.getOrDefault(FILTER_UNSUPPORTED_TABLES, "false"));
+  }
+
+  @Override
+  protected TableOperations newTableOps(TableIdentifier identifier) {
+    return new BigQueryTableOperations(
+        client,
+        fileIO,
+        projectId,
+        // Sometimes extensions have the namespace contain the table name too, so we are forced to
+        // allow invalid namespace and just take the first part here like other catalog
+        // implementations do.
+        identifier.namespace().level(0),
+        identifier.name(),
+        conf);
+  }
+
+  @Override
+  protected String defaultWarehouseLocation(TableIdentifier identifier) {
+    String locationUri = null;
+    DatasetReference datasetReference = toDatasetReference(identifier.namespace());
+    Dataset dataset = client.getDataset(datasetReference);
+    if (dataset != null && dataset.getExternalCatalogDatasetOptions() != null) {
+      locationUri = dataset.getExternalCatalogDatasetOptions().getDefaultStorageLocationUri();
+    }
+    return String.format(
+        "%s/%s",
+        Strings.isNullOrEmpty(locationUri)
+            ? datasetReference.getDatasetId()
+            : LocationUtil.stripTrailingSlash(locationUri),
+        identifier.name());
+  }
+
+  @Override
+  public List<TableIdentifier> listTables(Namespace namespace) {
+    validateNamespace(namespace);
+
+    return client.listTables(toDatasetReference(namespace), filterUnsupportedTables).stream()
+        .map(
+            table -> TableIdentifier.of(namespace.level(0), table.getTableReference().getTableId()))
+        .collect(ImmutableList.toImmutableList());
+  }
+
+  @Override
+  public boolean dropTable(TableIdentifier identifier, boolean purge) {
+    try {
+      TableOperations ops = newTableOps(identifier);
+      TableMetadata lastMetadata = ops.current();
+
+      client.deleteTable(toTableReference(identifier));
+
+      if (purge && lastMetadata != null) {
+        CatalogUtil.dropTableData(ops.io(), lastMetadata);
+      }
+    } catch (NoSuchTableException e) { // Not catching a NoSuchIcebergTableException on purpose
+      return false; // The documentation says just return false in this case
+    }
+    return true;
+  }
+
+  @Override
+  public void renameTable(TableIdentifier from, TableIdentifier to) {
+    validateNamespace(from.namespace());
+    validateNamespace(to.namespace());
+
+    try {
+      client.getDataset(toDatasetReference(to.namespace()));
+      client.getDataset(toDatasetReference(from.namespace()));
+    } catch (NoSuchNamespaceException e) {
+      throw e;
+    }
+
+    TableReference fromTableRef = toTableReference(from);
+    TableReference toTableRef = toTableReference(to);
+
+    // Check if the source table exists

Review Comment:
   why not check this via `tableExists()`?



-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org

For queries about this service, please contact Infrastructure at:
users@infra.apache.org


---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For additional commands, e-mail: issues-help@iceberg.apache.org


Re: [PR] Catalog: Add BigQuery Metastore Catalog Support [iceberg]

Posted by "talatuyarer (via GitHub)" <gi...@apache.org>.
talatuyarer commented on code in PR #12808:
URL: https://github.com/apache/iceberg/pull/12808#discussion_r2085097917


##########
bigquery/src/main/java/org/apache/iceberg/gcp/bigquery/BigQueryMetastoreCatalog.java:
##########
@@ -0,0 +1,366 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.iceberg.gcp.bigquery;
+
+import com.google.api.services.bigquery.model.Dataset;
+import com.google.api.services.bigquery.model.DatasetList.Datasets;
+import com.google.api.services.bigquery.model.DatasetReference;
+import com.google.api.services.bigquery.model.ExternalCatalogDatasetOptions;
+import com.google.api.services.bigquery.model.TableReference;
+import com.google.cloud.ServiceOptions;
+import com.google.cloud.bigquery.BigQueryOptions;
+import java.io.IOException;
+import java.io.UncheckedIOException;
+import java.security.GeneralSecurityException;
+import java.util.List;
+import java.util.Locale;
+import java.util.Map;
+import java.util.Objects;
+import java.util.Set;
+import org.apache.hadoop.conf.Configurable;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.iceberg.BaseMetastoreCatalog;
+import org.apache.iceberg.CatalogProperties;
+import org.apache.iceberg.CatalogUtil;
+import org.apache.iceberg.TableMetadata;
+import org.apache.iceberg.TableOperations;
+import org.apache.iceberg.catalog.Namespace;
+import org.apache.iceberg.catalog.SupportsNamespaces;
+import org.apache.iceberg.catalog.TableIdentifier;
+import org.apache.iceberg.exceptions.NoSuchNamespaceException;
+import org.apache.iceberg.exceptions.NoSuchTableException;
+import org.apache.iceberg.io.FileIO;
+import org.apache.iceberg.relocated.com.google.common.annotations.VisibleForTesting;
+import org.apache.iceberg.relocated.com.google.common.base.Preconditions;
+import org.apache.iceberg.relocated.com.google.common.base.Strings;
+import org.apache.iceberg.relocated.com.google.common.collect.ImmutableList;
+import org.apache.iceberg.relocated.com.google.common.collect.ImmutableMap;
+import org.apache.iceberg.relocated.com.google.common.collect.Maps;
+import org.apache.iceberg.util.LocationUtil;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/** Iceberg Bigquery Metastore Catalog implementation. */
+public class BigQueryMetastoreCatalog extends BaseMetastoreCatalog
+    implements SupportsNamespaces, Configurable {
+
+  // User provided properties.
+  public static final String PROJECT_ID = "gcp.bigquery.project-id";
+  public static final String GCP_LOCATION = "gcp.bigquery.location";
+  public static final String LIST_ALL_TABLES = "gcp.bigquery.list-all-tables";
+
+  private static final Logger LOG = LoggerFactory.getLogger(BigQueryMetastoreCatalog.class);
+
+  private static final String DEFAULT_GCP_LOCATION = "us";
+
+  private String catalogName;
+  private Map<String, String> catalogProperties;
+  private FileIO fileIO;
+  private Configuration conf;
+  private String projectId;
+  private String projectLocation;
+  private BigQueryMetastoreClient client;
+  private boolean listAllTables;
+  private String warehouseLocation;
+
+  public BigQueryMetastoreCatalog() {}
+
+  @Override
+  public void initialize(String name, Map<String, String> properties) {
+    Preconditions.checkArgument(
+        properties.containsKey(PROJECT_ID),
+        "Invalid GCP project: %s must be specified",
+        PROJECT_ID);
+
+    this.projectId = properties.get(PROJECT_ID);
+    this.projectLocation = properties.getOrDefault(GCP_LOCATION, DEFAULT_GCP_LOCATION);
+
+    BigQueryOptions options =
+        BigQueryOptions.newBuilder()
+            .setProjectId(projectId)
+            .setLocation(projectLocation)
+            .setRetrySettings(ServiceOptions.getDefaultRetrySettings())
+            .build();
+
+    try {
+      client = new BigQueryMetastoreClientImpl(options);
+    } catch (IOException e) {
+      throw new UncheckedIOException("Creating BigQuery client failed", e);
+    } catch (GeneralSecurityException e) {
+      throw new RuntimeException("Creating BigQuery client failed due to a security issue", e);
+    }
+
+    initialize(name, properties, projectId, projectLocation, client);
+  }
+
+  @VisibleForTesting
+  void initialize(
+      String name,
+      Map<String, String> properties,
+      String initialProjectId,
+      String initialLocation,
+      BigQueryMetastoreClient bigQueryMetaStoreClient) {
+    Preconditions.checkArgument(bigQueryMetaStoreClient != null, "Invalid BigQuery client: null");
+    this.catalogName = name;
+    this.catalogProperties = ImmutableMap.copyOf(properties);
+    this.projectId = initialProjectId;
+    this.projectLocation = initialLocation;
+    this.client = bigQueryMetaStoreClient;
+
+    if (this.conf == null) {
+      LOG.warn("No configuration was set, using the default environment Configuration");
+      this.conf = new Configuration();
+    }
+
+    LOG.info("Using BigQuery Metastore Iceberg Catalog: {}", name);
+
+    if (properties.containsKey(CatalogProperties.WAREHOUSE_LOCATION)) {
+      this.warehouseLocation =
+          LocationUtil.stripTrailingSlash(properties.get(CatalogProperties.WAREHOUSE_LOCATION));
+    }
+
+    this.fileIO =
+        CatalogUtil.loadFileIO(
+            properties.getOrDefault(
+                CatalogProperties.FILE_IO_IMPL, "org.apache.iceberg.io.ResolvingFileIO"),
+            properties,
+            conf);
+
+    this.listAllTables = Boolean.parseBoolean(properties.getOrDefault(LIST_ALL_TABLES, "true"));
+  }
+
+  @Override
+  protected TableOperations newTableOps(TableIdentifier identifier) {
+    return new BigQueryTableOperations(client, fileIO, toTableReference(identifier), conf);
+  }
+
+  @Override
+  protected String defaultWarehouseLocation(TableIdentifier identifier) {
+    String locationUri = null;
+    DatasetReference datasetReference = toDatasetReference(identifier.namespace());
+    Dataset dataset = client.load(datasetReference);
+    if (dataset != null && dataset.getExternalCatalogDatasetOptions() != null) {
+      locationUri = dataset.getExternalCatalogDatasetOptions().getDefaultStorageLocationUri();
+    }
+
+    return String.format(
+        "%s/%s",
+        Strings.isNullOrEmpty(locationUri)
+            ? createDefaultStorageLocationUri(datasetReference.getDatasetId())
+            : LocationUtil.stripTrailingSlash(locationUri),
+        identifier.name());
+  }
+
+  @Override
+  public List<TableIdentifier> listTables(Namespace namespace) {
+    validateNamespace(namespace);
+
+    return client.list(toDatasetReference(namespace), listAllTables).stream()
+        .map(
+            table -> TableIdentifier.of(namespace.level(0), table.getTableReference().getTableId()))
+        .collect(ImmutableList.toImmutableList());
+  }
+
+  @Override
+  public boolean dropTable(TableIdentifier identifier, boolean purge) {
+    try {
+      TableOperations ops = newTableOps(identifier);
+      TableMetadata lastMetadata = ops.current();
+
+      client.delete(toTableReference(identifier));
+
+      if (purge && lastMetadata != null) {
+        CatalogUtil.dropTableData(ops.io(), lastMetadata);
+      }
+    } catch (NoSuchTableException e) {
+      return false;
+    }
+
+    return true;
+  }
+
+  @Override
+  public void renameTable(TableIdentifier from, TableIdentifier to) {
+    // TODO: Enable once supported by BigQuery API.
+    throw new UnsupportedOperationException("Table rename operation is unsupported.");
+  }
+
+  @Override
+  public void createNamespace(Namespace namespace, Map<String, String> metadata) {
+    Dataset builder = new Dataset();
+    DatasetReference datasetReference = toDatasetReference(namespace);
+    builder.setLocation(this.projectLocation);
+    builder.setDatasetReference(datasetReference);
+    builder.setExternalCatalogDatasetOptions(
+        BigQueryMetastoreUtils.createExternalCatalogDatasetOptions(
+            createDefaultStorageLocationUri(datasetReference.getDatasetId()), metadata));
+
+    client.create(builder);
+  }
+
+  @Override
+  public List<Namespace> listNamespaces() {
+    try {
+      return listNamespaces(Namespace.empty());
+    } catch (NoSuchNamespaceException e) {
+      return ImmutableList.of();
+    }
+  }
+
+  /**
+   * Since this catalog only supports one-level namespaces, it always returns an empty list unless
+   * passed an empty namespace to list all namespaces within the catalog.
+   */
+  @Override
+  public List<Namespace> listNamespaces(Namespace namespace) {
+    List<Datasets> allDatasets = client.list(projectId);
+
+    ImmutableList<Namespace> namespaces =
+        allDatasets.stream().map(this::toNamespace).collect(ImmutableList.toImmutableList());
+    if (namespaces.isEmpty()) {
+      throw new NoSuchNamespaceException("Namespace does not exist: %s", namespace);
+    }
+
+    return namespaces;
+  }
+
+  @Override
+  public boolean dropNamespace(Namespace namespace) {
+    try {
+      client.delete(toDatasetReference(namespace));
+      // We don't delete the data folder for safety, which aligns with Hive Metastore's default
+      // behavior.
+      // We can support database or catalog level config controlling file deletion in the future.
+      return true;
+    } catch (NoSuchNamespaceException e) {
+      return false;
+    }
+  }
+
+  @Override
+  public boolean setProperties(Namespace namespace, Map<String, String> properties) {
+    Dataset dataset = client.load(toDatasetReference(namespace));
+
+    ExternalCatalogDatasetOptions existingOptions = dataset.getExternalCatalogDatasetOptions();
+    Map<String, String> existingParameters =
+        existingOptions != null ? existingOptions.getParameters() : null;
+
+    Map<String, String> newParameters = Maps.newHashMap();
+    if (existingParameters != null) {
+      newParameters.putAll(existingParameters);
+    }
+
+    newParameters.putAll(properties);
+
+    if (Objects.equals(existingParameters, newParameters)) {
+      // No change in parameters detected
+      return false;
+    }
+
+    client.setParameters(toDatasetReference(namespace), properties);

Review Comment:
   Yeah it was. I moved the "load, compare, conditionally-update" logic from the `BigQueryMetastoreCatalog` into the `BigQueryMetastoreClient`. 



-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org

For queries about this service, please contact Infrastructure at:
users@infra.apache.org


---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For additional commands, e-mail: issues-help@iceberg.apache.org


Re: [PR] Catalog: Add BigQuery Metastore Catalog Support [iceberg]

Posted by "danielcweeks (via GitHub)" <gi...@apache.org>.
danielcweeks commented on code in PR #12808:
URL: https://github.com/apache/iceberg/pull/12808#discussion_r2071899340


##########
bigquery/src/main/java/org/apache/iceberg/gcp/bigquery/BigQueryMetastoreClientImpl.java:
##########
@@ -0,0 +1,581 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.iceberg.gcp.bigquery;
+
+import com.google.api.client.googleapis.javanet.GoogleNetHttpTransport;
+import com.google.api.client.http.HttpHeaders;
+import com.google.api.client.http.HttpResponse;
+import com.google.api.client.http.HttpResponseException;
+import com.google.api.client.http.HttpStatusCodes;
+import com.google.api.client.json.gson.GsonFactory;
+import com.google.api.client.util.Data;
+import com.google.api.services.bigquery.Bigquery;
+import com.google.api.services.bigquery.BigqueryScopes;
+import com.google.api.services.bigquery.model.Dataset;
+import com.google.api.services.bigquery.model.DatasetList;
+import com.google.api.services.bigquery.model.DatasetList.Datasets;
+import com.google.api.services.bigquery.model.DatasetReference;
+import com.google.api.services.bigquery.model.ExternalCatalogDatasetOptions;
+import com.google.api.services.bigquery.model.ExternalCatalogTableOptions;
+import com.google.api.services.bigquery.model.Table;
+import com.google.api.services.bigquery.model.TableList;
+import com.google.api.services.bigquery.model.TableList.Tables;
+import com.google.api.services.bigquery.model.TableReference;
+import com.google.api.services.bigquery.model.TableSchema;
+import com.google.auth.http.HttpCredentialsAdapter;
+import com.google.auth.oauth2.GoogleCredentials;
+import com.google.cloud.BaseServiceException;
+import com.google.cloud.ExceptionHandler;
+import com.google.cloud.bigquery.BigQueryErrorMessages;
+import com.google.cloud.bigquery.BigQueryOptions;
+import com.google.cloud.bigquery.BigQueryRetryConfig;
+import com.google.cloud.bigquery.BigQueryRetryHelper;
+import java.io.IOException;
+import java.nio.charset.StandardCharsets;
+import java.security.GeneralSecurityException;
+import java.util.List;
+import java.util.Locale;
+import java.util.Map;
+import java.util.Set;
+import java.util.stream.Collectors;
+import java.util.stream.Stream;
+import org.apache.iceberg.BaseMetastoreTableOperations;
+import org.apache.iceberg.exceptions.AlreadyExistsException;
+import org.apache.iceberg.exceptions.BadRequestException;
+import org.apache.iceberg.exceptions.ForbiddenException;
+import org.apache.iceberg.exceptions.NoSuchIcebergTableException;
+import org.apache.iceberg.exceptions.NoSuchNamespaceException;
+import org.apache.iceberg.exceptions.NoSuchTableException;
+import org.apache.iceberg.exceptions.NotAuthorizedException;
+import org.apache.iceberg.exceptions.RuntimeIOException;
+import org.apache.iceberg.exceptions.ServiceFailureException;
+import org.apache.iceberg.exceptions.ServiceUnavailableException;
+import org.apache.iceberg.exceptions.ValidationException;
+import org.apache.iceberg.relocated.com.google.common.base.Preconditions;
+import org.apache.iceberg.relocated.com.google.common.collect.Lists;
+import org.apache.iceberg.relocated.com.google.common.collect.Maps;
+
+/** A client of Google Bigquery Metastore functions over the BigQuery service. */
+public final class BigQueryMetastoreClientImpl implements BigQueryMetastoreClient {
+
+  private final Bigquery client;
+  private final BigQueryOptions bigqueryOptions;
+
+  public static final ExceptionHandler.Interceptor EXCEPTION_HANDLER_INTERCEPTOR =
+      new ExceptionHandler.Interceptor() {
+
+        private static final long serialVersionUID = -8429573486870467828L;

Review Comment:
   Why do we need a serialVersionUID?  I don't think this gets serialized and we really should only be concerned about serializable version compatibility if it's necessary.



-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org

For queries about this service, please contact Infrastructure at:
users@infra.apache.org


---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For additional commands, e-mail: issues-help@iceberg.apache.org


Re: [PR] Catalog: Add BigQuery Metastore Catalog Support [iceberg]

Posted by "nastra (via GitHub)" <gi...@apache.org>.
nastra commented on code in PR #12808:
URL: https://github.com/apache/iceberg/pull/12808#discussion_r2066100812


##########
bigquery/src/test/java/org/apache/iceberg/gcp/bigquery/BigQueryTableOperationsTest.java:
##########
@@ -0,0 +1,274 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.iceberg.gcp.bigquery;
+
+import static org.apache.iceberg.gcp.bigquery.BigQueryMetastoreCatalog.PROJECT_ID;
+import static org.apache.iceberg.types.Types.NestedField.required;
+import static org.assertj.core.api.Assertions.assertThat;
+import static org.assertj.core.api.Assertions.assertThatThrownBy;
+import static org.mockito.Mockito.any;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.reset;
+import static org.mockito.Mockito.times;
+import static org.mockito.Mockito.verify;
+import static org.mockito.Mockito.when;
+
+import com.google.api.services.bigquery.model.Dataset;
+import com.google.api.services.bigquery.model.DatasetReference;
+import com.google.api.services.bigquery.model.ExternalCatalogDatasetOptions;
+import com.google.api.services.bigquery.model.ExternalCatalogTableOptions;
+import com.google.api.services.bigquery.model.StorageDescriptor;
+import com.google.api.services.bigquery.model.Table;
+import com.google.api.services.bigquery.model.TableReference;
+import java.io.File;
+import java.io.IOException;
+import java.util.Collections;
+import java.util.Optional;
+import org.apache.commons.io.FileUtils;
+import org.apache.commons.io.filefilter.TrueFileFilter;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.iceberg.CatalogProperties;
+import org.apache.iceberg.PartitionSpec;
+import org.apache.iceberg.Schema;
+import org.apache.iceberg.catalog.TableIdentifier;
+import org.apache.iceberg.exceptions.CommitFailedException;
+import org.apache.iceberg.exceptions.NoSuchTableException;
+import org.apache.iceberg.exceptions.ValidationException;
+import org.apache.iceberg.relocated.com.google.common.collect.ImmutableMap;
+import org.apache.iceberg.types.Types;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.io.TempDir;
+import org.mockito.ArgumentCaptor;
+
+public class BigQueryTableOperationsTest {
+
+  @TempDir private File tempFolder;
+  public static final String METADATA_LOCATION_PROP = "metadata_location";
+  private static final String GCP_PROJECT = "my-project";
+  private static final String GCP_REGION = "us";
+  private static final String DATASET_ID = "db";
+  private static final String TABLE_ID = "tbl";
+  private static final TableIdentifier SPARK_TABLE_ID = TableIdentifier.of(DATASET_ID, TABLE_ID);
+
+  private static final TableReference TABLE_REFERENCE =
+      new TableReference().setProjectId(GCP_PROJECT).setDatasetId(DATASET_ID).setTableId(TABLE_ID);
+
+  private final BigQueryMetastoreClient bigQueryMetaStoreClient =
+      mock(BigQueryMetastoreClient.class);
+
+  private BigQueryMetastoreCatalog bigQueryMetastoreCatalog;
+  private BigQueryTableOperations tableOps;
+
+  @BeforeEach
+  public void before() {
+    this.bigQueryMetastoreCatalog = new BigQueryMetastoreCatalog();
+    this.bigQueryMetastoreCatalog.setConf(new Configuration());
+    String warehouseLocation = tempFolder.toPath().resolve("hive-warehouse").toString();
+
+    bigQueryMetastoreCatalog.initialize(
+        "CATALOG_ID",
+        ImmutableMap.of(
+            PROJECT_ID,
+            GCP_PROJECT,
+            CatalogProperties.WAREHOUSE_LOCATION,
+            warehouseLocation,
+            CatalogProperties.FILE_IO_IMPL,
+            "org.apache.iceberg.hadoop.HadoopFileIO"),
+        GCP_PROJECT,
+        GCP_REGION,
+        bigQueryMetaStoreClient);
+    this.tableOps = (BigQueryTableOperations) bigQueryMetastoreCatalog.newTableOps(SPARK_TABLE_ID);
+  }
+
+  @Test
+  public void testFetchLatestMetadataFromBigQuery() throws Exception {
+    Table createdTable = createTestTable();
+    reset(bigQueryMetaStoreClient);
+    when(bigQueryMetaStoreClient.load(TABLE_REFERENCE)).thenReturn(createdTable);
+
+    tableOps.refresh();
+    assertThat(
+            createdTable
+                .getExternalCatalogTableOptions()
+                .getParameters()
+                .getOrDefault(METADATA_LOCATION_PROP, ""))
+        .isEqualTo(tableOps.currentMetadataLocation());
+
+    reset(bigQueryMetaStoreClient);
+    when(bigQueryMetaStoreClient.load(TABLE_REFERENCE))
+        .thenThrow(new NoSuchTableException("error message getTable"));
+    // Refresh fails when table is not found but metadata already presents.
+    assertThatThrownBy(() -> tableOps.refresh())
+        .isInstanceOf(NoSuchTableException.class)
+        .hasMessageContaining("error message getTable");
+  }
+
+  @Test
+  public void testFailForNonIcebergTable() {
+    when(bigQueryMetaStoreClient.load(TABLE_REFERENCE))
+        .thenReturn(new Table().setTableReference(TABLE_REFERENCE));
+
+    assertThatThrownBy(() -> tableOps.refresh())
+        .isInstanceOf(ValidationException.class)
+        .hasMessageContaining("metadata location not found");
+  }
+
+  @Test
+  public void testNoOpWhenMetadataAndTableNotFound() {
+    when(bigQueryMetaStoreClient.load(TABLE_REFERENCE))
+        .thenThrow(new NoSuchTableException("error message getTable"));
+    // Table not found won't cause errors when the metadata is null.
+    assertThat(tableOps.currentMetadataLocation()).isNull();
+    tableOps.refresh();
+  }
+
+  @Test
+  public void testTableNameAsExpected() {
+    assertThat(tableOps.tableName()).isEqualTo("db.tbl");
+  }
+
+  @Test
+  public void testUseEtagForUpdateTable() throws Exception {
+    Table tableWithEtag = createTestTable().setEtag("etag");
+    reset(bigQueryMetaStoreClient);
+    when(bigQueryMetaStoreClient.load(TABLE_REFERENCE)).thenReturn(tableWithEtag, tableWithEtag);
+
+    org.apache.iceberg.Table loadedTable = bigQueryMetastoreCatalog.loadTable(SPARK_TABLE_ID);
+
+    when(bigQueryMetaStoreClient.update(any(), any())).thenReturn(tableWithEtag);
+    loadedTable.updateSchema().addColumn("n", Types.IntegerType.get()).commit();
+
+    ArgumentCaptor<TableReference> tableReferenceArgumentCaptor =
+        ArgumentCaptor.forClass(TableReference.class);
+    ArgumentCaptor<Table> tableArgumentCaptor = ArgumentCaptor.forClass(Table.class);
+    verify(bigQueryMetaStoreClient, times(1))
+        .update(tableReferenceArgumentCaptor.capture(), tableArgumentCaptor.capture());
+    assertThat(tableReferenceArgumentCaptor.getValue()).isEqualTo(TABLE_REFERENCE);
+    assertThat(tableArgumentCaptor.getValue().getEtag()).isEqualTo("etag");
+  }
+
+  @Test
+  public void testFailWhenEtagMismatch() throws Exception {
+    Table tableWithEtag = createTestTable().setEtag("etag");
+    reset(bigQueryMetaStoreClient);
+    when(bigQueryMetaStoreClient.load(TABLE_REFERENCE)).thenReturn(tableWithEtag, tableWithEtag);

Review Comment:
   ```suggestion
       when(bigQueryMetaStoreClient.load(TABLE_REFERENCE)).thenReturn(tableWithEtag);
   ```



-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org

For queries about this service, please contact Infrastructure at:
users@infra.apache.org


---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For additional commands, e-mail: issues-help@iceberg.apache.org


Re: [PR] Catalog: Add BigQuery Metastore Catalog Support [iceberg]

Posted by "talatuyarer (via GitHub)" <gi...@apache.org>.
talatuyarer commented on code in PR #12808:
URL: https://github.com/apache/iceberg/pull/12808#discussion_r2081045506


##########
bigquery/src/main/java/org/apache/iceberg/gcp/bigquery/BigQueryMetastoreClientImpl.java:
##########
@@ -0,0 +1,579 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.iceberg.gcp.bigquery;
+
+import com.google.api.client.googleapis.javanet.GoogleNetHttpTransport;
+import com.google.api.client.http.HttpHeaders;
+import com.google.api.client.http.HttpResponse;
+import com.google.api.client.http.HttpResponseException;
+import com.google.api.client.http.HttpStatusCodes;
+import com.google.api.client.json.gson.GsonFactory;
+import com.google.api.client.util.Data;
+import com.google.api.services.bigquery.Bigquery;
+import com.google.api.services.bigquery.BigqueryScopes;
+import com.google.api.services.bigquery.model.Dataset;
+import com.google.api.services.bigquery.model.DatasetList;
+import com.google.api.services.bigquery.model.DatasetList.Datasets;
+import com.google.api.services.bigquery.model.DatasetReference;
+import com.google.api.services.bigquery.model.ExternalCatalogDatasetOptions;
+import com.google.api.services.bigquery.model.ExternalCatalogTableOptions;
+import com.google.api.services.bigquery.model.Table;
+import com.google.api.services.bigquery.model.TableList;
+import com.google.api.services.bigquery.model.TableList.Tables;
+import com.google.api.services.bigquery.model.TableReference;
+import com.google.api.services.bigquery.model.TableSchema;
+import com.google.auth.http.HttpCredentialsAdapter;
+import com.google.auth.oauth2.GoogleCredentials;
+import com.google.cloud.BaseServiceException;
+import com.google.cloud.ExceptionHandler;
+import com.google.cloud.bigquery.BigQueryErrorMessages;
+import com.google.cloud.bigquery.BigQueryOptions;
+import com.google.cloud.bigquery.BigQueryRetryConfig;
+import com.google.cloud.bigquery.BigQueryRetryHelper;
+import java.io.IOException;
+import java.nio.charset.StandardCharsets;
+import java.security.GeneralSecurityException;
+import java.util.List;
+import java.util.Locale;
+import java.util.Map;
+import java.util.Set;
+import java.util.stream.Collectors;
+import java.util.stream.Stream;
+import org.apache.iceberg.BaseMetastoreTableOperations;
+import org.apache.iceberg.exceptions.AlreadyExistsException;
+import org.apache.iceberg.exceptions.BadRequestException;
+import org.apache.iceberg.exceptions.ForbiddenException;
+import org.apache.iceberg.exceptions.NoSuchIcebergTableException;
+import org.apache.iceberg.exceptions.NoSuchNamespaceException;
+import org.apache.iceberg.exceptions.NoSuchTableException;
+import org.apache.iceberg.exceptions.NotAuthorizedException;
+import org.apache.iceberg.exceptions.RuntimeIOException;
+import org.apache.iceberg.exceptions.ServiceFailureException;
+import org.apache.iceberg.exceptions.ServiceUnavailableException;
+import org.apache.iceberg.exceptions.ValidationException;
+import org.apache.iceberg.relocated.com.google.common.base.Preconditions;
+import org.apache.iceberg.relocated.com.google.common.collect.Lists;
+import org.apache.iceberg.relocated.com.google.common.collect.Maps;
+
+/** A client of Google Bigquery Metastore functions over the BigQuery service. */
+public final class BigQueryMetastoreClientImpl implements BigQueryMetastoreClient {
+
+  private final Bigquery client;
+  private final BigQueryOptions bigqueryOptions;
+
+  public static final ExceptionHandler.Interceptor EXCEPTION_HANDLER_INTERCEPTOR =
+      new ExceptionHandler.Interceptor() {
+
+        @Override
+        public RetryResult afterEval(Exception exception, RetryResult retryResult) {
+          return ExceptionHandler.Interceptor.RetryResult.CONTINUE_EVALUATION;
+        }
+
+        @Override
+        public RetryResult beforeEval(Exception exception) {
+          if (exception instanceof BaseServiceException) {
+            boolean retriable = ((BaseServiceException) exception).isRetryable();
+            return retriable
+                ? ExceptionHandler.Interceptor.RetryResult.RETRY
+                : ExceptionHandler.Interceptor.RetryResult.CONTINUE_EVALUATION;
+          }
+
+          return ExceptionHandler.Interceptor.RetryResult.CONTINUE_EVALUATION;
+        }
+      };
+
+  // Retry config with error messages and regex for rate limit exceeded errors.
+  private static final BigQueryRetryConfig DEFAULT_RETRY_CONFIG =
+      BigQueryRetryConfig.newBuilder()
+          .retryOnMessage(BigQueryErrorMessages.RATE_LIMIT_EXCEEDED_MSG)
+          .retryOnMessage(BigQueryErrorMessages.JOB_RATE_LIMIT_EXCEEDED_MSG)
+          .retryOnRegEx(BigQueryErrorMessages.RetryRegExPatterns.RATE_LIMIT_EXCEEDED_REGEX)
+          .build();
+
+  public static final ExceptionHandler BIGQUERY_EXCEPTION_HANDLER =
+      ExceptionHandler.newBuilder()
+          .abortOn(RuntimeException.class)
+          // Retry on connection failures due to transient network issues.
+          .retryOn(java.net.ConnectException.class)
+          // Retry to recover from temporary DNS resolution failures.
+          .retryOn(java.net.UnknownHostException.class)
+          .addInterceptors(EXCEPTION_HANDLER_INTERCEPTOR)
+          .build();
+
+  /** Constructs a client of the Google BigQuery service. */
+  public BigQueryMetastoreClientImpl(BigQueryOptions options)
+      throws IOException, GeneralSecurityException {
+    // Initialize client that will be used to send requests. This client only needs to be created
+    // once, and can be reused for multiple requests
+    HttpCredentialsAdapter httpCredentialsAdapter =
+        new HttpCredentialsAdapter(
+            GoogleCredentials.getApplicationDefault().createScoped(BigqueryScopes.all()));

Review Comment:
   Thank you @ebyhr Good catch! If it is not very critical for you. I would like to address that support after merging this. I already received so much comments on this PR. I want to merge this first. This PR will cover most of use cases and also provide main basic functionality for GCP users.



-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org

For queries about this service, please contact Infrastructure at:
users@infra.apache.org


---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For additional commands, e-mail: issues-help@iceberg.apache.org


Re: [PR] Catalog: Add BigQuery Metastore Catalog Support [iceberg]

Posted by "nastra (via GitHub)" <gi...@apache.org>.
nastra commented on code in PR #12808:
URL: https://github.com/apache/iceberg/pull/12808#discussion_r2055650817


##########
bigquery/src/main/java/org/apache/iceberg/gcp/bigquery/BigQueryMetastoreCatalog.java:
##########
@@ -0,0 +1,400 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.iceberg.gcp.bigquery;
+
+import com.google.api.services.bigquery.model.Dataset;
+import com.google.api.services.bigquery.model.DatasetList.Datasets;
+import com.google.api.services.bigquery.model.DatasetReference;
+import com.google.api.services.bigquery.model.ExternalCatalogDatasetOptions;
+import com.google.api.services.bigquery.model.TableReference;
+import com.google.cloud.ServiceOptions;
+import com.google.cloud.bigquery.BigQueryOptions;
+import java.io.IOException;
+import java.security.GeneralSecurityException;
+import java.util.List;
+import java.util.Locale;
+import java.util.Map;
+import java.util.Objects;
+import java.util.Set;
+import org.apache.hadoop.conf.Configurable;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.iceberg.BaseMetastoreCatalog;
+import org.apache.iceberg.CatalogProperties;
+import org.apache.iceberg.CatalogUtil;
+import org.apache.iceberg.TableMetadata;
+import org.apache.iceberg.TableOperations;
+import org.apache.iceberg.catalog.Namespace;
+import org.apache.iceberg.catalog.SupportsNamespaces;
+import org.apache.iceberg.catalog.TableIdentifier;
+import org.apache.iceberg.exceptions.AlreadyExistsException;
+import org.apache.iceberg.exceptions.NoSuchNamespaceException;
+import org.apache.iceberg.exceptions.NoSuchTableException;
+import org.apache.iceberg.exceptions.ServiceFailureException;
+import org.apache.iceberg.exceptions.ValidationException;
+import org.apache.iceberg.io.FileIO;
+import org.apache.iceberg.relocated.com.google.common.annotations.VisibleForTesting;
+import org.apache.iceberg.relocated.com.google.common.base.Preconditions;
+import org.apache.iceberg.relocated.com.google.common.base.Strings;
+import org.apache.iceberg.relocated.com.google.common.collect.ImmutableList;
+import org.apache.iceberg.relocated.com.google.common.collect.ImmutableMap;
+import org.apache.iceberg.relocated.com.google.common.collect.Maps;
+import org.apache.iceberg.util.LocationUtil;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/** Iceberg Bigquery Metastore Catalog implementation. */
+public final class BigQueryMetastoreCatalog extends BaseMetastoreCatalog
+    implements SupportsNamespaces, Configurable {
+
+  // User provided properties.
+  public static final String PROJECT_ID = "gcp-project";
+  public static final String GCP_LOCATION = "gcp-location";
+  public static final String FILTER_UNSUPPORTED_TABLES = "filter-unsupported-tables";
+  public static final String TESTING_ENABLED = "testing-enabled";
+
+  public static final String HIVE_METASTORE_WAREHOUSE_DIR = "hive.metastore.warehouse.dir";
+
+  private static final Logger LOG = LoggerFactory.getLogger(BigQueryMetastoreCatalog.class);
+
+  private static final String DEFAULT_GCP_LOCATION = "us";
+
+  private String catalogName;
+  private Map<String, String> catalogProperties;
+  private FileIO fileIO;
+  private Configuration conf;
+  private String projectId;
+  private String location;
+  private BigQueryMetastoreClient client;
+  private boolean filterUnsupportedTables;
+
+  // Must have a no-arg constructor to be dynamically loaded
+  // initialize(String name, Map<String, String> properties) will be called to complete
+  // initialization
+  public BigQueryMetastoreCatalog() {}
+
+  @Override
+  public void initialize(String inputName, Map<String, String> properties) {
+    Preconditions.checkArgument(
+        properties.containsKey(PROJECT_ID), "GCP project must be specified");
+
+    projectId = properties.get(PROJECT_ID);
+    location = properties.getOrDefault(GCP_LOCATION, DEFAULT_GCP_LOCATION);
+    boolean testingEnabled =
+        Boolean.parseBoolean(properties.getOrDefault(TESTING_ENABLED, "false"));
+
+    BigQueryOptions options =
+        BigQueryOptions.newBuilder()
+            .setProjectId(projectId)
+            .setLocation(location)
+            .setRetrySettings(ServiceOptions.getDefaultRetrySettings())
+            .build();
+
+    try {
+      if (testingEnabled) {
+        client = new FakeBigQueryMetastoreClient(options);
+
+      } else {
+        client = new BigQueryMetastoreClientImpl(options);
+      }
+    } catch (IOException e) {
+      throw new ServiceFailureException(e, "Creating BigQuery client failed");
+    } catch (GeneralSecurityException e) {
+      throw new ValidationException(e, "Creating BigQuery client failed due to a security issue");
+    }
+    initialize(inputName, properties, projectId, location, client);
+  }
+
+  @VisibleForTesting
+  void initialize(
+      String inputName,
+      Map<String, String> properties,
+      String initialProjectId,
+      String initialLocation,
+      BigQueryMetastoreClient bigQueryMetaStoreClient) {
+    Preconditions.checkArgument(bigQueryMetaStoreClient != null, "BigQuery client can not be null");
+    this.catalogName = inputName;
+    this.catalogProperties = ImmutableMap.copyOf(properties);
+    this.projectId = initialProjectId;
+    this.location = initialLocation;
+    this.client = bigQueryMetaStoreClient;
+
+    if (this.conf == null) {
+      LOG.warn("No configuration was set, using the default environment Configuration");
+      this.conf = new Configuration();
+    }
+
+    LOG.info("Using BigQuery Metastore Iceberg Catalog: {}", inputName);
+
+    if (properties.containsKey(CatalogProperties.WAREHOUSE_LOCATION)) {
+      // Iceberg always removes trailing slash to avoid paths like "<folder>//data" in file systems
+      // like s3.
+      this.conf.set(
+          HIVE_METASTORE_WAREHOUSE_DIR,
+          LocationUtil.stripTrailingSlash(properties.get(CatalogProperties.WAREHOUSE_LOCATION)));
+    }
+
+    String fileIoImpl =
+        properties.getOrDefault(
+            CatalogProperties.FILE_IO_IMPL, "org.apache.iceberg.hadoop.HadoopFileIO");
+    this.fileIO = CatalogUtil.loadFileIO(fileIoImpl, properties, conf);
+
+    this.filterUnsupportedTables =
+        Boolean.parseBoolean(properties.getOrDefault(FILTER_UNSUPPORTED_TABLES, "false"));
+  }
+
+  @Override
+  protected TableOperations newTableOps(TableIdentifier identifier) {
+    return new BigQueryTableOperations(
+        client,
+        fileIO,
+        projectId,
+        // Sometimes extensions have the namespace contain the table name too, so we are forced to
+        // allow invalid namespace and just take the first part here like other catalog
+        // implementations do.
+        identifier.namespace().level(0),
+        identifier.name(),
+        conf);
+  }
+
+  @Override
+  protected String defaultWarehouseLocation(TableIdentifier identifier) {
+    String locationUri = null;
+    DatasetReference datasetReference = toDatasetReference(identifier.namespace());
+    Dataset dataset = client.getDataset(datasetReference);
+    if (dataset != null && dataset.getExternalCatalogDatasetOptions() != null) {
+      locationUri = dataset.getExternalCatalogDatasetOptions().getDefaultStorageLocationUri();
+    }
+    return String.format(
+        "%s/%s",
+        Strings.isNullOrEmpty(locationUri)
+            ? datasetReference.getDatasetId()
+            : LocationUtil.stripTrailingSlash(locationUri),
+        identifier.name());
+  }
+
+  @Override
+  public List<TableIdentifier> listTables(Namespace namespace) {
+    validateNamespace(namespace);
+
+    return client.listTables(toDatasetReference(namespace), filterUnsupportedTables).stream()
+        .map(
+            table -> TableIdentifier.of(namespace.level(0), table.getTableReference().getTableId()))
+        .collect(ImmutableList.toImmutableList());
+  }
+
+  @Override
+  public boolean dropTable(TableIdentifier identifier, boolean purge) {
+    try {
+      TableOperations ops = newTableOps(identifier);
+      TableMetadata lastMetadata = ops.current();
+
+      client.deleteTable(toTableReference(identifier));
+
+      if (purge && lastMetadata != null) {
+        CatalogUtil.dropTableData(ops.io(), lastMetadata);
+      }
+    } catch (NoSuchTableException e) { // Not catching a NoSuchIcebergTableException on purpose
+      return false; // The documentation says just return false in this case
+    }
+    return true;
+  }
+
+  @Override
+  public void renameTable(TableIdentifier from, TableIdentifier to) {
+    validateNamespace(from.namespace());
+    validateNamespace(to.namespace());
+
+    try {

Review Comment:
   why not check if the `to` namespace actually exists via `namespaceExists(to.namespace())`?



-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org

For queries about this service, please contact Infrastructure at:
users@infra.apache.org


---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For additional commands, e-mail: issues-help@iceberg.apache.org


Re: [PR] Catalog: Add BigQuery Metastore Catalog Support [iceberg]

Posted by "talatuyarer (via GitHub)" <gi...@apache.org>.
talatuyarer commented on code in PR #12808:
URL: https://github.com/apache/iceberg/pull/12808#discussion_r2054484675


##########
gcp/src/main/java/org/apache/iceberg/gcp/GCPProperties.java:
##########
@@ -28,6 +28,9 @@
 import org.apache.iceberg.util.PropertyUtil;
 
 public class GCPProperties implements Serializable {
+  /** The GCP project ID. Required. */
+  public static final String PROJECT_ID = "gcp-project";

Review Comment:
   I agree with you. I removed. 



-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org

For queries about this service, please contact Infrastructure at:
users@infra.apache.org


---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For additional commands, e-mail: issues-help@iceberg.apache.org


Re: [PR] Catalog: Add BigQuery Metastore Catalog Support [iceberg]

Posted by "talatuyarer (via GitHub)" <gi...@apache.org>.
talatuyarer commented on code in PR #12808:
URL: https://github.com/apache/iceberg/pull/12808#discussion_r2054476270


##########
bigquery/src/main/java/org/apache/iceberg/gcp/bigquery/BigQueryMetastoreUtils.java:
##########
@@ -0,0 +1,97 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.iceberg.gcp.bigquery;
+
+import com.google.api.services.bigquery.model.ExternalCatalogDatasetOptions;
+import com.google.api.services.bigquery.model.ExternalCatalogTableOptions;
+import com.google.api.services.bigquery.model.SerDeInfo;
+import com.google.api.services.bigquery.model.StorageDescriptor;
+import java.util.Map;
+
+/** Shared utilities for BigQuery Metastore specific functions and constants. */
+public final class BigQueryMetastoreUtils {
+
+  private BigQueryMetastoreUtils() {}
+
+  public static final String HIVE_SERIALIZATION_LIBRARY =
+      "org.apache.iceberg.mr.hive.HiveIcebergSerDe";
+  public static final String HIVE_FILE_INPUT_FORMAT =
+      "org.apache.iceberg.mr.hive.HiveIcebergInputFormat";
+  public static final String HIVE_FILE_OUTPUT_FORMAT =
+      "org.apache.iceberg.mr.hive.HiveIcebergOutputFormat";
+
+  // TODO(b/341933455): Consider using "org.apache.iceberg.mr.hive.HiveIcebergSerDe" when

Review Comment:
   I removed all places. 



-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org

For queries about this service, please contact Infrastructure at:
users@infra.apache.org


---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For additional commands, e-mail: issues-help@iceberg.apache.org


Re: [PR] Catalog: Add BigQuery Metastore Catalog Support [iceberg]

Posted by "talatuyarer (via GitHub)" <gi...@apache.org>.
talatuyarer commented on code in PR #12808:
URL: https://github.com/apache/iceberg/pull/12808#discussion_r2056513109


##########
bigquery/src/test/java/org/apache/iceberg/gcp/bigquery/BigQueryTableOperationsTest.java:
##########
@@ -0,0 +1,228 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.iceberg.gcp.bigquery;
+
+import static org.apache.iceberg.gcp.bigquery.BigQueryMetastoreCatalog.PROJECT_ID;
+import static org.assertj.core.api.Assertions.assertThat;
+import static org.assertj.core.api.Assertions.assertThatThrownBy;
+import static org.mockito.Mockito.any;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.reset;
+import static org.mockito.Mockito.times;
+import static org.mockito.Mockito.verify;
+import static org.mockito.Mockito.when;
+
+import com.google.api.services.bigquery.model.Dataset;
+import com.google.api.services.bigquery.model.DatasetReference;
+import com.google.api.services.bigquery.model.ExternalCatalogDatasetOptions;
+import com.google.api.services.bigquery.model.ExternalCatalogTableOptions;
+import com.google.api.services.bigquery.model.Table;
+import com.google.api.services.bigquery.model.TableReference;
+import java.io.File;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.iceberg.CatalogProperties;
+import org.apache.iceberg.PartitionSpec;
+import org.apache.iceberg.Schema;
+import org.apache.iceberg.catalog.TableIdentifier;
+import org.apache.iceberg.exceptions.CommitFailedException;
+import org.apache.iceberg.exceptions.NoSuchTableException;
+import org.apache.iceberg.exceptions.ValidationException;
+import org.apache.iceberg.relocated.com.google.common.collect.ImmutableMap;
+import org.apache.iceberg.types.Types;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.io.TempDir;
+import org.mockito.ArgumentCaptor;
+
+public class BigQueryTableOperationsTest {
+
+  @TempDir private File tempFolder;
+
+  private static final String GCP_PROJECT = "my-project";
+  private static final String GCP_REGION = "us";
+  private static final String DATASET_ID = "db";
+  private static final String TABLE_ID = "tbl";
+  private static final TableIdentifier SPARK_TABLE_ID = TableIdentifier.of(DATASET_ID, TABLE_ID);
+
+  private static final TableReference TABLE_REFERENCE =
+      new TableReference().setProjectId(GCP_PROJECT).setDatasetId(DATASET_ID).setTableId(TABLE_ID);
+
+  private final BigQueryMetastoreClient bigQueryMetaStoreClient =
+      mock(BigQueryMetastoreClient.class);
+
+  private BigQueryMetastoreCatalog bigQueryMetastoreCatalog;
+  private BigQueryTableOperations tableOps;
+
+  @BeforeEach
+  public void before() {
+    this.bigQueryMetastoreCatalog = new BigQueryMetastoreCatalog();
+    this.bigQueryMetastoreCatalog.setConf(new Configuration());
+    String warehouseLocation = tempFolder.toPath().resolve("hive-warehouse").toString();
+
+    bigQueryMetastoreCatalog.initialize(
+        "CATALOG_ID",
+        /* properties= */ ImmutableMap.of(
+            PROJECT_ID,
+            GCP_PROJECT,
+            CatalogProperties.WAREHOUSE_LOCATION,
+            warehouseLocation,
+            CatalogProperties.FILE_IO_IMPL,
+            "org.apache.iceberg.hadoop.HadoopFileIO"),
+        GCP_PROJECT,
+        GCP_REGION,
+        bigQueryMetaStoreClient);
+    this.tableOps = (BigQueryTableOperations) bigQueryMetastoreCatalog.newTableOps(SPARK_TABLE_ID);
+  }
+
+  @Test
+  public void testDoFresh_fetchLatestMetadataFromBigQuery() throws Exception {

Review Comment:
   You are right. My bad



-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org

For queries about this service, please contact Infrastructure at:
users@infra.apache.org


---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For additional commands, e-mail: issues-help@iceberg.apache.org


Re: [PR] Catalog: Add BigQuery Metastore Catalog Support [iceberg]

Posted by "talatuyarer (via GitHub)" <gi...@apache.org>.
talatuyarer commented on code in PR #12808:
URL: https://github.com/apache/iceberg/pull/12808#discussion_r2059726132


##########
bigquery/src/test/java/org/apache/iceberg/gcp/bigquery/TestBigQueryCatalog.java:
##########
@@ -0,0 +1,673 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.iceberg.gcp.bigquery;
+
+import static org.apache.iceberg.CatalogUtil.ICEBERG_CATALOG_TYPE;
+import static org.apache.iceberg.CatalogUtil.ICEBERG_CATALOG_TYPE_BIGQUERY;
+import static org.apache.iceberg.gcp.bigquery.BigQueryMetastoreCatalog.PROJECT_ID;
+import static org.apache.iceberg.gcp.bigquery.BigQueryMetastoreCatalog.TESTING_ENABLED;
+import static org.apache.iceberg.types.Types.NestedField.optional;
+import static org.assertj.core.api.Assertions.assertThat;
+import static org.assertj.core.api.Assertions.assertThatThrownBy;
+
+import java.io.File;
+import java.util.List;
+import java.util.Map;
+import org.apache.iceberg.CatalogProperties;
+import org.apache.iceberg.CatalogUtil;
+import org.apache.iceberg.PartitionSpec;
+import org.apache.iceberg.Schema;
+import org.apache.iceberg.SortOrder;
+import org.apache.iceberg.Table;
+import org.apache.iceberg.TableUtil;
+import org.apache.iceberg.Transaction;
+import org.apache.iceberg.catalog.CatalogTests;
+import org.apache.iceberg.catalog.Namespace;
+import org.apache.iceberg.catalog.TableIdentifier;
+import org.apache.iceberg.exceptions.NoSuchNamespaceException;
+import org.apache.iceberg.exceptions.NoSuchTableException;
+import org.apache.iceberg.expressions.Expressions;
+import org.apache.iceberg.relocated.com.google.common.collect.ImmutableMap;
+import org.apache.iceberg.types.Types;
+import org.junit.jupiter.api.AfterEach;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Disabled;
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.io.TempDir;
+import org.junit.jupiter.params.ParameterizedTest;
+import org.junit.jupiter.params.provider.ValueSource;
+
+public class TestBigQueryCatalog extends CatalogTests<BigQueryMetastoreCatalog> {
+
+  private static final String GCP_PROJECT = "project-id";
+  private static final String CATALOG_ID = "catalog-name";
+
+  // another schema that is not the same
+  protected static final Schema OTHER_SCHEMA_OPTIONAL =
+      new Schema(optional(1, "some_id", Types.IntegerType.get()));
+
+  // Schema passed to create tables
+  protected static final Schema SCHEMA_OPTIONAL =
+      new Schema(
+          optional(3, "id", Types.IntegerType.get(), "unique ID 🤪"),
+          optional(4, "data", Types.StringType.get()));
+
+  // This is the actual schema for the table, with column IDs reassigned
+  protected static final Schema REPLACE_SCHEMA_OPTIONAL =
+      new Schema(
+          optional(2, "id", Types.IntegerType.get(), "unique ID 🤪"),
+          optional(3, "data", Types.StringType.get()));
+
+  protected static final PartitionSpec REPLACE_SPEC_OPTIONAL =
+      PartitionSpec.builderFor(REPLACE_SCHEMA_OPTIONAL).bucket("id", 16).withSpecId(1).build();
+
+  protected static final SortOrder REPLACE_WRITE_ORDER_OPTIONAL =

Review Comment:
   yes it can, but I copied them from CatalogTests I just changed with optional field. I want to keep as is. Soon I will remove these tests whenever we start supporting required filed change. 



-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org

For queries about this service, please contact Infrastructure at:
users@infra.apache.org


---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For additional commands, e-mail: issues-help@iceberg.apache.org


Re: [PR] Catalog: Add BigQuery Metastore Catalog Support [iceberg]

Posted by "talatuyarer (via GitHub)" <gi...@apache.org>.
talatuyarer commented on code in PR #12808:
URL: https://github.com/apache/iceberg/pull/12808#discussion_r2059760489


##########
bigquery/src/test/java/org/apache/iceberg/gcp/bigquery/TestBigQueryCatalog.java:
##########
@@ -0,0 +1,673 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.iceberg.gcp.bigquery;
+
+import static org.apache.iceberg.CatalogUtil.ICEBERG_CATALOG_TYPE;
+import static org.apache.iceberg.CatalogUtil.ICEBERG_CATALOG_TYPE_BIGQUERY;
+import static org.apache.iceberg.gcp.bigquery.BigQueryMetastoreCatalog.PROJECT_ID;
+import static org.apache.iceberg.gcp.bigquery.BigQueryMetastoreCatalog.TESTING_ENABLED;
+import static org.apache.iceberg.types.Types.NestedField.optional;
+import static org.assertj.core.api.Assertions.assertThat;
+import static org.assertj.core.api.Assertions.assertThatThrownBy;
+
+import java.io.File;
+import java.util.List;
+import java.util.Map;
+import org.apache.iceberg.CatalogProperties;
+import org.apache.iceberg.CatalogUtil;
+import org.apache.iceberg.PartitionSpec;
+import org.apache.iceberg.Schema;
+import org.apache.iceberg.SortOrder;
+import org.apache.iceberg.Table;
+import org.apache.iceberg.TableUtil;
+import org.apache.iceberg.Transaction;
+import org.apache.iceberg.catalog.CatalogTests;
+import org.apache.iceberg.catalog.Namespace;
+import org.apache.iceberg.catalog.TableIdentifier;
+import org.apache.iceberg.exceptions.NoSuchNamespaceException;
+import org.apache.iceberg.exceptions.NoSuchTableException;
+import org.apache.iceberg.expressions.Expressions;
+import org.apache.iceberg.relocated.com.google.common.collect.ImmutableMap;
+import org.apache.iceberg.types.Types;
+import org.junit.jupiter.api.AfterEach;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Disabled;
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.io.TempDir;
+import org.junit.jupiter.params.ParameterizedTest;
+import org.junit.jupiter.params.provider.ValueSource;
+
+public class TestBigQueryCatalog extends CatalogTests<BigQueryMetastoreCatalog> {
+
+  private static final String GCP_PROJECT = "project-id";
+  private static final String CATALOG_ID = "catalog-name";
+
+  // another schema that is not the same
+  protected static final Schema OTHER_SCHEMA_OPTIONAL =
+      new Schema(optional(1, "some_id", Types.IntegerType.get()));
+
+  // Schema passed to create tables
+  protected static final Schema SCHEMA_OPTIONAL =
+      new Schema(
+          optional(3, "id", Types.IntegerType.get(), "unique ID 🤪"),
+          optional(4, "data", Types.StringType.get()));
+
+  // This is the actual schema for the table, with column IDs reassigned
+  protected static final Schema REPLACE_SCHEMA_OPTIONAL =
+      new Schema(
+          optional(2, "id", Types.IntegerType.get(), "unique ID 🤪"),
+          optional(3, "data", Types.StringType.get()));
+
+  protected static final PartitionSpec REPLACE_SPEC_OPTIONAL =
+      PartitionSpec.builderFor(REPLACE_SCHEMA_OPTIONAL).bucket("id", 16).withSpecId(1).build();
+
+  protected static final SortOrder REPLACE_WRITE_ORDER_OPTIONAL =
+      SortOrder.builderFor(REPLACE_SCHEMA_OPTIONAL)
+          .asc(Expressions.bucket("id", 16))
+          .asc("id")
+          .build();
+
+  @TempDir private File tempFolder;
+  private BigQueryMetastoreCatalog catalog;
+  private String warehouseLocation;
+
+  @BeforeEach
+  public void before() throws Exception {
+    catalog = initCatalog(CATALOG_ID, ImmutableMap.of());
+  }
+
+  @AfterEach
+  public void after() throws Exception {
+    // Drop all tables in all datasets
+    List<Namespace> namespaces = catalog().listNamespaces(Namespace.empty());
+    for (Namespace namespace : namespaces) {
+      List<TableIdentifier> tables = catalog().listTables(namespace);
+      for (TableIdentifier table : tables) {
+        try {
+          catalog().dropTable(table, true); // drop with purge
+        } catch (NoSuchTableException e) {
+          // Table might be dropped by a previous iteration, ignore
+        }
+      }
+
+      // Drop all datasets (except any initial ones)
+      try {
+        catalog().dropNamespace(namespace);
+      } catch (NoSuchNamespaceException e) {
+        // Namespace might be dropped by a previous iteration, ignore
+      }
+    }
+  }
+
+  @Override
+  protected boolean requiresNamespaceCreate() {
+    return true;
+  }
+
+  @Override
+  protected boolean supportsNamesWithSlashes() {
+    return false;
+  }
+
+  @Override
+  protected boolean supportsNamesWithDot() {
+    return false;
+  }
+
+  @Override
+  public BigQueryMetastoreCatalog catalog() {
+    return catalog;
+  }
+
+  @Override
+  protected BigQueryMetastoreCatalog initCatalog(
+      String catalogName, Map<String, String> additionalProperties) {
+
+    warehouseLocation = tempFolder.toPath().resolve("hive-warehouse").toString();
+
+    Map<String, String> properties =
+        Map.of(
+            ICEBERG_CATALOG_TYPE,
+            ICEBERG_CATALOG_TYPE_BIGQUERY,
+            PROJECT_ID,
+            GCP_PROJECT,
+            CatalogProperties.WAREHOUSE_LOCATION,
+            warehouseLocation,
+            TESTING_ENABLED,
+            "true",
+            CatalogProperties.TABLE_DEFAULT_PREFIX + "default-key1",
+            "catalog-default-key1",
+            CatalogProperties.TABLE_DEFAULT_PREFIX + "default-key2",
+            "catalog-default-key2",
+            CatalogProperties.TABLE_DEFAULT_PREFIX + "override-key3",
+            "catalog-default-key3",
+            CatalogProperties.TABLE_OVERRIDE_PREFIX + "override-key3",
+            "catalog-override-key3",
+            CatalogProperties.TABLE_OVERRIDE_PREFIX + "override-key4",
+            "catalog-override-key4");
+
+    BigQueryMetastoreCatalog tmpCatalog =
+        (BigQueryMetastoreCatalog)
+            CatalogUtil.buildIcebergCatalog(
+                catalogName,
+                ImmutableMap.<String, String>builder()
+                    .putAll(properties)
+                    .putAll(additionalProperties)
+                    .build(),
+                null);
+
+    return tmpCatalog;
+  }
+
+  @Test
+  public void testRenameTable() {
+    if (requiresNamespaceCreate()) {
+      catalog.createNamespace(NS);
+    }
+
+    assertThat(catalog.tableExists(TABLE))
+        .as("Source table should not exist before create")
+        .isFalse();
+
+    catalog.buildTable(TABLE, SCHEMA).create();
+    assertThat(catalog.tableExists(TABLE)).as("Table should exist after create").isTrue();
+
+    assertThat(catalog.tableExists(RENAMED_TABLE))
+        .as("Destination table should not exist before rename")
+        .isFalse();
+
+    assertThatThrownBy(() -> catalog.renameTable(TABLE, RENAMED_TABLE))
+        .isInstanceOf(UnsupportedOperationException.class)
+        .hasMessageContaining("Table rename operation is unsupported");
+
+    catalog.dropTable(TABLE);
+  }
+
+  @Disabled(
+      "BigQuery does not allow adding a REQUIRED column to an existing table schema. "
+          + "When this operation is attempted, an exception gets thrown.")
+  @Test
+  public void testOverrideTablePropertiesReplaceTransaction() {}

Review Comment:
   Sounds good to me. I can remove all optional filed tests now. What about V3 spec tests such as createTableTransaction or testCreateTableWithDefaultColumnValue ? Bigquery does not support some V3 features. I will disable them ? 



-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org

For queries about this service, please contact Infrastructure at:
users@infra.apache.org


---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For additional commands, e-mail: issues-help@iceberg.apache.org


Re: [PR] Catalog: Add BigQuery Metastore Catalog Support [iceberg]

Posted by "talatuyarer (via GitHub)" <gi...@apache.org>.
talatuyarer commented on code in PR #12808:
URL: https://github.com/apache/iceberg/pull/12808#discussion_r2059829768


##########
bigquery/src/test/java/org/apache/iceberg/gcp/bigquery/TestBigQueryCatalog.java:
##########
@@ -0,0 +1,673 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.iceberg.gcp.bigquery;
+
+import static org.apache.iceberg.CatalogUtil.ICEBERG_CATALOG_TYPE;
+import static org.apache.iceberg.CatalogUtil.ICEBERG_CATALOG_TYPE_BIGQUERY;
+import static org.apache.iceberg.gcp.bigquery.BigQueryMetastoreCatalog.PROJECT_ID;
+import static org.apache.iceberg.gcp.bigquery.BigQueryMetastoreCatalog.TESTING_ENABLED;
+import static org.apache.iceberg.types.Types.NestedField.optional;
+import static org.assertj.core.api.Assertions.assertThat;
+import static org.assertj.core.api.Assertions.assertThatThrownBy;
+
+import java.io.File;
+import java.util.List;
+import java.util.Map;
+import org.apache.iceberg.CatalogProperties;
+import org.apache.iceberg.CatalogUtil;
+import org.apache.iceberg.PartitionSpec;
+import org.apache.iceberg.Schema;
+import org.apache.iceberg.SortOrder;
+import org.apache.iceberg.Table;
+import org.apache.iceberg.TableUtil;
+import org.apache.iceberg.Transaction;
+import org.apache.iceberg.catalog.CatalogTests;
+import org.apache.iceberg.catalog.Namespace;
+import org.apache.iceberg.catalog.TableIdentifier;
+import org.apache.iceberg.exceptions.NoSuchNamespaceException;
+import org.apache.iceberg.exceptions.NoSuchTableException;
+import org.apache.iceberg.expressions.Expressions;
+import org.apache.iceberg.relocated.com.google.common.collect.ImmutableMap;
+import org.apache.iceberg.types.Types;
+import org.junit.jupiter.api.AfterEach;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Disabled;
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.io.TempDir;
+import org.junit.jupiter.params.ParameterizedTest;
+import org.junit.jupiter.params.provider.ValueSource;
+
+public class TestBigQueryCatalog extends CatalogTests<BigQueryMetastoreCatalog> {
+
+  private static final String GCP_PROJECT = "project-id";
+  private static final String CATALOG_ID = "catalog-name";
+
+  // another schema that is not the same
+  protected static final Schema OTHER_SCHEMA_OPTIONAL =
+      new Schema(optional(1, "some_id", Types.IntegerType.get()));
+
+  // Schema passed to create tables
+  protected static final Schema SCHEMA_OPTIONAL =
+      new Schema(
+          optional(3, "id", Types.IntegerType.get(), "unique ID 🤪"),
+          optional(4, "data", Types.StringType.get()));
+
+  // This is the actual schema for the table, with column IDs reassigned
+  protected static final Schema REPLACE_SCHEMA_OPTIONAL =
+      new Schema(
+          optional(2, "id", Types.IntegerType.get(), "unique ID 🤪"),
+          optional(3, "data", Types.StringType.get()));
+
+  protected static final PartitionSpec REPLACE_SPEC_OPTIONAL =
+      PartitionSpec.builderFor(REPLACE_SCHEMA_OPTIONAL).bucket("id", 16).withSpecId(1).build();
+
+  protected static final SortOrder REPLACE_WRITE_ORDER_OPTIONAL =
+      SortOrder.builderFor(REPLACE_SCHEMA_OPTIONAL)
+          .asc(Expressions.bucket("id", 16))
+          .asc("id")
+          .build();
+
+  @TempDir private File tempFolder;
+  private BigQueryMetastoreCatalog catalog;
+  private BigQueryMetastoreClient fakeBigQueryMetastoreClient;
+  private String warehouseLocation;
+
+  @BeforeEach
+  public void before() throws Exception {
+    catalog = initCatalog(CATALOG_ID, ImmutableMap.of());
+  }
+
+  @AfterEach
+  public void after() throws Exception {
+    // Drop all tables in all datasets
+    List<Namespace> namespaces = catalog().listNamespaces(Namespace.empty());
+    for (Namespace namespace : namespaces) {
+      List<TableIdentifier> tables = catalog().listTables(namespace);
+      for (TableIdentifier table : tables) {
+        try {
+          catalog().dropTable(table, true); // drop with purge
+        } catch (NoSuchTableException e) {
+          // Table might be dropped by a previous iteration, ignore
+        }
+      }
+
+      // Drop all datasets (except any initial ones)
+      try {
+        catalog().dropNamespace(namespace);
+      } catch (NoSuchNamespaceException e) {
+        // Namespace might be dropped by a previous iteration, ignore
+      }
+    }
+  }
+
+  @Override
+  protected boolean requiresNamespaceCreate() {
+    return true;
+  }
+
+  @Override
+  protected boolean supportsNamesWithSlashes() {
+    return false;
+  }
+
+  @Override
+  protected boolean supportsNamesWithDot() {
+    return false;
+  }
+
+  @Override
+  public BigQueryMetastoreCatalog catalog() {
+    return catalog;
+  }
+
+  @Override
+  protected BigQueryMetastoreCatalog initCatalog(
+      String catalogName, Map<String, String> additionalProperties) {
+
+    warehouseLocation = tempFolder.toPath().resolve("hive-warehouse").toString();
+
+    Map<String, String> properties =
+        Map.of(
+            ICEBERG_CATALOG_TYPE,
+            ICEBERG_CATALOG_TYPE_BIGQUERY,
+            PROJECT_ID,
+            GCP_PROJECT,
+            CatalogProperties.WAREHOUSE_LOCATION,
+            warehouseLocation,
+            TESTING_ENABLED,
+            "true",
+            CatalogProperties.TABLE_DEFAULT_PREFIX + "default-key1",
+            "catalog-default-key1",
+            CatalogProperties.TABLE_DEFAULT_PREFIX + "default-key2",
+            "catalog-default-key2",
+            CatalogProperties.TABLE_DEFAULT_PREFIX + "override-key3",
+            "catalog-default-key3",
+            CatalogProperties.TABLE_OVERRIDE_PREFIX + "override-key3",
+            "catalog-override-key3",
+            CatalogProperties.TABLE_OVERRIDE_PREFIX + "override-key4",
+            "catalog-override-key4");
+
+    BigQueryMetastoreCatalog tmpCatalog =
+        (BigQueryMetastoreCatalog)
+            CatalogUtil.buildIcebergCatalog(
+                catalogName,
+                ImmutableMap.<String, String>builder()
+                    .putAll(properties)
+                    .putAll(additionalProperties)
+                    .build(),
+                null);
+
+    return tmpCatalog;
+  }
+
+  @Test
+  public void testRenameTable() {
+    if (requiresNamespaceCreate()) {
+      catalog.createNamespace(NS);
+    }
+
+    assertThat(catalog.tableExists(TABLE))
+        .as("Source table should not exist before create")
+        .isFalse();
+
+    catalog.buildTable(TABLE, SCHEMA).create();
+    assertThat(catalog.tableExists(TABLE)).as("Table should exist after create").isTrue();
+
+    assertThat(catalog.tableExists(RENAMED_TABLE))
+        .as("Destination table should not exist before rename")
+        .isFalse();
+
+    assertThatThrownBy(() -> catalog.renameTable(TABLE, RENAMED_TABLE))
+        .isInstanceOf(UnsupportedOperationException.class)
+        .hasMessageContaining("Table rename operation is unsupported");
+
+    catalog.dropTable(TABLE);
+  }
+
+  @Disabled(
+      "BigQuery does not allow adding a REQUIRED column to an existing table schema. "
+          + "When this operation is attempted, an exception gets thrown.")
+  @Test
+  public void testOverrideTablePropertiesReplaceTransaction() {}
+
+  @Test
+  public void testConcurrentReplaceTransactionSchema() {

Review Comment:
   Removed all optional tests



-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org

For queries about this service, please contact Infrastructure at:
users@infra.apache.org


---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For additional commands, e-mail: issues-help@iceberg.apache.org


Re: [PR] Catalog: Add BigQuery Metastore Catalog Support [iceberg]

Posted by "nastra (via GitHub)" <gi...@apache.org>.
nastra commented on code in PR #12808:
URL: https://github.com/apache/iceberg/pull/12808#discussion_r2063890979


##########
bigquery/src/test/java/org/apache/iceberg/gcp/bigquery/BigQueryMetastoreTestUtils.java:
##########
@@ -0,0 +1,87 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.iceberg.gcp.bigquery;
+
+import static org.apache.iceberg.types.Types.NestedField.required;
+import static org.assertj.core.api.Assertions.assertThat;
+
+import com.google.api.services.bigquery.model.ExternalCatalogTableOptions;
+import com.google.api.services.bigquery.model.StorageDescriptor;
+import com.google.api.services.bigquery.model.Table;
+import com.google.api.services.bigquery.model.TableReference;
+import java.io.File;
+import java.io.IOException;
+import java.util.Collections;
+import java.util.Optional;
+import org.apache.commons.io.FileUtils;
+import org.apache.commons.io.filefilter.TrueFileFilter;
+import org.apache.iceberg.Schema;
+import org.apache.iceberg.catalog.TableIdentifier;
+import org.apache.iceberg.types.Types;
+
+/** Test utility methods for BigQuery Metastore Iceberg catalog. */
+public final class BigQueryMetastoreTestUtils {

Review Comment:
   There is only a single test class that uses this, so I think it makes more sense to just inline this into the respective test class instead of definining a util class



-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org

For queries about this service, please contact Infrastructure at:
users@infra.apache.org


---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For additional commands, e-mail: issues-help@iceberg.apache.org


Re: [PR] Catalog: Add BigQuery Metastore Catalog Support [iceberg]

Posted by "nastra (via GitHub)" <gi...@apache.org>.
nastra commented on code in PR #12808:
URL: https://github.com/apache/iceberg/pull/12808#discussion_r2065997639


##########
bigquery/src/main/java/org/apache/iceberg/gcp/bigquery/BigQueryMetastoreCatalog.java:
##########
@@ -0,0 +1,405 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.iceberg.gcp.bigquery;
+
+import com.google.api.services.bigquery.model.Dataset;
+import com.google.api.services.bigquery.model.DatasetList.Datasets;
+import com.google.api.services.bigquery.model.DatasetReference;
+import com.google.api.services.bigquery.model.ExternalCatalogDatasetOptions;
+import com.google.api.services.bigquery.model.TableReference;
+import com.google.cloud.ServiceOptions;
+import com.google.cloud.bigquery.BigQueryOptions;
+import java.io.IOException;
+import java.security.GeneralSecurityException;
+import java.util.List;
+import java.util.Locale;
+import java.util.Map;
+import java.util.Objects;
+import java.util.Set;
+import org.apache.hadoop.conf.Configurable;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.iceberg.BaseMetastoreCatalog;
+import org.apache.iceberg.CatalogProperties;
+import org.apache.iceberg.CatalogUtil;
+import org.apache.iceberg.TableMetadata;
+import org.apache.iceberg.TableOperations;
+import org.apache.iceberg.catalog.Namespace;
+import org.apache.iceberg.catalog.SupportsNamespaces;
+import org.apache.iceberg.catalog.TableIdentifier;
+import org.apache.iceberg.exceptions.AlreadyExistsException;
+import org.apache.iceberg.exceptions.NoSuchNamespaceException;
+import org.apache.iceberg.exceptions.NoSuchTableException;
+import org.apache.iceberg.exceptions.ServiceFailureException;
+import org.apache.iceberg.exceptions.ValidationException;
+import org.apache.iceberg.io.FileIO;
+import org.apache.iceberg.relocated.com.google.common.annotations.VisibleForTesting;
+import org.apache.iceberg.relocated.com.google.common.base.Preconditions;
+import org.apache.iceberg.relocated.com.google.common.base.Strings;
+import org.apache.iceberg.relocated.com.google.common.collect.ImmutableList;
+import org.apache.iceberg.relocated.com.google.common.collect.ImmutableMap;
+import org.apache.iceberg.relocated.com.google.common.collect.Maps;
+import org.apache.iceberg.util.LocationUtil;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/** Iceberg Bigquery Metastore Catalog implementation. */
+public class BigQueryMetastoreCatalog extends BaseMetastoreCatalog
+    implements SupportsNamespaces, Configurable {
+
+  // User provided properties.
+  public static final String PROJECT_ID = "gcp-project";
+  public static final String GCP_LOCATION = "gcp-location";
+  public static final String LIST_ALL_TABLES = "list-all-tables";
+
+  public static final String HIVE_METASTORE_WAREHOUSE_DIR = "hive.metastore.warehouse.dir";
+
+  private static final Logger LOG = LoggerFactory.getLogger(BigQueryMetastoreCatalog.class);
+
+  private static final String DEFAULT_GCP_LOCATION = "us";
+
+  private String catalogName;
+  private Map<String, String> catalogProperties;
+  private FileIO fileIO;
+  private Configuration conf;
+  private String projectId;
+  private String location;
+  private BigQueryMetastoreClient client;
+  private boolean listAllTables;
+
+  // Must have a no-arg constructor to be dynamically loaded
+  // initialize(String name, Map<String, String> properties) will be called to complete
+  // initialization
+  public BigQueryMetastoreCatalog() {}
+
+  @Override
+  public void initialize(String inputName, Map<String, String> properties) {
+    Preconditions.checkArgument(
+        properties.containsKey(PROJECT_ID),
+        "Invalid GCP project: %s must be specified",
+        PROJECT_ID);
+
+    projectId = properties.get(PROJECT_ID);

Review Comment:
   ```suggestion
       this.projectId = properties.get(PROJECT_ID);
   ```
   sorry I missed this during previous review rounds. The existing codebase uses `this` if instance fields get assigned a new value for better readability



-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org

For queries about this service, please contact Infrastructure at:
users@infra.apache.org


---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For additional commands, e-mail: issues-help@iceberg.apache.org


Re: [PR] Catalog: Add BigQuery Metastore Catalog Support [iceberg]

Posted by "danielcweeks (via GitHub)" <gi...@apache.org>.
danielcweeks commented on code in PR #12808:
URL: https://github.com/apache/iceberg/pull/12808#discussion_r2067296450


##########
bigquery/src/main/java/org/apache/iceberg/gcp/bigquery/BigQueryMetastoreCatalog.java:
##########
@@ -0,0 +1,405 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.iceberg.gcp.bigquery;
+
+import com.google.api.services.bigquery.model.Dataset;
+import com.google.api.services.bigquery.model.DatasetList.Datasets;
+import com.google.api.services.bigquery.model.DatasetReference;
+import com.google.api.services.bigquery.model.ExternalCatalogDatasetOptions;
+import com.google.api.services.bigquery.model.TableReference;
+import com.google.cloud.ServiceOptions;
+import com.google.cloud.bigquery.BigQueryOptions;
+import java.io.IOException;
+import java.security.GeneralSecurityException;
+import java.util.List;
+import java.util.Locale;
+import java.util.Map;
+import java.util.Objects;
+import java.util.Set;
+import org.apache.hadoop.conf.Configurable;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.iceberg.BaseMetastoreCatalog;
+import org.apache.iceberg.CatalogProperties;
+import org.apache.iceberg.CatalogUtil;
+import org.apache.iceberg.TableMetadata;
+import org.apache.iceberg.TableOperations;
+import org.apache.iceberg.catalog.Namespace;
+import org.apache.iceberg.catalog.SupportsNamespaces;
+import org.apache.iceberg.catalog.TableIdentifier;
+import org.apache.iceberg.exceptions.AlreadyExistsException;
+import org.apache.iceberg.exceptions.NoSuchNamespaceException;
+import org.apache.iceberg.exceptions.NoSuchTableException;
+import org.apache.iceberg.exceptions.ServiceFailureException;
+import org.apache.iceberg.exceptions.ValidationException;
+import org.apache.iceberg.io.FileIO;
+import org.apache.iceberg.relocated.com.google.common.annotations.VisibleForTesting;
+import org.apache.iceberg.relocated.com.google.common.base.Preconditions;
+import org.apache.iceberg.relocated.com.google.common.base.Strings;
+import org.apache.iceberg.relocated.com.google.common.collect.ImmutableList;
+import org.apache.iceberg.relocated.com.google.common.collect.ImmutableMap;
+import org.apache.iceberg.relocated.com.google.common.collect.Maps;
+import org.apache.iceberg.util.LocationUtil;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/** Iceberg Bigquery Metastore Catalog implementation. */
+public class BigQueryMetastoreCatalog extends BaseMetastoreCatalog
+    implements SupportsNamespaces, Configurable {
+
+  // User provided properties.
+  public static final String PROJECT_ID = "gcp-project";
+  public static final String GCP_LOCATION = "gcp-location";
+  public static final String LIST_ALL_TABLES = "list-all-tables";
+
+  public static final String HIVE_METASTORE_WAREHOUSE_DIR = "hive.metastore.warehouse.dir";
+
+  private static final Logger LOG = LoggerFactory.getLogger(BigQueryMetastoreCatalog.class);
+
+  private static final String DEFAULT_GCP_LOCATION = "us";
+
+  private String catalogName;
+  private Map<String, String> catalogProperties;
+  private FileIO fileIO;
+  private Configuration conf;
+  private String projectId;
+  private String location;
+  private BigQueryMetastoreClient client;
+  private boolean listAllTables;
+
+  // Must have a no-arg constructor to be dynamically loaded
+  // initialize(String name, Map<String, String> properties) will be called to complete
+  // initialization
+  public BigQueryMetastoreCatalog() {}
+
+  @Override
+  public void initialize(String inputName, Map<String, String> properties) {

Review Comment:
   ```suggestion
     public void initialize(String name, Map<String, String> properties) {
   ```
   
   I'm not sure where `inputName` came from, but this is actually the catalog name, just using `name` or `catalogName` is less confusing.



-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org

For queries about this service, please contact Infrastructure at:
users@infra.apache.org


---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For additional commands, e-mail: issues-help@iceberg.apache.org


Re: [PR] Catalog: Add BigQuery Metastore Catalog Support [iceberg]

Posted by "nastra (via GitHub)" <gi...@apache.org>.
nastra commented on code in PR #12808:
URL: https://github.com/apache/iceberg/pull/12808#discussion_r2066070261


##########
bigquery/src/test/java/org/apache/iceberg/gcp/bigquery/BigQueryTableOperationsTest.java:
##########
@@ -0,0 +1,274 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.iceberg.gcp.bigquery;
+
+import static org.apache.iceberg.gcp.bigquery.BigQueryMetastoreCatalog.PROJECT_ID;
+import static org.apache.iceberg.types.Types.NestedField.required;
+import static org.assertj.core.api.Assertions.assertThat;
+import static org.assertj.core.api.Assertions.assertThatThrownBy;
+import static org.mockito.Mockito.any;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.reset;
+import static org.mockito.Mockito.times;
+import static org.mockito.Mockito.verify;
+import static org.mockito.Mockito.when;
+
+import com.google.api.services.bigquery.model.Dataset;
+import com.google.api.services.bigquery.model.DatasetReference;
+import com.google.api.services.bigquery.model.ExternalCatalogDatasetOptions;
+import com.google.api.services.bigquery.model.ExternalCatalogTableOptions;
+import com.google.api.services.bigquery.model.StorageDescriptor;
+import com.google.api.services.bigquery.model.Table;
+import com.google.api.services.bigquery.model.TableReference;
+import java.io.File;
+import java.io.IOException;
+import java.util.Collections;
+import java.util.Optional;
+import org.apache.commons.io.FileUtils;
+import org.apache.commons.io.filefilter.TrueFileFilter;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.iceberg.CatalogProperties;
+import org.apache.iceberg.PartitionSpec;
+import org.apache.iceberg.Schema;
+import org.apache.iceberg.catalog.TableIdentifier;
+import org.apache.iceberg.exceptions.CommitFailedException;
+import org.apache.iceberg.exceptions.NoSuchTableException;
+import org.apache.iceberg.exceptions.ValidationException;
+import org.apache.iceberg.relocated.com.google.common.collect.ImmutableMap;
+import org.apache.iceberg.types.Types;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.io.TempDir;
+import org.mockito.ArgumentCaptor;
+
+public class BigQueryTableOperationsTest {

Review Comment:
   ```suggestion
   public class TestBigQueryTableOperations {
   ```



-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org

For queries about this service, please contact Infrastructure at:
users@infra.apache.org


---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For additional commands, e-mail: issues-help@iceberg.apache.org


Re: [PR] Catalog: Add BigQuery Metastore Catalog Support [iceberg]

Posted by "nastra (via GitHub)" <gi...@apache.org>.
nastra commented on code in PR #12808:
URL: https://github.com/apache/iceberg/pull/12808#discussion_r2067914332


##########
bigquery/src/main/java/org/apache/iceberg/gcp/bigquery/BigQueryMetastoreClientImpl.java:
##########
@@ -0,0 +1,594 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.iceberg.gcp.bigquery;
+
+import com.google.api.client.googleapis.javanet.GoogleNetHttpTransport;
+import com.google.api.client.http.HttpHeaders;
+import com.google.api.client.http.HttpResponse;
+import com.google.api.client.http.HttpResponseException;
+import com.google.api.client.http.HttpStatusCodes;
+import com.google.api.client.json.gson.GsonFactory;
+import com.google.api.client.util.Data;
+import com.google.api.services.bigquery.Bigquery;
+import com.google.api.services.bigquery.BigqueryScopes;
+import com.google.api.services.bigquery.model.Dataset;
+import com.google.api.services.bigquery.model.DatasetList;
+import com.google.api.services.bigquery.model.DatasetList.Datasets;
+import com.google.api.services.bigquery.model.DatasetReference;
+import com.google.api.services.bigquery.model.ExternalCatalogDatasetOptions;
+import com.google.api.services.bigquery.model.ExternalCatalogTableOptions;
+import com.google.api.services.bigquery.model.Table;
+import com.google.api.services.bigquery.model.TableList;
+import com.google.api.services.bigquery.model.TableList.Tables;
+import com.google.api.services.bigquery.model.TableReference;
+import com.google.api.services.bigquery.model.TableSchema;
+import com.google.auth.http.HttpCredentialsAdapter;
+import com.google.auth.oauth2.GoogleCredentials;
+import com.google.cloud.BaseServiceException;
+import com.google.cloud.ExceptionHandler;
+import com.google.cloud.bigquery.BigQueryErrorMessages;
+import com.google.cloud.bigquery.BigQueryOptions;
+import com.google.cloud.bigquery.BigQueryRetryConfig;
+import com.google.cloud.bigquery.BigQueryRetryHelper;
+import java.io.IOException;
+import java.nio.charset.StandardCharsets;
+import java.security.GeneralSecurityException;
+import java.util.List;
+import java.util.Locale;
+import java.util.Map;
+import java.util.Set;
+import java.util.stream.Collectors;
+import java.util.stream.Stream;
+import org.apache.iceberg.BaseMetastoreTableOperations;
+import org.apache.iceberg.exceptions.AlreadyExistsException;
+import org.apache.iceberg.exceptions.BadRequestException;
+import org.apache.iceberg.exceptions.ForbiddenException;
+import org.apache.iceberg.exceptions.NoSuchIcebergTableException;
+import org.apache.iceberg.exceptions.NoSuchNamespaceException;
+import org.apache.iceberg.exceptions.NoSuchTableException;
+import org.apache.iceberg.exceptions.NotAuthorizedException;
+import org.apache.iceberg.exceptions.RuntimeIOException;
+import org.apache.iceberg.exceptions.ServiceFailureException;
+import org.apache.iceberg.exceptions.ServiceUnavailableException;
+import org.apache.iceberg.exceptions.ValidationException;
+import org.apache.iceberg.relocated.com.google.common.base.Preconditions;
+import org.apache.iceberg.relocated.com.google.common.collect.Lists;
+import org.apache.iceberg.relocated.com.google.common.collect.Maps;
+
+/** A client of Google Bigquery Metastore functions over the BigQuery service. */
+public final class BigQueryMetastoreClientImpl implements BigQueryMetastoreClient {
+
+  public static final ExceptionHandler.Interceptor EXCEPTION_HANDLER_INTERCEPTOR =
+      new ExceptionHandler.Interceptor() {
+
+        private static final long serialVersionUID = -8429573486870467828L;
+
+        @Override
+        public RetryResult afterEval(Exception exception, RetryResult retryResult) {
+          return ExceptionHandler.Interceptor.RetryResult.CONTINUE_EVALUATION;
+        }
+
+        @Override
+        public RetryResult beforeEval(Exception exception) {
+          if (exception instanceof BaseServiceException) {
+            boolean retriable = ((BaseServiceException) exception).isRetryable();
+            return retriable
+                ? ExceptionHandler.Interceptor.RetryResult.RETRY
+                : ExceptionHandler.Interceptor.RetryResult.CONTINUE_EVALUATION;
+          }
+
+          return ExceptionHandler.Interceptor.RetryResult.CONTINUE_EVALUATION;
+        }
+      };
+
+  private final Bigquery client;
+  private final BigQueryOptions bigqueryOptions;
+
+  // Retry config with error messages and regex for rate limit exceeded errors.
+  private static final BigQueryRetryConfig DEFAULT_RETRY_CONFIG =
+      BigQueryRetryConfig.newBuilder()
+          .retryOnMessage(BigQueryErrorMessages.RATE_LIMIT_EXCEEDED_MSG)
+          .retryOnMessage(BigQueryErrorMessages.JOB_RATE_LIMIT_EXCEEDED_MSG)
+          .retryOnRegEx(BigQueryErrorMessages.RetryRegExPatterns.RATE_LIMIT_EXCEEDED_REGEX)
+          .build();
+
+  public static final ExceptionHandler BIGQUERY_EXCEPTION_HANDLER =
+      ExceptionHandler.newBuilder()
+          .abortOn(RuntimeException.class)
+          .retryOn(
+              java.net.ConnectException
+                  .class) // Retry on connection failures due to transient network issues.
+          .retryOn(
+              java.net.UnknownHostException
+                  .class) // Retry to recover from temporary DNS resolution failures.
+          .addInterceptors(EXCEPTION_HANDLER_INTERCEPTOR)
+          .build();
+
+  /** Constructs a client of the Google BigQuery service. */
+  public BigQueryMetastoreClientImpl(BigQueryOptions options)
+      throws IOException, GeneralSecurityException {
+    // Initialize client that will be used to send requests. This client only needs to be created
+    // once, and can be reused for multiple requests
+    HttpCredentialsAdapter httpCredentialsAdapter =
+        new HttpCredentialsAdapter(
+            GoogleCredentials.getApplicationDefault().createScoped(BigqueryScopes.all()));
+    this.client =
+        new Bigquery.Builder(
+                GoogleNetHttpTransport.newTrustedTransport(),
+                GsonFactory.getDefaultInstance(),
+                httpRequest -> {
+                  httpCredentialsAdapter.initialize(httpRequest);
+                  // Instead of throwing exceptions, analyze the HttpResponse object and inspect its
+                  // status code. This will allow BigQuery API errors to be converted into Iceberg
+                  // exceptions.
+                  httpRequest.setThrowExceptionOnExecuteError(false);
+                })
+            .setApplicationName("BigQuery Metastore Iceberg Catalog Plugin")
+            .build();
+    this.bigqueryOptions = options;
+  }
+
+  @Override
+  public Dataset create(Dataset dataset) {
+    Dataset response = null;
+    try {
+      response =
+          BigQueryRetryHelper.runWithRetries(
+              () -> internalCreate(dataset),
+              bigqueryOptions.getRetrySettings(),
+              BIGQUERY_EXCEPTION_HANDLER,
+              bigqueryOptions.getClock(),
+              DEFAULT_RETRY_CONFIG);
+    } catch (BigQueryRetryHelper.BigQueryRetryHelperException e) {
+      handleBigQueryRetryException(e);
+    }
+    return response;
+  }
+
+  @SuppressWarnings("FormatStringAnnotation")
+  private Dataset internalCreate(Dataset dataset) {
+    try {
+      HttpResponse response =
+          client
+              .datasets()
+              .insert(dataset.getDatasetReference().getProjectId(), dataset)
+              .executeUnparsed();
+      return convertExceptionIfUnsuccessful(response).parseAs(Dataset.class);
+    } catch (IOException e) {
+      throw new RuntimeIOException(e);
+    } catch (AlreadyExistsException e) {
+      throw new AlreadyExistsException("Namespace already exists: " + dataset.getId());
+    }
+  }
+
+  @Override
+  @SuppressWarnings("FormatStringAnnotation")
+  public Dataset load(DatasetReference datasetReference) {
+    try {
+      HttpResponse response =
+          client
+              .datasets()
+              .get(datasetReference.getProjectId(), datasetReference.getDatasetId())
+              .executeUnparsed();
+      if (response.getStatusCode() == HttpStatusCodes.STATUS_CODE_NOT_FOUND) {
+        throw new NoSuchNamespaceException(
+            "Namespace does not exist: " + datasetReference.getDatasetId());
+      }
+
+      return convertExceptionIfUnsuccessful(response).parseAs(Dataset.class);
+    } catch (IOException e) {
+      throw new RuntimeIOException(e);
+    }
+  }
+
+  @Override
+  @SuppressWarnings("FormatStringAnnotation")
+  public void delete(DatasetReference datasetReference) {
+    try {
+      HttpResponse response =
+          client
+              .datasets()
+              .delete(datasetReference.getProjectId(), datasetReference.getDatasetId())
+              .executeUnparsed();
+      if (response.getStatusCode() == HttpStatusCodes.STATUS_CODE_NOT_FOUND) {
+        throw new NoSuchNamespaceException(
+            "Namespace does not exist: " + datasetReference.getDatasetId());
+      }
+
+      convertExceptionIfUnsuccessful(response);
+    } catch (IOException e) {
+      throw new RuntimeIOException(e);
+    }
+  }
+
+  @Override
+  public Dataset setParameters(DatasetReference datasetReference, Map<String, String> parameters) {
+    Dataset dataset = load(datasetReference);
+    ExternalCatalogDatasetOptions externalCatalogDatasetOptions =
+        dataset.getExternalCatalogDatasetOptions() == null
+            ? new ExternalCatalogDatasetOptions()
+            : dataset.getExternalCatalogDatasetOptions();
+    Map<String, String> finalParameters =
+        externalCatalogDatasetOptions.getParameters() == null
+            ? Maps.newHashMap()
+            : externalCatalogDatasetOptions.getParameters();
+    finalParameters.putAll(parameters);
+
+    dataset.setExternalCatalogDatasetOptions(
+        externalCatalogDatasetOptions.setParameters(finalParameters));
+
+    return internalUpdate(dataset);
+  }
+
+  @Override
+  public Dataset removeParameters(DatasetReference datasetReference, Set<String> parameters) {
+    Dataset dataset = load(datasetReference);
+    ExternalCatalogDatasetOptions externalCatalogDatasetOptions =
+        dataset.getExternalCatalogDatasetOptions() == null
+            ? new ExternalCatalogDatasetOptions()
+            : dataset.getExternalCatalogDatasetOptions();
+    Map<String, String> finalParameters =
+        externalCatalogDatasetOptions.getParameters() == null
+            ? Maps.newHashMap()
+            : externalCatalogDatasetOptions.getParameters();
+    parameters.forEach(finalParameters::remove);
+
+    dataset.setExternalCatalogDatasetOptions(
+        externalCatalogDatasetOptions.setParameters(finalParameters));
+
+    return internalUpdate(dataset);
+  }
+
+  @Override
+  public List<Datasets> list(String projectId) {
+    try {
+      String nextPageToken = null;
+      List<Datasets> datasets = Lists.newArrayList();
+      do {
+        HttpResponse pageResponse =
+            client.datasets().list(projectId).setPageToken(nextPageToken).executeUnparsed();
+        DatasetList result =
+            convertExceptionIfUnsuccessful(pageResponse).parseAs(DatasetList.class);
+        nextPageToken = result.getNextPageToken();
+        if (result.getDatasets() != null) {
+          datasets.addAll(result.getDatasets());
+        }
+
+      } while (nextPageToken != null && !nextPageToken.isEmpty());
+      return datasets;
+    } catch (IOException e) {
+      throw new RuntimeIOException(e);
+    }
+  }
+
+  @Override
+  public Table create(Table table) {
+    // Ensure it is an Iceberg table supported by the BigQuery metastore catalog.
+    validateTable(table);
+    // TODO: Ensure table creation is idempotent when handling retries.
+    Table response = null;
+    try {
+      response =
+          BigQueryRetryHelper.runWithRetries(
+              () -> internalCreate(table),
+              bigqueryOptions.getRetrySettings(),
+              BIGQUERY_EXCEPTION_HANDLER,
+              bigqueryOptions.getClock(),
+              DEFAULT_RETRY_CONFIG);
+    } catch (BigQueryRetryHelper.BigQueryRetryHelperException e) {
+      handleBigQueryRetryException(e);
+    }
+
+    return response;
+  }
+
+  @SuppressWarnings("FormatStringAnnotation")
+  private Table internalCreate(Table table) {
+    try {
+      HttpResponse response =
+          client
+              .tables()
+              .insert(
+                  Preconditions.checkNotNull(table.getTableReference()).getProjectId(),
+                  Preconditions.checkNotNull(table.getTableReference()).getDatasetId(),
+                  table)
+              .executeUnparsed();
+      return convertExceptionIfUnsuccessful(response).parseAs(Table.class);
+    } catch (IOException e) {
+      throw new RuntimeIOException(e);
+    } catch (AlreadyExistsException e) {
+      throw new AlreadyExistsException(String.format("Table %s already exists", table), e);
+    }
+  }
+
+  @Override
+  @SuppressWarnings("FormatStringAnnotation")
+  public Table load(TableReference tableReference) {
+    try {
+      HttpResponse response =
+          client
+              .tables()
+              .get(
+                  tableReference.getProjectId(),
+                  tableReference.getDatasetId(),
+                  tableReference.getTableId())
+              .executeUnparsed();
+      if (response.getStatusCode() == HttpStatusCodes.STATUS_CODE_NOT_FOUND) {
+        throw new NoSuchTableException(response.getStatusMessage());
+      }
+
+      return validateTable(convertExceptionIfUnsuccessful(response).parseAs(Table.class));
+    } catch (IOException e) {
+      throw new RuntimeIOException(e);
+    }
+  }
+
+  @Override
+  @SuppressWarnings("FormatStringAnnotation")
+  public Table update(TableReference tableReference, Table table) {
+    // Ensure it is an Iceberg table supported by the BQ metastore catalog.
+    validateTable(table);
+
+    ExternalCatalogTableOptions newExternalCatalogTableOptions =
+        new ExternalCatalogTableOptions()
+            .setStorageDescriptor(table.getExternalCatalogTableOptions().getStorageDescriptor())
+            .setConnectionId(table.getExternalCatalogTableOptions().getConnectionId())
+            .setParameters(table.getExternalCatalogTableOptions().getParameters());
+    Table updatedTable =
+        new Table()
+            .setExternalCatalogTableOptions(newExternalCatalogTableOptions)
+            // Must set the schema as null for using schema auto-detect.
+            .setSchema(Data.nullOf(TableSchema.class));
+
+    Table response = null;
+    try {
+      response =
+          BigQueryRetryHelper.runWithRetries(
+              () -> internalUpdate(tableReference, updatedTable, table.getEtag()),
+              bigqueryOptions.getRetrySettings(),
+              BIGQUERY_EXCEPTION_HANDLER,
+              bigqueryOptions.getClock(),
+              DEFAULT_RETRY_CONFIG);
+    } catch (BigQueryRetryHelper.BigQueryRetryHelperException e) {
+      handleBigQueryRetryException(e);
+    }
+
+    return response;
+  }
+
+  @SuppressWarnings("FormatStringAnnotation")

Review Comment:
   All of them can be removed and the exception instantiation can be updated to `throw new NoSuchTableException("%s", response.getStatusMessage())`. We also use the same approach in other places in the codebase, such as in `ErrorHandlers`



-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org

For queries about this service, please contact Infrastructure at:
users@infra.apache.org


---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For additional commands, e-mail: issues-help@iceberg.apache.org


Re: [PR] Catalog: Add BigQuery Metastore Catalog Support [iceberg]

Posted by "talatuyarer (via GitHub)" <gi...@apache.org>.
talatuyarer commented on code in PR #12808:
URL: https://github.com/apache/iceberg/pull/12808#discussion_r2064339989


##########
bigquery/src/test/java/org/apache/iceberg/gcp/bigquery/BigQueryMetastoreTestUtils.java:
##########
@@ -0,0 +1,87 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.iceberg.gcp.bigquery;
+
+import static org.apache.iceberg.types.Types.NestedField.required;
+import static org.assertj.core.api.Assertions.assertThat;
+
+import com.google.api.services.bigquery.model.ExternalCatalogTableOptions;
+import com.google.api.services.bigquery.model.StorageDescriptor;
+import com.google.api.services.bigquery.model.Table;
+import com.google.api.services.bigquery.model.TableReference;
+import java.io.File;
+import java.io.IOException;
+import java.util.Collections;
+import java.util.Optional;
+import org.apache.commons.io.FileUtils;
+import org.apache.commons.io.filefilter.TrueFileFilter;
+import org.apache.iceberg.Schema;
+import org.apache.iceberg.catalog.TableIdentifier;
+import org.apache.iceberg.types.Types;
+
+/** Test utility methods for BigQuery Metastore Iceberg catalog. */
+public final class BigQueryMetastoreTestUtils {
+  private BigQueryMetastoreTestUtils() {}
+
+  public static final String METADATA_LOCATION_PROP = "metadata_location";
+
+  public static Schema getTestSchema() {
+    return new Schema(
+        required(1, "id", Types.IntegerType.get(), "unique ID"),
+        required(2, "data", Types.StringType.get()));
+  }
+
+  public static Table createTestTable(
+      File tempFolder,
+      BigQueryMetastoreCatalog bigQueryMetastoreCatalog,
+      TableReference tableReference)
+      throws IOException {
+    Schema schema = getTestSchema();
+    TableIdentifier tableIdentifier =
+        TableIdentifier.of(tableReference.getDatasetId(), tableReference.getTableId());
+    String tableDir = tempFolder.toPath().resolve(tableReference.getTableId()).toString();
+
+    bigQueryMetastoreCatalog
+        .buildTable(tableIdentifier, schema)
+        .withLocation(tableDir)
+        .createTransaction()
+        .commitTransaction();
+
+    Optional<String> metadataLocation = getIcebergMetadataFilePath(tableDir);
+    assertThat(metadataLocation).isPresent();
+    return new Table()
+        .setTableReference(tableReference)
+        .setExternalCatalogTableOptions(
+            new ExternalCatalogTableOptions()
+                .setStorageDescriptor(new StorageDescriptor().setLocationUri(tableDir))
+                .setParameters(
+                    Collections.singletonMap(METADATA_LOCATION_PROP, metadataLocation.get())));
+  }
+
+  public static Optional<String> getIcebergMetadataFilePath(String tableDir) throws IOException {

Review Comment:
   The function finds metadata and create a Iceberg Table. We dont have Table object. I changed its name but kept function as is.



-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org

For queries about this service, please contact Infrastructure at:
users@infra.apache.org


---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For additional commands, e-mail: issues-help@iceberg.apache.org


Re: [PR] Catalog: Add BigQuery Metastore Catalog Support [iceberg]

Posted by "nastra (via GitHub)" <gi...@apache.org>.
nastra commented on code in PR #12808:
URL: https://github.com/apache/iceberg/pull/12808#discussion_r2066100567


##########
bigquery/src/test/java/org/apache/iceberg/gcp/bigquery/BigQueryTableOperationsTest.java:
##########
@@ -0,0 +1,274 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.iceberg.gcp.bigquery;
+
+import static org.apache.iceberg.gcp.bigquery.BigQueryMetastoreCatalog.PROJECT_ID;
+import static org.apache.iceberg.types.Types.NestedField.required;
+import static org.assertj.core.api.Assertions.assertThat;
+import static org.assertj.core.api.Assertions.assertThatThrownBy;
+import static org.mockito.Mockito.any;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.reset;
+import static org.mockito.Mockito.times;
+import static org.mockito.Mockito.verify;
+import static org.mockito.Mockito.when;
+
+import com.google.api.services.bigquery.model.Dataset;
+import com.google.api.services.bigquery.model.DatasetReference;
+import com.google.api.services.bigquery.model.ExternalCatalogDatasetOptions;
+import com.google.api.services.bigquery.model.ExternalCatalogTableOptions;
+import com.google.api.services.bigquery.model.StorageDescriptor;
+import com.google.api.services.bigquery.model.Table;
+import com.google.api.services.bigquery.model.TableReference;
+import java.io.File;
+import java.io.IOException;
+import java.util.Collections;
+import java.util.Optional;
+import org.apache.commons.io.FileUtils;
+import org.apache.commons.io.filefilter.TrueFileFilter;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.iceberg.CatalogProperties;
+import org.apache.iceberg.PartitionSpec;
+import org.apache.iceberg.Schema;
+import org.apache.iceberg.catalog.TableIdentifier;
+import org.apache.iceberg.exceptions.CommitFailedException;
+import org.apache.iceberg.exceptions.NoSuchTableException;
+import org.apache.iceberg.exceptions.ValidationException;
+import org.apache.iceberg.relocated.com.google.common.collect.ImmutableMap;
+import org.apache.iceberg.types.Types;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.io.TempDir;
+import org.mockito.ArgumentCaptor;
+
+public class BigQueryTableOperationsTest {
+
+  @TempDir private File tempFolder;
+  public static final String METADATA_LOCATION_PROP = "metadata_location";
+  private static final String GCP_PROJECT = "my-project";
+  private static final String GCP_REGION = "us";
+  private static final String DATASET_ID = "db";
+  private static final String TABLE_ID = "tbl";
+  private static final TableIdentifier SPARK_TABLE_ID = TableIdentifier.of(DATASET_ID, TABLE_ID);
+
+  private static final TableReference TABLE_REFERENCE =
+      new TableReference().setProjectId(GCP_PROJECT).setDatasetId(DATASET_ID).setTableId(TABLE_ID);
+
+  private final BigQueryMetastoreClient bigQueryMetaStoreClient =
+      mock(BigQueryMetastoreClient.class);
+
+  private BigQueryMetastoreCatalog bigQueryMetastoreCatalog;
+  private BigQueryTableOperations tableOps;
+
+  @BeforeEach
+  public void before() {
+    this.bigQueryMetastoreCatalog = new BigQueryMetastoreCatalog();
+    this.bigQueryMetastoreCatalog.setConf(new Configuration());
+    String warehouseLocation = tempFolder.toPath().resolve("hive-warehouse").toString();
+
+    bigQueryMetastoreCatalog.initialize(
+        "CATALOG_ID",
+        ImmutableMap.of(
+            PROJECT_ID,
+            GCP_PROJECT,
+            CatalogProperties.WAREHOUSE_LOCATION,
+            warehouseLocation,
+            CatalogProperties.FILE_IO_IMPL,
+            "org.apache.iceberg.hadoop.HadoopFileIO"),
+        GCP_PROJECT,
+        GCP_REGION,
+        bigQueryMetaStoreClient);
+    this.tableOps = (BigQueryTableOperations) bigQueryMetastoreCatalog.newTableOps(SPARK_TABLE_ID);
+  }
+
+  @Test
+  public void testFetchLatestMetadataFromBigQuery() throws Exception {
+    Table createdTable = createTestTable();
+    reset(bigQueryMetaStoreClient);
+    when(bigQueryMetaStoreClient.load(TABLE_REFERENCE)).thenReturn(createdTable);
+
+    tableOps.refresh();
+    assertThat(
+            createdTable
+                .getExternalCatalogTableOptions()
+                .getParameters()
+                .getOrDefault(METADATA_LOCATION_PROP, ""))
+        .isEqualTo(tableOps.currentMetadataLocation());
+
+    reset(bigQueryMetaStoreClient);
+    when(bigQueryMetaStoreClient.load(TABLE_REFERENCE))
+        .thenThrow(new NoSuchTableException("error message getTable"));
+    // Refresh fails when table is not found but metadata already presents.
+    assertThatThrownBy(() -> tableOps.refresh())
+        .isInstanceOf(NoSuchTableException.class)
+        .hasMessageContaining("error message getTable");
+  }
+
+  @Test
+  public void testFailForNonIcebergTable() {
+    when(bigQueryMetaStoreClient.load(TABLE_REFERENCE))
+        .thenReturn(new Table().setTableReference(TABLE_REFERENCE));
+
+    assertThatThrownBy(() -> tableOps.refresh())
+        .isInstanceOf(ValidationException.class)
+        .hasMessageContaining("metadata location not found");
+  }
+
+  @Test
+  public void testNoOpWhenMetadataAndTableNotFound() {
+    when(bigQueryMetaStoreClient.load(TABLE_REFERENCE))
+        .thenThrow(new NoSuchTableException("error message getTable"));
+    // Table not found won't cause errors when the metadata is null.
+    assertThat(tableOps.currentMetadataLocation()).isNull();
+    tableOps.refresh();
+  }
+
+  @Test
+  public void testTableNameAsExpected() {
+    assertThat(tableOps.tableName()).isEqualTo("db.tbl");
+  }
+
+  @Test
+  public void testUseEtagForUpdateTable() throws Exception {
+    Table tableWithEtag = createTestTable().setEtag("etag");
+    reset(bigQueryMetaStoreClient);
+    when(bigQueryMetaStoreClient.load(TABLE_REFERENCE)).thenReturn(tableWithEtag, tableWithEtag);

Review Comment:
   ```suggestion
       when(bigQueryMetaStoreClient.load(TABLE_REFERENCE)).thenReturn(tableWithEtag);
   ```
   
   I don't think the second one is actually needed



-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org

For queries about this service, please contact Infrastructure at:
users@infra.apache.org


---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For additional commands, e-mail: issues-help@iceberg.apache.org


Re: [PR] Catalog: Add BigQuery Metastore Catalog Support [iceberg]

Posted by "talatuyarer (via GitHub)" <gi...@apache.org>.
talatuyarer commented on code in PR #12808:
URL: https://github.com/apache/iceberg/pull/12808#discussion_r2067700635


##########
bigquery/src/main/java/org/apache/iceberg/gcp/bigquery/BigQueryMetastoreCatalog.java:
##########
@@ -0,0 +1,405 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.iceberg.gcp.bigquery;
+
+import com.google.api.services.bigquery.model.Dataset;
+import com.google.api.services.bigquery.model.DatasetList.Datasets;
+import com.google.api.services.bigquery.model.DatasetReference;
+import com.google.api.services.bigquery.model.ExternalCatalogDatasetOptions;
+import com.google.api.services.bigquery.model.TableReference;
+import com.google.cloud.ServiceOptions;
+import com.google.cloud.bigquery.BigQueryOptions;
+import java.io.IOException;
+import java.security.GeneralSecurityException;
+import java.util.List;
+import java.util.Locale;
+import java.util.Map;
+import java.util.Objects;
+import java.util.Set;
+import org.apache.hadoop.conf.Configurable;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.iceberg.BaseMetastoreCatalog;
+import org.apache.iceberg.CatalogProperties;
+import org.apache.iceberg.CatalogUtil;
+import org.apache.iceberg.TableMetadata;
+import org.apache.iceberg.TableOperations;
+import org.apache.iceberg.catalog.Namespace;
+import org.apache.iceberg.catalog.SupportsNamespaces;
+import org.apache.iceberg.catalog.TableIdentifier;
+import org.apache.iceberg.exceptions.AlreadyExistsException;
+import org.apache.iceberg.exceptions.NoSuchNamespaceException;
+import org.apache.iceberg.exceptions.NoSuchTableException;
+import org.apache.iceberg.exceptions.ServiceFailureException;
+import org.apache.iceberg.exceptions.ValidationException;
+import org.apache.iceberg.io.FileIO;
+import org.apache.iceberg.relocated.com.google.common.annotations.VisibleForTesting;
+import org.apache.iceberg.relocated.com.google.common.base.Preconditions;
+import org.apache.iceberg.relocated.com.google.common.base.Strings;
+import org.apache.iceberg.relocated.com.google.common.collect.ImmutableList;
+import org.apache.iceberg.relocated.com.google.common.collect.ImmutableMap;
+import org.apache.iceberg.relocated.com.google.common.collect.Maps;
+import org.apache.iceberg.util.LocationUtil;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/** Iceberg Bigquery Metastore Catalog implementation. */
+public class BigQueryMetastoreCatalog extends BaseMetastoreCatalog
+    implements SupportsNamespaces, Configurable {
+
+  // User provided properties.
+  public static final String PROJECT_ID = "gcp-project";
+  public static final String GCP_LOCATION = "gcp-location";
+  public static final String LIST_ALL_TABLES = "list-all-tables";
+
+  public static final String HIVE_METASTORE_WAREHOUSE_DIR = "hive.metastore.warehouse.dir";
+
+  private static final Logger LOG = LoggerFactory.getLogger(BigQueryMetastoreCatalog.class);
+
+  private static final String DEFAULT_GCP_LOCATION = "us";
+
+  private String catalogName;
+  private Map<String, String> catalogProperties;
+  private FileIO fileIO;
+  private Configuration conf;
+  private String projectId;
+  private String location;
+  private BigQueryMetastoreClient client;
+  private boolean listAllTables;
+
+  // Must have a no-arg constructor to be dynamically loaded
+  // initialize(String name, Map<String, String> properties) will be called to complete
+  // initialization
+  public BigQueryMetastoreCatalog() {}
+
+  @Override
+  public void initialize(String inputName, Map<String, String> properties) {
+    Preconditions.checkArgument(
+        properties.containsKey(PROJECT_ID),
+        "Invalid GCP project: %s must be specified",
+        PROJECT_ID);
+
+    projectId = properties.get(PROJECT_ID);
+    location = properties.getOrDefault(GCP_LOCATION, DEFAULT_GCP_LOCATION);
+
+    BigQueryOptions options =
+        BigQueryOptions.newBuilder()
+            .setProjectId(projectId)
+            .setLocation(location)
+            .setRetrySettings(ServiceOptions.getDefaultRetrySettings())
+            .build();
+
+    try {
+      client = new BigQueryMetastoreClientImpl(options);
+    } catch (IOException e) {
+      throw new ServiceFailureException(e, "Creating BigQuery client failed");
+    } catch (GeneralSecurityException e) {
+      throw new ValidationException(e, "Creating BigQuery client failed due to a security issue");
+    }
+
+    initialize(inputName, properties, projectId, location, client);
+  }
+
+  @VisibleForTesting
+  void initialize(
+      String inputName,
+      Map<String, String> properties,
+      String initialProjectId,
+      String initialLocation,
+      BigQueryMetastoreClient bigQueryMetaStoreClient) {
+    Preconditions.checkArgument(bigQueryMetaStoreClient != null, "Invalid BigQuery client: null");
+    this.catalogName = inputName;
+    this.catalogProperties = ImmutableMap.copyOf(properties);
+    this.projectId = initialProjectId;
+    this.location = initialLocation;
+    this.client = bigQueryMetaStoreClient;
+
+    if (this.conf == null) {
+      LOG.warn("No configuration was set, using the default environment Configuration");
+      this.conf = new Configuration();
+    }
+
+    LOG.info("Using BigQuery Metastore Iceberg Catalog: {}", inputName);
+
+    if (properties.containsKey(CatalogProperties.WAREHOUSE_LOCATION)) {
+      this.conf.set(
+          HIVE_METASTORE_WAREHOUSE_DIR,
+          LocationUtil.stripTrailingSlash(properties.get(CatalogProperties.WAREHOUSE_LOCATION)));
+    }
+
+    String fileIoImpl =
+        properties.getOrDefault(
+            CatalogProperties.FILE_IO_IMPL, "org.apache.iceberg.hadoop.HadoopFileIO");
+    this.fileIO = CatalogUtil.loadFileIO(fileIoImpl, properties, conf);
+
+    this.listAllTables = Boolean.parseBoolean(properties.getOrDefault(LIST_ALL_TABLES, "true"));
+  }
+
+  @Override
+  protected TableOperations newTableOps(TableIdentifier identifier) {
+    return new BigQueryTableOperations(client, fileIO, toTableReference(identifier), conf);
+  }
+
+  @Override
+  protected String defaultWarehouseLocation(TableIdentifier identifier) {
+    String locationUri = null;
+    DatasetReference datasetReference = toDatasetReference(identifier.namespace());
+    Dataset dataset = client.load(datasetReference);
+    if (dataset != null && dataset.getExternalCatalogDatasetOptions() != null) {
+      locationUri = dataset.getExternalCatalogDatasetOptions().getDefaultStorageLocationUri();
+    }
+
+    return String.format(
+        "%s/%s",
+        Strings.isNullOrEmpty(locationUri)
+            ? createDefaultStorageLocationUri(datasetReference.getDatasetId())
+            : LocationUtil.stripTrailingSlash(locationUri),
+        identifier.name());
+  }
+
+  @Override
+  public List<TableIdentifier> listTables(Namespace namespace) {
+    validateNamespace(namespace);
+
+    return client.list(toDatasetReference(namespace), listAllTables).stream()
+        .map(
+            table -> TableIdentifier.of(namespace.level(0), table.getTableReference().getTableId()))
+        .collect(ImmutableList.toImmutableList());
+  }
+
+  @Override
+  public boolean dropTable(TableIdentifier identifier, boolean purge) {
+    try {
+      TableOperations ops = newTableOps(identifier);
+      TableMetadata lastMetadata = ops.current();
+
+      client.delete(toTableReference(identifier));
+
+      if (purge && lastMetadata != null) {
+        CatalogUtil.dropTableData(ops.io(), lastMetadata);
+      }
+    } catch (NoSuchTableException e) {
+      return false;
+    }
+
+    return true;
+  }
+
+  @Override
+  public void renameTable(TableIdentifier from, TableIdentifier to) {
+    validateNamespace(from.namespace());
+    validateNamespace(to.namespace());
+
+    if (!namespaceExists(to.namespace())) {
+      throw new NoSuchNamespaceException(
+          "Cannot rename %s to %s. Namespace does not exist: %s",
+          from.name(), to.name(), to.namespace());
+    }
+
+    if (!namespaceExists(from.namespace())) {
+      throw new NoSuchNamespaceException(
+          "Cannot rename %s to %s. Namespace does not exist: %s",
+          from.name(), to.name(), from.namespace());
+    }
+
+    if (!tableExists(from)) {
+      throw new NoSuchTableException(
+          "Cannot rename %s to %s. Table does not exist: %s", from.name(), to.name(), from.name());
+    }
+
+    if (!from.namespace().equals(to.namespace())) {
+      throw new ValidationException(
+          "Cannot rename %s to %s. New table name must be in the same namespace",
+          from.name(), to.name());
+    }
+
+    if (tableExists(to)) {
+      throw new AlreadyExistsException(
+          "Cannot rename %s to %s. Table already exists: %s", from.name(), to.name(), to.name());
+    }
+
+    // TODO: Enable once supported by BigQuery API.
+    throw new UnsupportedOperationException("Table rename operation is unsupported.");

Review Comment:
   Sounds good I disabled catalogtests for rename.



-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org

For queries about this service, please contact Infrastructure at:
users@infra.apache.org


---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For additional commands, e-mail: issues-help@iceberg.apache.org


Re: [PR] Catalog: Add BigQuery Metastore Catalog Support [iceberg]

Posted by "talatuyarer (via GitHub)" <gi...@apache.org>.
talatuyarer commented on code in PR #12808:
URL: https://github.com/apache/iceberg/pull/12808#discussion_r2067871268


##########
bigquery/src/main/java/org/apache/iceberg/gcp/bigquery/BigQueryMetastoreCatalog.java:
##########
@@ -0,0 +1,394 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.iceberg.gcp.bigquery;
+
+import com.google.api.services.bigquery.model.Dataset;
+import com.google.api.services.bigquery.model.DatasetList.Datasets;
+import com.google.api.services.bigquery.model.DatasetReference;
+import com.google.api.services.bigquery.model.ExternalCatalogDatasetOptions;
+import com.google.api.services.bigquery.model.TableReference;
+import com.google.cloud.ServiceOptions;
+import com.google.cloud.bigquery.BigQueryOptions;
+import java.io.IOException;
+import java.security.GeneralSecurityException;
+import java.util.List;
+import java.util.Locale;
+import java.util.Map;
+import java.util.Objects;
+import java.util.Set;
+import org.apache.hadoop.conf.Configurable;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.iceberg.BaseMetastoreCatalog;
+import org.apache.iceberg.CatalogProperties;
+import org.apache.iceberg.CatalogUtil;
+import org.apache.iceberg.TableMetadata;
+import org.apache.iceberg.TableOperations;
+import org.apache.iceberg.catalog.Namespace;
+import org.apache.iceberg.catalog.SupportsNamespaces;
+import org.apache.iceberg.catalog.TableIdentifier;
+import org.apache.iceberg.exceptions.AlreadyExistsException;
+import org.apache.iceberg.exceptions.NoSuchNamespaceException;
+import org.apache.iceberg.exceptions.NoSuchTableException;
+import org.apache.iceberg.exceptions.ServiceFailureException;
+import org.apache.iceberg.exceptions.ValidationException;
+import org.apache.iceberg.io.FileIO;
+import org.apache.iceberg.relocated.com.google.common.annotations.VisibleForTesting;
+import org.apache.iceberg.relocated.com.google.common.base.Preconditions;
+import org.apache.iceberg.relocated.com.google.common.base.Strings;
+import org.apache.iceberg.relocated.com.google.common.collect.ImmutableList;
+import org.apache.iceberg.relocated.com.google.common.collect.ImmutableMap;
+import org.apache.iceberg.relocated.com.google.common.collect.Maps;
+import org.apache.iceberg.util.LocationUtil;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/** Iceberg Bigquery Metastore Catalog implementation. */
+public class BigQueryMetastoreCatalog extends BaseMetastoreCatalog
+    implements SupportsNamespaces, Configurable {
+
+  // User provided properties.
+  public static final String PROJECT_ID = "gcp-project";
+  public static final String GCP_LOCATION = "gcp-location";
+  public static final String LIST_ALL_TABLES = "list-all-tables";
+
+  public static final String HIVE_METASTORE_WAREHOUSE_DIR = "hive.metastore.warehouse.dir";
+
+  private static final Logger LOG = LoggerFactory.getLogger(BigQueryMetastoreCatalog.class);
+
+  private static final String DEFAULT_GCP_LOCATION = "us";
+
+  private String catalogName;
+  private Map<String, String> catalogProperties;
+  private FileIO fileIO;
+  private Configuration conf;
+  private String projectId;
+  private String location;
+  private BigQueryMetastoreClient client;
+  private boolean listAllTables;
+
+  // Must have a no-arg constructor to be dynamically loaded
+  // initialize(String name, Map<String, String> properties) will be called to complete
+  // initialization
+  public BigQueryMetastoreCatalog() {}
+
+  @Override
+  public void initialize(String inputName, Map<String, String> properties) {
+    Preconditions.checkArgument(
+        properties.containsKey(PROJECT_ID), "GCP project must be specified");
+
+    projectId = properties.get(PROJECT_ID);
+    location = properties.getOrDefault(GCP_LOCATION, DEFAULT_GCP_LOCATION);
+
+    BigQueryOptions options =
+        BigQueryOptions.newBuilder()
+            .setProjectId(projectId)
+            .setLocation(location)
+            .setRetrySettings(ServiceOptions.getDefaultRetrySettings())
+            .build();
+
+    try {
+      client = new BigQueryMetastoreClientImpl(options);
+    } catch (IOException e) {
+      throw new ServiceFailureException(e, "Creating BigQuery client failed");
+    } catch (GeneralSecurityException e) {
+      throw new ValidationException(e, "Creating BigQuery client failed due to a security issue");
+    }
+
+    initialize(inputName, properties, projectId, location, client);
+  }
+
+  @VisibleForTesting
+  void initialize(
+      String inputName,
+      Map<String, String> properties,
+      String initialProjectId,
+      String initialLocation,
+      BigQueryMetastoreClient bigQueryMetaStoreClient) {
+    Preconditions.checkArgument(bigQueryMetaStoreClient != null, "BigQuery client can not be null");
+    this.catalogName = inputName;
+    this.catalogProperties = ImmutableMap.copyOf(properties);
+    this.projectId = initialProjectId;
+    this.location = initialLocation;
+    this.client = bigQueryMetaStoreClient;
+
+    if (this.conf == null) {
+      LOG.warn("No configuration was set, using the default environment Configuration");
+      this.conf = new Configuration();
+    }
+
+    LOG.info("Using BigQuery Metastore Iceberg Catalog: {}", inputName);
+
+    if (properties.containsKey(CatalogProperties.WAREHOUSE_LOCATION)) {
+      this.conf.set(
+          HIVE_METASTORE_WAREHOUSE_DIR,
+          LocationUtil.stripTrailingSlash(properties.get(CatalogProperties.WAREHOUSE_LOCATION)));
+    }
+
+    String fileIoImpl =
+        properties.getOrDefault(
+            CatalogProperties.FILE_IO_IMPL, "org.apache.iceberg.hadoop.HadoopFileIO");
+    this.fileIO = CatalogUtil.loadFileIO(fileIoImpl, properties, conf);
+
+    this.listAllTables = Boolean.parseBoolean(properties.getOrDefault(LIST_ALL_TABLES, "true"));
+  }
+
+  @Override
+  protected TableOperations newTableOps(TableIdentifier identifier) {
+    return new BigQueryTableOperations(client, fileIO, toTableReference(identifier), conf);
+  }
+
+  @Override
+  protected String defaultWarehouseLocation(TableIdentifier identifier) {
+    String locationUri = null;
+    DatasetReference datasetReference = toDatasetReference(identifier.namespace());
+    Dataset dataset = client.load(datasetReference);
+    if (dataset != null && dataset.getExternalCatalogDatasetOptions() != null) {
+      locationUri = dataset.getExternalCatalogDatasetOptions().getDefaultStorageLocationUri();
+    }
+
+    return String.format(
+        "%s/%s",
+        Strings.isNullOrEmpty(locationUri)
+            ? createDefaultStorageLocationUri(datasetReference.getDatasetId())
+            : LocationUtil.stripTrailingSlash(locationUri),
+        identifier.name());
+  }
+
+  @Override
+  public List<TableIdentifier> listTables(Namespace namespace) {
+    validateNamespace(namespace);
+
+    return client.list(toDatasetReference(namespace), listAllTables).stream()
+        .map(
+            table -> TableIdentifier.of(namespace.level(0), table.getTableReference().getTableId()))
+        .collect(ImmutableList.toImmutableList());
+  }
+
+  @Override
+  public boolean dropTable(TableIdentifier identifier, boolean purge) {
+    try {
+      TableOperations ops = newTableOps(identifier);
+      TableMetadata lastMetadata = ops.current();
+
+      client.delete(toTableReference(identifier));
+
+      if (purge && lastMetadata != null) {
+        CatalogUtil.dropTableData(ops.io(), lastMetadata);
+      }
+    } catch (NoSuchTableException e) {
+      return false;
+    }
+
+    return true;
+  }
+
+  @Override
+  public void renameTable(TableIdentifier from, TableIdentifier to) {
+    validateNamespace(from.namespace());
+    validateNamespace(to.namespace());
+
+    if (!namespaceExists(to.namespace())) {
+      throw new NoSuchNamespaceException("Namespace does not exist: %s", to.namespace());
+    }
+
+    if (!namespaceExists(from.namespace())) {
+      throw new NoSuchNamespaceException("Namespace does not exist: %s", from.namespace());
+    }
+
+    // Check if the source table exists
+    if (!tableExists(from)) {
+      throw new NoSuchTableException("Table does not exist: %s", from.name());
+    }
+
+    if (!from.namespace().equals(to.namespace())) {
+      throw new ValidationException("New table name must be in the same namespace");
+    }
+
+    // Check if the destination table already exists
+    if (tableExists(to)) {
+      throw new AlreadyExistsException("Table already exists: %s", to.name());
+    }
+
+    // TODO: Enable once supported by BigQuery API.
+    throw new UnsupportedOperationException("Table rename operation is unsupported.");
+  }
+
+  @Override
+  public void createNamespace(Namespace namespace, Map<String, String> metadata) {
+    Dataset builder = new Dataset();
+    DatasetReference datasetReference = toDatasetReference(namespace);
+    builder.setLocation(this.location);
+    builder.setDatasetReference(datasetReference);
+    builder.setExternalCatalogDatasetOptions(
+        BigQueryMetastoreUtils.createExternalCatalogDatasetOptions(
+            createDefaultStorageLocationUri(datasetReference.getDatasetId()), metadata));
+
+    client.create(builder);
+  }
+
+  @Override
+  public List<Namespace> listNamespaces() {
+    try {
+      return listNamespaces(Namespace.empty());
+    } catch (NoSuchNamespaceException e) {
+      return ImmutableList.of();
+    }
+  }
+
+  /**
+   * Since this catalog only supports one-level namespaces, it always returns an empty list unless
+   * passed an empty namespace to list all namespaces within the catalog.
+   */
+  @Override
+  public List<Namespace> listNamespaces(Namespace namespace) {
+    List<Datasets> allDatasets = client.list(projectId);
+
+    ImmutableList<Namespace> namespaces =
+        allDatasets.stream()
+            .map(BigQueryMetastoreCatalog::toNamespace)
+            .collect(ImmutableList.toImmutableList());
+    if (namespaces.isEmpty()) {
+      throw new NoSuchNamespaceException("Namespace does not exist: %s", namespace);
+    }
+
+    return namespaces;
+  }
+
+  @Override
+  public boolean dropNamespace(Namespace namespace) {
+    try {
+      client.delete(toDatasetReference(namespace));
+      // We don't delete the data folder for safety, which aligns with Hive Metastore's default
+      // behavior.
+      // We can support database or catalog level config controlling file deletion in the future.
+      return true;
+    } catch (NoSuchNamespaceException e) {
+      return false;
+    }
+  }
+
+  @Override
+  public boolean setProperties(Namespace namespace, Map<String, String> properties) {
+    Dataset dataset = client.load(toDatasetReference(namespace));
+
+    ExternalCatalogDatasetOptions existingOptions = dataset.getExternalCatalogDatasetOptions();
+    Map<String, String> existingParameters =
+        existingOptions != null ? existingOptions.getParameters() : null;
+
+    Map<String, String> newParameters = Maps.newHashMap();
+    if (existingParameters != null) {
+      newParameters.putAll(existingParameters);
+    }
+
+    newParameters.putAll(properties);
+
+    if (Objects.equals(existingParameters, newParameters)) {
+      // No change in parameters detected
+      return false;
+    }
+
+    client.setParameters(toDatasetReference(namespace), properties);
+    return true;
+  }
+
+  @Override
+  public boolean removeProperties(Namespace namespace, Set<String> properties) {
+    client.removeParameters(toDatasetReference(namespace), properties);
+    return true;
+  }
+
+  @SuppressWarnings("FormatStringAnnotation")
+  @Override
+  public Map<String, String> loadNamespaceMetadata(Namespace namespace) {
+    try {
+      return toMetadata(client.load(toDatasetReference(namespace)));
+    } catch (IllegalArgumentException e) {
+      throw new NoSuchNamespaceException(e.getMessage());
+    }
+  }
+
+  @Override
+  public String name() {
+    return catalogName;
+  }
+
+  @Override
+  protected Map<String, String> properties() {
+    return catalogProperties == null ? ImmutableMap.of() : catalogProperties;
+  }
+
+  @Override
+  public void setConf(Configuration conf) {
+    this.conf = new Configuration(conf);
+  }
+
+  @Override
+  public Configuration getConf() {
+    return this.conf;
+  }
+
+  private String createDefaultStorageLocationUri(String dbId) {
+    String warehouseLocation = conf.get(HIVE_METASTORE_WAREHOUSE_DIR);
+    Preconditions.checkNotNull(warehouseLocation, "Data warehouse location is not set");
+    return String.format("%s/%s.db", LocationUtil.stripTrailingSlash(warehouseLocation), dbId);
+  }
+
+  private static Namespace toNamespace(Datasets dataset) {

Review Comment:
   Thank you for solution.



-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org

For queries about this service, please contact Infrastructure at:
users@infra.apache.org


---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For additional commands, e-mail: issues-help@iceberg.apache.org


Re: [PR] Catalog: Add BigQuery Metastore Catalog Support [iceberg]

Posted by "talatuyarer (via GitHub)" <gi...@apache.org>.
talatuyarer commented on code in PR #12808:
URL: https://github.com/apache/iceberg/pull/12808#discussion_r2071957199


##########
bigquery/src/main/java/org/apache/iceberg/gcp/bigquery/BigQueryMetastoreClient.java:
##########
@@ -0,0 +1,121 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.iceberg.gcp.bigquery;
+
+import com.google.api.services.bigquery.model.Dataset;
+import com.google.api.services.bigquery.model.DatasetList.Datasets;
+import com.google.api.services.bigquery.model.DatasetReference;
+import com.google.api.services.bigquery.model.Table;
+import com.google.api.services.bigquery.model.TableList.Tables;
+import com.google.api.services.bigquery.model.TableReference;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+
+/**
+ * A client of Google BigQuery Metastore functions over the BigQuery service. Uses the Google
+ * BigQuery API.
+ */
+public interface BigQueryMetastoreClient {

Review Comment:
   Done. 



-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org

For queries about this service, please contact Infrastructure at:
users@infra.apache.org


---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For additional commands, e-mail: issues-help@iceberg.apache.org


Re: [PR] Catalog: Add BigQuery Metastore Catalog Support [iceberg]

Posted by "danielcweeks (via GitHub)" <gi...@apache.org>.
danielcweeks commented on code in PR #12808:
URL: https://github.com/apache/iceberg/pull/12808#discussion_r2072158736


##########
bigquery/src/main/java/org/apache/iceberg/gcp/bigquery/BigQueryMetastoreCatalog.java:
##########
@@ -0,0 +1,366 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.iceberg.gcp.bigquery;
+
+import com.google.api.services.bigquery.model.Dataset;
+import com.google.api.services.bigquery.model.DatasetList.Datasets;
+import com.google.api.services.bigquery.model.DatasetReference;
+import com.google.api.services.bigquery.model.ExternalCatalogDatasetOptions;
+import com.google.api.services.bigquery.model.TableReference;
+import com.google.cloud.ServiceOptions;
+import com.google.cloud.bigquery.BigQueryOptions;
+import java.io.IOException;
+import java.io.UncheckedIOException;
+import java.security.GeneralSecurityException;
+import java.util.List;
+import java.util.Locale;
+import java.util.Map;
+import java.util.Objects;
+import java.util.Set;
+import org.apache.hadoop.conf.Configurable;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.iceberg.BaseMetastoreCatalog;
+import org.apache.iceberg.CatalogProperties;
+import org.apache.iceberg.CatalogUtil;
+import org.apache.iceberg.TableMetadata;
+import org.apache.iceberg.TableOperations;
+import org.apache.iceberg.catalog.Namespace;
+import org.apache.iceberg.catalog.SupportsNamespaces;
+import org.apache.iceberg.catalog.TableIdentifier;
+import org.apache.iceberg.exceptions.NoSuchNamespaceException;
+import org.apache.iceberg.exceptions.NoSuchTableException;
+import org.apache.iceberg.io.FileIO;
+import org.apache.iceberg.relocated.com.google.common.annotations.VisibleForTesting;
+import org.apache.iceberg.relocated.com.google.common.base.Preconditions;
+import org.apache.iceberg.relocated.com.google.common.base.Strings;
+import org.apache.iceberg.relocated.com.google.common.collect.ImmutableList;
+import org.apache.iceberg.relocated.com.google.common.collect.ImmutableMap;
+import org.apache.iceberg.relocated.com.google.common.collect.Maps;
+import org.apache.iceberg.util.LocationUtil;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/** Iceberg Bigquery Metastore Catalog implementation. */
+public class BigQueryMetastoreCatalog extends BaseMetastoreCatalog
+    implements SupportsNamespaces, Configurable {
+
+  // User provided properties.
+  public static final String PROJECT_ID = "gcp.bigquery.project-id";
+  public static final String GCP_LOCATION = "gcp.bigquery.location";
+  public static final String LIST_ALL_TABLES = "gcp.bigquery.list-all-tables";
+
+  private static final Logger LOG = LoggerFactory.getLogger(BigQueryMetastoreCatalog.class);
+
+  private static final String DEFAULT_GCP_LOCATION = "us";
+
+  private String catalogName;
+  private Map<String, String> catalogProperties;
+  private FileIO fileIO;
+  private Configuration conf;
+  private String projectId;
+  private String projectLocation;
+  private BigQueryMetastoreClient client;
+  private boolean listAllTables;
+  private String warehouseLocation;
+
+  public BigQueryMetastoreCatalog() {}
+
+  @Override
+  public void initialize(String name, Map<String, String> properties) {
+    Preconditions.checkArgument(
+        properties.containsKey(PROJECT_ID),
+        "Invalid GCP project: %s must be specified",
+        PROJECT_ID);
+
+    this.projectId = properties.get(PROJECT_ID);
+    this.projectLocation = properties.getOrDefault(GCP_LOCATION, DEFAULT_GCP_LOCATION);
+
+    BigQueryOptions options =
+        BigQueryOptions.newBuilder()
+            .setProjectId(projectId)
+            .setLocation(projectLocation)
+            .setRetrySettings(ServiceOptions.getDefaultRetrySettings())
+            .build();
+
+    try {
+      client = new BigQueryMetastoreClientImpl(options);
+    } catch (IOException e) {
+      throw new UncheckedIOException("Creating BigQuery client failed", e);
+    } catch (GeneralSecurityException e) {
+      throw new RuntimeException("Creating BigQuery client failed due to a security issue", e);
+    }
+
+    initialize(name, properties, projectId, projectLocation, client);
+  }
+
+  @VisibleForTesting
+  void initialize(
+      String name,
+      Map<String, String> properties,
+      String initialProjectId,
+      String initialLocation,
+      BigQueryMetastoreClient bigQueryMetaStoreClient) {
+    Preconditions.checkArgument(bigQueryMetaStoreClient != null, "Invalid BigQuery client: null");
+    this.catalogName = name;
+    this.catalogProperties = ImmutableMap.copyOf(properties);
+    this.projectId = initialProjectId;
+    this.projectLocation = initialLocation;
+    this.client = bigQueryMetaStoreClient;
+
+    if (this.conf == null) {
+      LOG.warn("No configuration was set, using the default environment Configuration");
+      this.conf = new Configuration();
+    }
+
+    LOG.info("Using BigQuery Metastore Iceberg Catalog: {}", name);
+
+    if (properties.containsKey(CatalogProperties.WAREHOUSE_LOCATION)) {
+      this.warehouseLocation =
+          LocationUtil.stripTrailingSlash(properties.get(CatalogProperties.WAREHOUSE_LOCATION));
+    }
+
+    this.fileIO =
+        CatalogUtil.loadFileIO(
+            properties.getOrDefault(
+                CatalogProperties.FILE_IO_IMPL, "org.apache.iceberg.io.ResolvingFileIO"),
+            properties,
+            conf);
+
+    this.listAllTables = Boolean.parseBoolean(properties.getOrDefault(LIST_ALL_TABLES, "true"));
+  }
+
+  @Override
+  protected TableOperations newTableOps(TableIdentifier identifier) {
+    return new BigQueryTableOperations(client, fileIO, toTableReference(identifier), conf);
+  }
+
+  @Override
+  protected String defaultWarehouseLocation(TableIdentifier identifier) {
+    String locationUri = null;
+    DatasetReference datasetReference = toDatasetReference(identifier.namespace());
+    Dataset dataset = client.load(datasetReference);
+    if (dataset != null && dataset.getExternalCatalogDatasetOptions() != null) {
+      locationUri = dataset.getExternalCatalogDatasetOptions().getDefaultStorageLocationUri();
+    }
+
+    return String.format(
+        "%s/%s",
+        Strings.isNullOrEmpty(locationUri)
+            ? createDefaultStorageLocationUri(datasetReference.getDatasetId())
+            : LocationUtil.stripTrailingSlash(locationUri),
+        identifier.name());
+  }
+
+  @Override
+  public List<TableIdentifier> listTables(Namespace namespace) {
+    validateNamespace(namespace);
+
+    return client.list(toDatasetReference(namespace), listAllTables).stream()
+        .map(
+            table -> TableIdentifier.of(namespace.level(0), table.getTableReference().getTableId()))
+        .collect(ImmutableList.toImmutableList());
+  }
+
+  @Override
+  public boolean dropTable(TableIdentifier identifier, boolean purge) {
+    try {
+      TableOperations ops = newTableOps(identifier);
+      TableMetadata lastMetadata = ops.current();
+
+      client.delete(toTableReference(identifier));
+
+      if (purge && lastMetadata != null) {
+        CatalogUtil.dropTableData(ops.io(), lastMetadata);
+      }
+    } catch (NoSuchTableException e) {
+      return false;
+    }
+
+    return true;
+  }
+
+  @Override
+  public void renameTable(TableIdentifier from, TableIdentifier to) {
+    // TODO: Enable once supported by BigQuery API.
+    throw new UnsupportedOperationException("Table rename operation is unsupported.");
+  }
+
+  @Override
+  public void createNamespace(Namespace namespace, Map<String, String> metadata) {
+    Dataset builder = new Dataset();
+    DatasetReference datasetReference = toDatasetReference(namespace);
+    builder.setLocation(this.projectLocation);
+    builder.setDatasetReference(datasetReference);
+    builder.setExternalCatalogDatasetOptions(
+        BigQueryMetastoreUtils.createExternalCatalogDatasetOptions(
+            createDefaultStorageLocationUri(datasetReference.getDatasetId()), metadata));
+
+    client.create(builder);
+  }
+
+  @Override
+  public List<Namespace> listNamespaces() {
+    try {
+      return listNamespaces(Namespace.empty());
+    } catch (NoSuchNamespaceException e) {
+      return ImmutableList.of();
+    }
+  }
+
+  /**
+   * Since this catalog only supports one-level namespaces, it always returns an empty list unless
+   * passed an empty namespace to list all namespaces within the catalog.
+   */
+  @Override
+  public List<Namespace> listNamespaces(Namespace namespace) {
+    List<Datasets> allDatasets = client.list(projectId);
+
+    ImmutableList<Namespace> namespaces =
+        allDatasets.stream().map(this::toNamespace).collect(ImmutableList.toImmutableList());
+    if (namespaces.isEmpty()) {
+      throw new NoSuchNamespaceException("Namespace does not exist: %s", namespace);
+    }
+
+    return namespaces;
+  }
+
+  @Override
+  public boolean dropNamespace(Namespace namespace) {
+    try {
+      client.delete(toDatasetReference(namespace));

Review Comment:
   It's not clear to me that we're throwing properly if the namespace is not empty (per the Catalog API)



-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org

For queries about this service, please contact Infrastructure at:
users@infra.apache.org


---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For additional commands, e-mail: issues-help@iceberg.apache.org


Re: [PR] Catalog: Add BigQuery Metastore Catalog Support [iceberg]

Posted by "danielcweeks (via GitHub)" <gi...@apache.org>.
danielcweeks commented on code in PR #12808:
URL: https://github.com/apache/iceberg/pull/12808#discussion_r2070905762


##########
bigquery/src/main/java/org/apache/iceberg/gcp/bigquery/BigQueryMetastoreCatalog.java:
##########
@@ -0,0 +1,370 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.iceberg.gcp.bigquery;
+
+import com.google.api.services.bigquery.model.Dataset;
+import com.google.api.services.bigquery.model.DatasetList.Datasets;
+import com.google.api.services.bigquery.model.DatasetReference;
+import com.google.api.services.bigquery.model.ExternalCatalogDatasetOptions;
+import com.google.api.services.bigquery.model.TableReference;
+import com.google.cloud.ServiceOptions;
+import com.google.cloud.bigquery.BigQueryOptions;
+import java.io.IOException;
+import java.security.GeneralSecurityException;
+import java.util.List;
+import java.util.Locale;
+import java.util.Map;
+import java.util.Objects;
+import java.util.Set;
+import org.apache.hadoop.conf.Configurable;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.iceberg.BaseMetastoreCatalog;
+import org.apache.iceberg.CatalogProperties;
+import org.apache.iceberg.CatalogUtil;
+import org.apache.iceberg.TableMetadata;
+import org.apache.iceberg.TableOperations;
+import org.apache.iceberg.catalog.Namespace;
+import org.apache.iceberg.catalog.SupportsNamespaces;
+import org.apache.iceberg.catalog.TableIdentifier;
+import org.apache.iceberg.exceptions.NoSuchNamespaceException;
+import org.apache.iceberg.exceptions.NoSuchTableException;
+import org.apache.iceberg.exceptions.ServiceFailureException;
+import org.apache.iceberg.exceptions.ValidationException;
+import org.apache.iceberg.io.FileIO;
+import org.apache.iceberg.relocated.com.google.common.annotations.VisibleForTesting;
+import org.apache.iceberg.relocated.com.google.common.base.Preconditions;
+import org.apache.iceberg.relocated.com.google.common.base.Strings;
+import org.apache.iceberg.relocated.com.google.common.collect.ImmutableList;
+import org.apache.iceberg.relocated.com.google.common.collect.ImmutableMap;
+import org.apache.iceberg.relocated.com.google.common.collect.Maps;
+import org.apache.iceberg.util.LocationUtil;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/** Iceberg Bigquery Metastore Catalog implementation. */
+public class BigQueryMetastoreCatalog extends BaseMetastoreCatalog
+    implements SupportsNamespaces, Configurable {
+
+  // User provided properties.
+  public static final String PROJECT_ID = "gcp.bigquery.project-id";
+  public static final String GCP_LOCATION = "gcp.bigquery.location";
+  public static final String LIST_ALL_TABLES = "gcp.bigquery.list-all-tables";
+
+  private static final Logger LOG = LoggerFactory.getLogger(BigQueryMetastoreCatalog.class);
+
+  private static final String DEFAULT_GCP_LOCATION = "us";
+
+  private String catalogName;
+  private Map<String, String> catalogProperties;
+  private FileIO fileIO;
+  private Configuration conf;
+  private String projectId;
+  private String projectLocation;
+  private BigQueryMetastoreClient client;
+  private boolean listAllTables;
+  private String warehouseLocation;
+
+  // Must have a no-arg constructor to be dynamically loaded
+  // initialize(String name, Map<String, String> properties) will be called to complete
+  // initialization

Review Comment:
   We should use standard javadoc block comments (not line comments) for public methods/constructors



-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org

For queries about this service, please contact Infrastructure at:
users@infra.apache.org


---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For additional commands, e-mail: issues-help@iceberg.apache.org


Re: [PR] Catalog: Add BigQuery Metastore Catalog Support [iceberg]

Posted by "talatuyarer (via GitHub)" <gi...@apache.org>.
talatuyarer commented on code in PR #12808:
URL: https://github.com/apache/iceberg/pull/12808#discussion_r2070953832


##########
bigquery/src/main/java/org/apache/iceberg/gcp/bigquery/BigQueryMetastoreCatalog.java:
##########
@@ -0,0 +1,370 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.iceberg.gcp.bigquery;
+
+import com.google.api.services.bigquery.model.Dataset;
+import com.google.api.services.bigquery.model.DatasetList.Datasets;
+import com.google.api.services.bigquery.model.DatasetReference;
+import com.google.api.services.bigquery.model.ExternalCatalogDatasetOptions;
+import com.google.api.services.bigquery.model.TableReference;
+import com.google.cloud.ServiceOptions;
+import com.google.cloud.bigquery.BigQueryOptions;
+import java.io.IOException;
+import java.security.GeneralSecurityException;
+import java.util.List;
+import java.util.Locale;
+import java.util.Map;
+import java.util.Objects;
+import java.util.Set;
+import org.apache.hadoop.conf.Configurable;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.iceberg.BaseMetastoreCatalog;
+import org.apache.iceberg.CatalogProperties;
+import org.apache.iceberg.CatalogUtil;
+import org.apache.iceberg.TableMetadata;
+import org.apache.iceberg.TableOperations;
+import org.apache.iceberg.catalog.Namespace;
+import org.apache.iceberg.catalog.SupportsNamespaces;
+import org.apache.iceberg.catalog.TableIdentifier;
+import org.apache.iceberg.exceptions.NoSuchNamespaceException;
+import org.apache.iceberg.exceptions.NoSuchTableException;
+import org.apache.iceberg.exceptions.ServiceFailureException;
+import org.apache.iceberg.exceptions.ValidationException;
+import org.apache.iceberg.io.FileIO;
+import org.apache.iceberg.relocated.com.google.common.annotations.VisibleForTesting;
+import org.apache.iceberg.relocated.com.google.common.base.Preconditions;
+import org.apache.iceberg.relocated.com.google.common.base.Strings;
+import org.apache.iceberg.relocated.com.google.common.collect.ImmutableList;
+import org.apache.iceberg.relocated.com.google.common.collect.ImmutableMap;
+import org.apache.iceberg.relocated.com.google.common.collect.Maps;
+import org.apache.iceberg.util.LocationUtil;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/** Iceberg Bigquery Metastore Catalog implementation. */
+public class BigQueryMetastoreCatalog extends BaseMetastoreCatalog
+    implements SupportsNamespaces, Configurable {
+
+  // User provided properties.
+  public static final String PROJECT_ID = "gcp.bigquery.project-id";
+  public static final String GCP_LOCATION = "gcp.bigquery.location";
+  public static final String LIST_ALL_TABLES = "gcp.bigquery.list-all-tables";
+
+  private static final Logger LOG = LoggerFactory.getLogger(BigQueryMetastoreCatalog.class);
+
+  private static final String DEFAULT_GCP_LOCATION = "us";
+
+  private String catalogName;
+  private Map<String, String> catalogProperties;
+  private FileIO fileIO;
+  private Configuration conf;
+  private String projectId;
+  private String projectLocation;
+  private BigQueryMetastoreClient client;
+  private boolean listAllTables;
+  private String warehouseLocation;
+
+  // Must have a no-arg constructor to be dynamically loaded
+  // initialize(String name, Map<String, String> properties) will be called to complete
+  // initialization
+  public BigQueryMetastoreCatalog() {}
+
+  @Override
+  public void initialize(String name, Map<String, String> properties) {
+    Preconditions.checkArgument(
+        properties.containsKey(PROJECT_ID),
+        "Invalid GCP project: %s must be specified",
+        PROJECT_ID);
+
+    this.projectId = properties.get(PROJECT_ID);
+    this.projectLocation = properties.getOrDefault(GCP_LOCATION, DEFAULT_GCP_LOCATION);
+
+    BigQueryOptions options =
+        BigQueryOptions.newBuilder()
+            .setProjectId(projectId)
+            .setLocation(projectLocation)
+            .setRetrySettings(ServiceOptions.getDefaultRetrySettings())
+            .build();
+
+    try {
+      client = new BigQueryMetastoreClientImpl(options);
+    } catch (IOException e) {
+      throw new ServiceFailureException(e, "Creating BigQuery client failed");

Review Comment:
   Done



-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org

For queries about this service, please contact Infrastructure at:
users@infra.apache.org


---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For additional commands, e-mail: issues-help@iceberg.apache.org


Re: [PR] Catalog: Add BigQuery Metastore Catalog Support [iceberg]

Posted by "talatuyarer (via GitHub)" <gi...@apache.org>.
talatuyarer commented on PR #12808:
URL: https://github.com/apache/iceberg/pull/12808#issuecomment-2808517721

   @nastra Would you like to review a brand new Catalog which uses CatalogTests 😃 


-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org

For queries about this service, please contact Infrastructure at:
users@infra.apache.org


---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For additional commands, e-mail: issues-help@iceberg.apache.org


Re: [PR] Catalog: Add BigQuery Metastore Catalog Support [iceberg]

Posted by "deepaknettem (via GitHub)" <gi...@apache.org>.
deepaknettem commented on code in PR #12808:
URL: https://github.com/apache/iceberg/pull/12808#discussion_r2047473029


##########
bigquery/src/main/java/org/apache/iceberg/gcp/bigquery/BigQueryMetaStoreClientImpl.java:
##########
@@ -0,0 +1,620 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.iceberg.gcp.bigquery;
+
+import com.google.api.client.googleapis.javanet.GoogleNetHttpTransport;
+import com.google.api.client.http.HttpHeaders;
+import com.google.api.client.http.HttpResponse;
+import com.google.api.client.http.HttpResponseException;
+import com.google.api.client.http.HttpStatusCodes;
+import com.google.api.client.json.gson.GsonFactory;
+import com.google.api.client.util.Data;
+import com.google.api.services.bigquery.Bigquery;
+import com.google.api.services.bigquery.BigqueryScopes;
+import com.google.api.services.bigquery.model.Dataset;
+import com.google.api.services.bigquery.model.DatasetList;
+import com.google.api.services.bigquery.model.DatasetList.Datasets;
+import com.google.api.services.bigquery.model.DatasetReference;
+import com.google.api.services.bigquery.model.ExternalCatalogDatasetOptions;
+import com.google.api.services.bigquery.model.ExternalCatalogTableOptions;
+import com.google.api.services.bigquery.model.Table;
+import com.google.api.services.bigquery.model.TableList;
+import com.google.api.services.bigquery.model.TableList.Tables;
+import com.google.api.services.bigquery.model.TableReference;
+import com.google.api.services.bigquery.model.TableSchema;
+import com.google.auth.http.HttpCredentialsAdapter;
+import com.google.auth.oauth2.GoogleCredentials;
+import com.google.cloud.BaseServiceException;
+import com.google.cloud.ExceptionHandler;
+import com.google.cloud.bigquery.BigQueryErrorMessages;
+import com.google.cloud.bigquery.BigQueryOptions;
+import com.google.cloud.bigquery.BigQueryRetryConfig;
+import com.google.cloud.bigquery.BigQueryRetryHelper;
+import java.io.IOException;
+import java.nio.charset.StandardCharsets;
+import java.security.GeneralSecurityException;
+import java.util.List;
+import java.util.Locale;
+import java.util.Map;
+import java.util.Set;
+import java.util.concurrent.Callable;
+import java.util.stream.Collectors;
+import java.util.stream.Stream;
+import org.apache.iceberg.BaseMetastoreTableOperations;
+import org.apache.iceberg.exceptions.AlreadyExistsException;
+import org.apache.iceberg.exceptions.BadRequestException;
+import org.apache.iceberg.exceptions.ForbiddenException;
+import org.apache.iceberg.exceptions.NoSuchIcebergTableException;
+import org.apache.iceberg.exceptions.NoSuchNamespaceException;
+import org.apache.iceberg.exceptions.NoSuchTableException;
+import org.apache.iceberg.exceptions.NotAuthorizedException;
+import org.apache.iceberg.exceptions.RuntimeIOException;
+import org.apache.iceberg.exceptions.ServiceFailureException;
+import org.apache.iceberg.exceptions.ServiceUnavailableException;
+import org.apache.iceberg.exceptions.ValidationException;
+import org.apache.iceberg.relocated.com.google.common.base.Preconditions;
+import org.apache.iceberg.relocated.com.google.common.collect.Lists;
+import org.apache.iceberg.relocated.com.google.common.collect.Maps;
+
+/** A client of Google Bigquery Metastore functions over the BigQuery service. */
+public final class BigQueryMetaStoreClientImpl implements BigQueryMetaStoreClient {
+
+  public static final ExceptionHandler.Interceptor EXCEPTION_HANDLER_INTERCEPTOR =
+      new ExceptionHandler.Interceptor() {
+
+        private static final long serialVersionUID = -8429573486870467828L;
+
+        @Override
+        public RetryResult afterEval(Exception exception, RetryResult retryResult) {
+          return ExceptionHandler.Interceptor.RetryResult.CONTINUE_EVALUATION;
+        }
+
+        @Override
+        public RetryResult beforeEval(Exception exception) {
+          if (exception instanceof BaseServiceException) {
+            boolean retriable = ((BaseServiceException) exception).isRetryable();
+            return retriable
+                ? ExceptionHandler.Interceptor.RetryResult.RETRY
+                : ExceptionHandler.Interceptor.RetryResult.CONTINUE_EVALUATION;
+          }
+          return ExceptionHandler.Interceptor.RetryResult.CONTINUE_EVALUATION;
+        }
+      };
+
+  private final Bigquery client;
+  private final BigQueryOptions bigqueryOptions;
+
+  // Retry config with error messages and regex for rate limit exceeded errors.
+  private static final BigQueryRetryConfig DEFAULT_RETRY_CONFIG =
+      BigQueryRetryConfig.newBuilder()
+          .retryOnMessage(BigQueryErrorMessages.RATE_LIMIT_EXCEEDED_MSG)
+          .retryOnMessage(BigQueryErrorMessages.JOB_RATE_LIMIT_EXCEEDED_MSG)
+          .retryOnRegEx(BigQueryErrorMessages.RetryRegExPatterns.RATE_LIMIT_EXCEEDED_REGEX)
+          .build();
+
+  public static final ExceptionHandler BIGQUERY_EXCEPTION_HANDLER =
+      ExceptionHandler.newBuilder()
+          .abortOn(RuntimeException.class)
+          .retryOn(java.net.ConnectException.class) // retry on Connection Exception
+          .retryOn(java.net.UnknownHostException.class) // retry on UnknownHostException
+          .addInterceptors(EXCEPTION_HANDLER_INTERCEPTOR)
+          .build();
+
+  /** Constructs a client of the Google BigQuery service. */
+  public BigQueryMetaStoreClientImpl(BigQueryOptions options)
+      throws IOException, GeneralSecurityException {
+    // Initialize client that will be used to send requests. This client only needs to be created
+    // once, and can be reused for multiple requests
+    HttpCredentialsAdapter httpCredentialsAdapter =
+        new HttpCredentialsAdapter(
+            GoogleCredentials.getApplicationDefault().createScoped(BigqueryScopes.all()));
+    this.client =
+        new Bigquery.Builder(
+                GoogleNetHttpTransport.newTrustedTransport(),
+                GsonFactory.getDefaultInstance(),
+                httpRequest -> {
+                  httpCredentialsAdapter.initialize(httpRequest);
+                  // Instead of throwing exceptions, analyze the HttpResponse object and inspect its
+                  // status code. This will allow BigQuery API errors to be converted into Iceberg
+                  // exceptions.
+                  httpRequest.setThrowExceptionOnExecuteError(false);
+                })
+            .setApplicationName("BigQuery Iceberg Catalog Plugin")

Review Comment:
   nit: s/BigQuery/BigQuery metastore



-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org

For queries about this service, please contact Infrastructure at:
users@infra.apache.org


---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For additional commands, e-mail: issues-help@iceberg.apache.org


Re: [PR] Catalog: Add BigQuery Metastore Catalog Support [iceberg]

Posted by "nastra (via GitHub)" <gi...@apache.org>.
nastra commented on code in PR #12808:
URL: https://github.com/apache/iceberg/pull/12808#discussion_r2055586233


##########
bigquery/src/main/java/org/apache/iceberg/gcp/bigquery/BigQueryMetastoreCatalog.java:
##########
@@ -0,0 +1,400 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.iceberg.gcp.bigquery;
+
+import com.google.api.services.bigquery.model.Dataset;
+import com.google.api.services.bigquery.model.DatasetList.Datasets;
+import com.google.api.services.bigquery.model.DatasetReference;
+import com.google.api.services.bigquery.model.ExternalCatalogDatasetOptions;
+import com.google.api.services.bigquery.model.TableReference;
+import com.google.cloud.ServiceOptions;
+import com.google.cloud.bigquery.BigQueryOptions;
+import java.io.IOException;
+import java.security.GeneralSecurityException;
+import java.util.List;
+import java.util.Locale;
+import java.util.Map;
+import java.util.Objects;
+import java.util.Set;
+import org.apache.hadoop.conf.Configurable;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.iceberg.BaseMetastoreCatalog;
+import org.apache.iceberg.CatalogProperties;
+import org.apache.iceberg.CatalogUtil;
+import org.apache.iceberg.TableMetadata;
+import org.apache.iceberg.TableOperations;
+import org.apache.iceberg.catalog.Namespace;
+import org.apache.iceberg.catalog.SupportsNamespaces;
+import org.apache.iceberg.catalog.TableIdentifier;
+import org.apache.iceberg.exceptions.AlreadyExistsException;
+import org.apache.iceberg.exceptions.NoSuchNamespaceException;
+import org.apache.iceberg.exceptions.NoSuchTableException;
+import org.apache.iceberg.exceptions.ServiceFailureException;
+import org.apache.iceberg.exceptions.ValidationException;
+import org.apache.iceberg.io.FileIO;
+import org.apache.iceberg.relocated.com.google.common.annotations.VisibleForTesting;
+import org.apache.iceberg.relocated.com.google.common.base.Preconditions;
+import org.apache.iceberg.relocated.com.google.common.base.Strings;
+import org.apache.iceberg.relocated.com.google.common.collect.ImmutableList;
+import org.apache.iceberg.relocated.com.google.common.collect.ImmutableMap;
+import org.apache.iceberg.relocated.com.google.common.collect.Maps;
+import org.apache.iceberg.util.LocationUtil;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/** Iceberg Bigquery Metastore Catalog implementation. */
+public final class BigQueryMetastoreCatalog extends BaseMetastoreCatalog
+    implements SupportsNamespaces, Configurable {
+
+  // User provided properties.
+  public static final String PROJECT_ID = "gcp-project";
+  public static final String GCP_LOCATION = "gcp-location";
+  public static final String FILTER_UNSUPPORTED_TABLES = "filter-unsupported-tables";
+  public static final String TESTING_ENABLED = "testing-enabled";
+
+  public static final String HIVE_METASTORE_WAREHOUSE_DIR = "hive.metastore.warehouse.dir";
+
+  private static final Logger LOG = LoggerFactory.getLogger(BigQueryMetastoreCatalog.class);
+
+  private static final String DEFAULT_GCP_LOCATION = "us";
+
+  private String catalogName;
+  private Map<String, String> catalogProperties;
+  private FileIO fileIO;
+  private Configuration conf;
+  private String projectId;
+  private String location;
+  private BigQueryMetastoreClient client;
+  private boolean filterUnsupportedTables;
+
+  // Must have a no-arg constructor to be dynamically loaded
+  // initialize(String name, Map<String, String> properties) will be called to complete
+  // initialization
+  public BigQueryMetastoreCatalog() {}
+
+  @Override
+  public void initialize(String inputName, Map<String, String> properties) {
+    Preconditions.checkArgument(
+        properties.containsKey(PROJECT_ID), "GCP project must be specified");
+
+    projectId = properties.get(PROJECT_ID);
+    location = properties.getOrDefault(GCP_LOCATION, DEFAULT_GCP_LOCATION);
+    boolean testingEnabled =
+        Boolean.parseBoolean(properties.getOrDefault(TESTING_ENABLED, "false"));
+
+    BigQueryOptions options =
+        BigQueryOptions.newBuilder()
+            .setProjectId(projectId)
+            .setLocation(location)
+            .setRetrySettings(ServiceOptions.getDefaultRetrySettings())
+            .build();
+
+    try {
+      if (testingEnabled) {
+        client = new FakeBigQueryMetastoreClient(options);
+
+      } else {
+        client = new BigQueryMetastoreClientImpl(options);
+      }
+    } catch (IOException e) {
+      throw new ServiceFailureException(e, "Creating BigQuery client failed");
+    } catch (GeneralSecurityException e) {
+      throw new ValidationException(e, "Creating BigQuery client failed due to a security issue");
+    }
+    initialize(inputName, properties, projectId, location, client);
+  }
+
+  @VisibleForTesting
+  void initialize(
+      String inputName,
+      Map<String, String> properties,
+      String initialProjectId,
+      String initialLocation,
+      BigQueryMetastoreClient bigQueryMetaStoreClient) {
+    Preconditions.checkArgument(bigQueryMetaStoreClient != null, "BigQuery client can not be null");
+    this.catalogName = inputName;
+    this.catalogProperties = ImmutableMap.copyOf(properties);
+    this.projectId = initialProjectId;
+    this.location = initialLocation;
+    this.client = bigQueryMetaStoreClient;
+
+    if (this.conf == null) {
+      LOG.warn("No configuration was set, using the default environment Configuration");
+      this.conf = new Configuration();
+    }
+
+    LOG.info("Using BigQuery Metastore Iceberg Catalog: {}", inputName);
+
+    if (properties.containsKey(CatalogProperties.WAREHOUSE_LOCATION)) {
+      // Iceberg always removes trailing slash to avoid paths like "<folder>//data" in file systems
+      // like s3.
+      this.conf.set(
+          HIVE_METASTORE_WAREHOUSE_DIR,
+          LocationUtil.stripTrailingSlash(properties.get(CatalogProperties.WAREHOUSE_LOCATION)));
+    }
+
+    String fileIoImpl =
+        properties.getOrDefault(
+            CatalogProperties.FILE_IO_IMPL, "org.apache.iceberg.hadoop.HadoopFileIO");
+    this.fileIO = CatalogUtil.loadFileIO(fileIoImpl, properties, conf);
+
+    this.filterUnsupportedTables =
+        Boolean.parseBoolean(properties.getOrDefault(FILTER_UNSUPPORTED_TABLES, "false"));
+  }
+
+  @Override
+  protected TableOperations newTableOps(TableIdentifier identifier) {
+    return new BigQueryTableOperations(
+        client,
+        fileIO,
+        projectId,
+        // Sometimes extensions have the namespace contain the table name too, so we are forced to

Review Comment:
   I don't think I understand this comment. Isn't it really because BigQuery only support single-level namespaces? At least that's what is being indicated in `validateNamespace`. So maybe just remove this comment to avoid confusion for readers?



-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org

For queries about this service, please contact Infrastructure at:
users@infra.apache.org


---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For additional commands, e-mail: issues-help@iceberg.apache.org


Re: [PR] Catalog: Add BigQuery Metastore Catalog Support [iceberg]

Posted by "nastra (via GitHub)" <gi...@apache.org>.
nastra commented on code in PR #12808:
URL: https://github.com/apache/iceberg/pull/12808#discussion_r2055621275


##########
bigquery/src/main/java/org/apache/iceberg/gcp/bigquery/BigQueryMetastoreCatalog.java:
##########
@@ -0,0 +1,400 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.iceberg.gcp.bigquery;
+
+import com.google.api.services.bigquery.model.Dataset;
+import com.google.api.services.bigquery.model.DatasetList.Datasets;
+import com.google.api.services.bigquery.model.DatasetReference;
+import com.google.api.services.bigquery.model.ExternalCatalogDatasetOptions;
+import com.google.api.services.bigquery.model.TableReference;
+import com.google.cloud.ServiceOptions;
+import com.google.cloud.bigquery.BigQueryOptions;
+import java.io.IOException;
+import java.security.GeneralSecurityException;
+import java.util.List;
+import java.util.Locale;
+import java.util.Map;
+import java.util.Objects;
+import java.util.Set;
+import org.apache.hadoop.conf.Configurable;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.iceberg.BaseMetastoreCatalog;
+import org.apache.iceberg.CatalogProperties;
+import org.apache.iceberg.CatalogUtil;
+import org.apache.iceberg.TableMetadata;
+import org.apache.iceberg.TableOperations;
+import org.apache.iceberg.catalog.Namespace;
+import org.apache.iceberg.catalog.SupportsNamespaces;
+import org.apache.iceberg.catalog.TableIdentifier;
+import org.apache.iceberg.exceptions.AlreadyExistsException;
+import org.apache.iceberg.exceptions.NoSuchNamespaceException;
+import org.apache.iceberg.exceptions.NoSuchTableException;
+import org.apache.iceberg.exceptions.ServiceFailureException;
+import org.apache.iceberg.exceptions.ValidationException;
+import org.apache.iceberg.io.FileIO;
+import org.apache.iceberg.relocated.com.google.common.annotations.VisibleForTesting;
+import org.apache.iceberg.relocated.com.google.common.base.Preconditions;
+import org.apache.iceberg.relocated.com.google.common.base.Strings;
+import org.apache.iceberg.relocated.com.google.common.collect.ImmutableList;
+import org.apache.iceberg.relocated.com.google.common.collect.ImmutableMap;
+import org.apache.iceberg.relocated.com.google.common.collect.Maps;
+import org.apache.iceberg.util.LocationUtil;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/** Iceberg Bigquery Metastore Catalog implementation. */
+public final class BigQueryMetastoreCatalog extends BaseMetastoreCatalog
+    implements SupportsNamespaces, Configurable {
+
+  // User provided properties.
+  public static final String PROJECT_ID = "gcp-project";
+  public static final String GCP_LOCATION = "gcp-location";
+  public static final String FILTER_UNSUPPORTED_TABLES = "filter-unsupported-tables";
+  public static final String TESTING_ENABLED = "testing-enabled";
+
+  public static final String HIVE_METASTORE_WAREHOUSE_DIR = "hive.metastore.warehouse.dir";
+
+  private static final Logger LOG = LoggerFactory.getLogger(BigQueryMetastoreCatalog.class);
+
+  private static final String DEFAULT_GCP_LOCATION = "us";
+
+  private String catalogName;
+  private Map<String, String> catalogProperties;
+  private FileIO fileIO;
+  private Configuration conf;
+  private String projectId;
+  private String location;
+  private BigQueryMetastoreClient client;
+  private boolean filterUnsupportedTables;
+
+  // Must have a no-arg constructor to be dynamically loaded
+  // initialize(String name, Map<String, String> properties) will be called to complete
+  // initialization
+  public BigQueryMetastoreCatalog() {}
+
+  @Override
+  public void initialize(String inputName, Map<String, String> properties) {
+    Preconditions.checkArgument(
+        properties.containsKey(PROJECT_ID), "GCP project must be specified");
+
+    projectId = properties.get(PROJECT_ID);
+    location = properties.getOrDefault(GCP_LOCATION, DEFAULT_GCP_LOCATION);
+    boolean testingEnabled =
+        Boolean.parseBoolean(properties.getOrDefault(TESTING_ENABLED, "false"));
+
+    BigQueryOptions options =
+        BigQueryOptions.newBuilder()
+            .setProjectId(projectId)
+            .setLocation(location)
+            .setRetrySettings(ServiceOptions.getDefaultRetrySettings())
+            .build();
+
+    try {
+      if (testingEnabled) {
+        client = new FakeBigQueryMetastoreClient(options);
+
+      } else {
+        client = new BigQueryMetastoreClientImpl(options);
+      }
+    } catch (IOException e) {
+      throw new ServiceFailureException(e, "Creating BigQuery client failed");
+    } catch (GeneralSecurityException e) {
+      throw new ValidationException(e, "Creating BigQuery client failed due to a security issue");
+    }
+    initialize(inputName, properties, projectId, location, client);
+  }
+
+  @VisibleForTesting
+  void initialize(
+      String inputName,
+      Map<String, String> properties,
+      String initialProjectId,
+      String initialLocation,
+      BigQueryMetastoreClient bigQueryMetaStoreClient) {
+    Preconditions.checkArgument(bigQueryMetaStoreClient != null, "BigQuery client can not be null");
+    this.catalogName = inputName;
+    this.catalogProperties = ImmutableMap.copyOf(properties);
+    this.projectId = initialProjectId;
+    this.location = initialLocation;
+    this.client = bigQueryMetaStoreClient;
+
+    if (this.conf == null) {
+      LOG.warn("No configuration was set, using the default environment Configuration");
+      this.conf = new Configuration();
+    }
+
+    LOG.info("Using BigQuery Metastore Iceberg Catalog: {}", inputName);
+
+    if (properties.containsKey(CatalogProperties.WAREHOUSE_LOCATION)) {
+      // Iceberg always removes trailing slash to avoid paths like "<folder>//data" in file systems
+      // like s3.
+      this.conf.set(
+          HIVE_METASTORE_WAREHOUSE_DIR,
+          LocationUtil.stripTrailingSlash(properties.get(CatalogProperties.WAREHOUSE_LOCATION)));
+    }
+
+    String fileIoImpl =
+        properties.getOrDefault(
+            CatalogProperties.FILE_IO_IMPL, "org.apache.iceberg.hadoop.HadoopFileIO");
+    this.fileIO = CatalogUtil.loadFileIO(fileIoImpl, properties, conf);
+
+    this.filterUnsupportedTables =
+        Boolean.parseBoolean(properties.getOrDefault(FILTER_UNSUPPORTED_TABLES, "false"));
+  }
+
+  @Override
+  protected TableOperations newTableOps(TableIdentifier identifier) {
+    return new BigQueryTableOperations(
+        client,
+        fileIO,
+        projectId,
+        // Sometimes extensions have the namespace contain the table name too, so we are forced to
+        // allow invalid namespace and just take the first part here like other catalog
+        // implementations do.
+        identifier.namespace().level(0),
+        identifier.name(),
+        conf);
+  }
+
+  @Override
+  protected String defaultWarehouseLocation(TableIdentifier identifier) {
+    String locationUri = null;
+    DatasetReference datasetReference = toDatasetReference(identifier.namespace());
+    Dataset dataset = client.getDataset(datasetReference);
+    if (dataset != null && dataset.getExternalCatalogDatasetOptions() != null) {
+      locationUri = dataset.getExternalCatalogDatasetOptions().getDefaultStorageLocationUri();
+    }
+    return String.format(
+        "%s/%s",
+        Strings.isNullOrEmpty(locationUri)
+            ? datasetReference.getDatasetId()
+            : LocationUtil.stripTrailingSlash(locationUri),
+        identifier.name());
+  }
+
+  @Override
+  public List<TableIdentifier> listTables(Namespace namespace) {
+    validateNamespace(namespace);
+
+    return client.listTables(toDatasetReference(namespace), filterUnsupportedTables).stream()
+        .map(
+            table -> TableIdentifier.of(namespace.level(0), table.getTableReference().getTableId()))
+        .collect(ImmutableList.toImmutableList());
+  }
+
+  @Override
+  public boolean dropTable(TableIdentifier identifier, boolean purge) {
+    try {
+      TableOperations ops = newTableOps(identifier);
+      TableMetadata lastMetadata = ops.current();
+
+      client.deleteTable(toTableReference(identifier));
+
+      if (purge && lastMetadata != null) {
+        CatalogUtil.dropTableData(ops.io(), lastMetadata);
+      }
+    } catch (NoSuchTableException e) { // Not catching a NoSuchIcebergTableException on purpose
+      return false; // The documentation says just return false in this case
+    }
+    return true;
+  }
+
+  @Override
+  public void renameTable(TableIdentifier from, TableIdentifier to) {
+    validateNamespace(from.namespace());
+    validateNamespace(to.namespace());
+
+    try {
+      client.getDataset(toDatasetReference(to.namespace()));
+      client.getDataset(toDatasetReference(from.namespace()));
+    } catch (NoSuchNamespaceException e) {
+      throw e;
+    }
+
+    TableReference fromTableRef = toTableReference(from);
+    TableReference toTableRef = toTableReference(to);
+
+    // Check if the source table exists
+    try {
+      client.getTable(fromTableRef);
+    } catch (NoSuchTableException e) {
+      throw new NoSuchTableException("Table does not exist: %s", from.name());
+    }
+
+    // Check if the destination table already exists
+    try {
+      client.getTable(toTableRef);
+      // If getTable succeeds, the destination table exists
+      throw new AlreadyExistsException("Table already exists: %s", to.name());
+    } catch (NoSuchTableException e) {
+      // Destination table does not exist, proceed with throwing exception
+    }
+
+    if (!from.namespace().equals(to.namespace())) {
+      throw new ValidationException("New table name must be in the same namespace");
+    }
+
+    // TODO(b/354981675): Enable once supported by the API.
+    throw new UnsupportedOperationException("Table rename operation is unsupported.");
+  }
+
+  @Override
+  public void createNamespace(Namespace namespace, Map<String, String> metadata) {
+    Dataset builder = new Dataset();
+    DatasetReference datasetReference = toDatasetReference(namespace);
+    builder.setLocation(this.location);
+    builder.setDatasetReference(datasetReference);
+    builder.setExternalCatalogDatasetOptions(
+        BigQueryMetastoreUtils.createExternalCatalogDatasetOptions(
+            datasetReference.getDatasetId(), metadata));
+
+    client.createDataset(builder);
+  }
+
+  /**
+   * Since this catalog only supports one-level namespaces, it always returns an empty list unless
+   * passed an empty namespace to list all namespaces within the catalog.
+   */
+  @Override
+  public List<Namespace> listNamespaces(Namespace namespace) {

Review Comment:
   this needs to check whether the namespace exists or not and throw a `NoSuchNamespaceException` if the namespace doesn't exist. I also realized we don't have a test for this in `CatalogTests`, so I opened https://github.com/apache/iceberg/pull/12873



-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org

For queries about this service, please contact Infrastructure at:
users@infra.apache.org


---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For additional commands, e-mail: issues-help@iceberg.apache.org


Re: [PR] Catalog: Add BigQuery Metastore Catalog Support [iceberg]

Posted by "talatuyarer (via GitHub)" <gi...@apache.org>.
talatuyarer commented on code in PR #12808:
URL: https://github.com/apache/iceberg/pull/12808#discussion_r2059719214


##########
bigquery/src/main/java/org/apache/iceberg/gcp/bigquery/BigQueryMetastoreCatalog.java:
##########
@@ -0,0 +1,400 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.iceberg.gcp.bigquery;
+
+import com.google.api.services.bigquery.model.Dataset;
+import com.google.api.services.bigquery.model.DatasetList.Datasets;
+import com.google.api.services.bigquery.model.DatasetReference;
+import com.google.api.services.bigquery.model.ExternalCatalogDatasetOptions;
+import com.google.api.services.bigquery.model.TableReference;
+import com.google.cloud.ServiceOptions;
+import com.google.cloud.bigquery.BigQueryOptions;
+import java.io.IOException;
+import java.security.GeneralSecurityException;
+import java.util.List;
+import java.util.Locale;
+import java.util.Map;
+import java.util.Objects;
+import java.util.Set;
+import org.apache.hadoop.conf.Configurable;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.iceberg.BaseMetastoreCatalog;
+import org.apache.iceberg.CatalogProperties;
+import org.apache.iceberg.CatalogUtil;
+import org.apache.iceberg.TableMetadata;
+import org.apache.iceberg.TableOperations;
+import org.apache.iceberg.catalog.Namespace;
+import org.apache.iceberg.catalog.SupportsNamespaces;
+import org.apache.iceberg.catalog.TableIdentifier;
+import org.apache.iceberg.exceptions.AlreadyExistsException;
+import org.apache.iceberg.exceptions.NoSuchNamespaceException;
+import org.apache.iceberg.exceptions.NoSuchTableException;
+import org.apache.iceberg.exceptions.ServiceFailureException;
+import org.apache.iceberg.exceptions.ValidationException;
+import org.apache.iceberg.io.FileIO;
+import org.apache.iceberg.relocated.com.google.common.annotations.VisibleForTesting;
+import org.apache.iceberg.relocated.com.google.common.base.Preconditions;
+import org.apache.iceberg.relocated.com.google.common.base.Strings;
+import org.apache.iceberg.relocated.com.google.common.collect.ImmutableList;
+import org.apache.iceberg.relocated.com.google.common.collect.ImmutableMap;
+import org.apache.iceberg.relocated.com.google.common.collect.Maps;
+import org.apache.iceberg.util.LocationUtil;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/** Iceberg Bigquery Metastore Catalog implementation. */
+public final class BigQueryMetastoreCatalog extends BaseMetastoreCatalog
+    implements SupportsNamespaces, Configurable {
+
+  // User provided properties.
+  public static final String PROJECT_ID = "gcp-project";
+  public static final String GCP_LOCATION = "gcp-location";
+  public static final String FILTER_UNSUPPORTED_TABLES = "filter-unsupported-tables";
+  public static final String TESTING_ENABLED = "testing-enabled";
+
+  public static final String HIVE_METASTORE_WAREHOUSE_DIR = "hive.metastore.warehouse.dir";
+
+  private static final Logger LOG = LoggerFactory.getLogger(BigQueryMetastoreCatalog.class);
+
+  private static final String DEFAULT_GCP_LOCATION = "us";
+
+  private String catalogName;
+  private Map<String, String> catalogProperties;
+  private FileIO fileIO;
+  private Configuration conf;
+  private String projectId;
+  private String location;
+  private BigQueryMetastoreClient client;
+  private boolean filterUnsupportedTables;
+
+  // Must have a no-arg constructor to be dynamically loaded
+  // initialize(String name, Map<String, String> properties) will be called to complete
+  // initialization
+  public BigQueryMetastoreCatalog() {}
+
+  @Override
+  public void initialize(String inputName, Map<String, String> properties) {
+    Preconditions.checkArgument(
+        properties.containsKey(PROJECT_ID), "GCP project must be specified");
+
+    projectId = properties.get(PROJECT_ID);
+    location = properties.getOrDefault(GCP_LOCATION, DEFAULT_GCP_LOCATION);
+    boolean testingEnabled =
+        Boolean.parseBoolean(properties.getOrDefault(TESTING_ENABLED, "false"));
+
+    BigQueryOptions options =
+        BigQueryOptions.newBuilder()
+            .setProjectId(projectId)
+            .setLocation(location)
+            .setRetrySettings(ServiceOptions.getDefaultRetrySettings())
+            .build();
+
+    try {
+      if (testingEnabled) {
+        client = new FakeBigQueryMetastoreClient(options);
+
+      } else {
+        client = new BigQueryMetastoreClientImpl(options);
+      }
+    } catch (IOException e) {
+      throw new ServiceFailureException(e, "Creating BigQuery client failed");
+    } catch (GeneralSecurityException e) {
+      throw new ValidationException(e, "Creating BigQuery client failed due to a security issue");
+    }
+    initialize(inputName, properties, projectId, location, client);
+  }
+
+  @VisibleForTesting
+  void initialize(
+      String inputName,
+      Map<String, String> properties,
+      String initialProjectId,
+      String initialLocation,
+      BigQueryMetastoreClient bigQueryMetaStoreClient) {
+    Preconditions.checkArgument(bigQueryMetaStoreClient != null, "BigQuery client can not be null");
+    this.catalogName = inputName;
+    this.catalogProperties = ImmutableMap.copyOf(properties);
+    this.projectId = initialProjectId;
+    this.location = initialLocation;
+    this.client = bigQueryMetaStoreClient;
+
+    if (this.conf == null) {
+      LOG.warn("No configuration was set, using the default environment Configuration");
+      this.conf = new Configuration();
+    }
+
+    LOG.info("Using BigQuery Metastore Iceberg Catalog: {}", inputName);
+
+    if (properties.containsKey(CatalogProperties.WAREHOUSE_LOCATION)) {
+      // Iceberg always removes trailing slash to avoid paths like "<folder>//data" in file systems
+      // like s3.
+      this.conf.set(
+          HIVE_METASTORE_WAREHOUSE_DIR,
+          LocationUtil.stripTrailingSlash(properties.get(CatalogProperties.WAREHOUSE_LOCATION)));
+    }
+
+    String fileIoImpl =
+        properties.getOrDefault(
+            CatalogProperties.FILE_IO_IMPL, "org.apache.iceberg.hadoop.HadoopFileIO");
+    this.fileIO = CatalogUtil.loadFileIO(fileIoImpl, properties, conf);
+
+    this.filterUnsupportedTables =
+        Boolean.parseBoolean(properties.getOrDefault(FILTER_UNSUPPORTED_TABLES, "false"));
+  }
+
+  @Override
+  protected TableOperations newTableOps(TableIdentifier identifier) {
+    return new BigQueryTableOperations(
+        client,
+        fileIO,
+        projectId,
+        // Sometimes extensions have the namespace contain the table name too, so we are forced to
+        // allow invalid namespace and just take the first part here like other catalog
+        // implementations do.
+        identifier.namespace().level(0),
+        identifier.name(),
+        conf);
+  }
+
+  @Override
+  protected String defaultWarehouseLocation(TableIdentifier identifier) {
+    String locationUri = null;
+    DatasetReference datasetReference = toDatasetReference(identifier.namespace());
+    Dataset dataset = client.getDataset(datasetReference);
+    if (dataset != null && dataset.getExternalCatalogDatasetOptions() != null) {
+      locationUri = dataset.getExternalCatalogDatasetOptions().getDefaultStorageLocationUri();
+    }
+    return String.format(
+        "%s/%s",
+        Strings.isNullOrEmpty(locationUri)
+            ? datasetReference.getDatasetId()
+            : LocationUtil.stripTrailingSlash(locationUri),
+        identifier.name());
+  }
+
+  @Override
+  public List<TableIdentifier> listTables(Namespace namespace) {
+    validateNamespace(namespace);
+
+    return client.listTables(toDatasetReference(namespace), filterUnsupportedTables).stream()
+        .map(
+            table -> TableIdentifier.of(namespace.level(0), table.getTableReference().getTableId()))
+        .collect(ImmutableList.toImmutableList());
+  }
+
+  @Override
+  public boolean dropTable(TableIdentifier identifier, boolean purge) {
+    try {
+      TableOperations ops = newTableOps(identifier);
+      TableMetadata lastMetadata = ops.current();
+
+      client.deleteTable(toTableReference(identifier));
+
+      if (purge && lastMetadata != null) {
+        CatalogUtil.dropTableData(ops.io(), lastMetadata);
+      }
+    } catch (NoSuchTableException e) { // Not catching a NoSuchIcebergTableException on purpose
+      return false; // The documentation says just return false in this case
+    }
+    return true;
+  }
+
+  @Override
+  public void renameTable(TableIdentifier from, TableIdentifier to) {
+    validateNamespace(from.namespace());
+    validateNamespace(to.namespace());
+
+    try {
+      client.getDataset(toDatasetReference(to.namespace()));
+      client.getDataset(toDatasetReference(from.namespace()));
+    } catch (NoSuchNamespaceException e) {
+      throw e;
+    }
+
+    TableReference fromTableRef = toTableReference(from);
+    TableReference toTableRef = toTableReference(to);
+
+    // Check if the source table exists
+    try {
+      client.getTable(fromTableRef);
+    } catch (NoSuchTableException e) {
+      throw new NoSuchTableException("Table does not exist: %s", from.name());
+    }
+
+    // Check if the destination table already exists
+    try {
+      client.getTable(toTableRef);
+      // If getTable succeeds, the destination table exists
+      throw new AlreadyExistsException("Table already exists: %s", to.name());
+    } catch (NoSuchTableException e) {
+      // Destination table does not exist, proceed with throwing exception
+    }
+
+    if (!from.namespace().equals(to.namespace())) {
+      throw new ValidationException("New table name must be in the same namespace");
+    }
+
+    // TODO(b/354981675): Enable once supported by the API.

Review Comment:
   Removed Sorry I missed this



-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org

For queries about this service, please contact Infrastructure at:
users@infra.apache.org


---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For additional commands, e-mail: issues-help@iceberg.apache.org


Re: [PR] Catalog: Add BigQuery Metastore Catalog Support [iceberg]

Posted by "talatuyarer (via GitHub)" <gi...@apache.org>.
talatuyarer commented on code in PR #12808:
URL: https://github.com/apache/iceberg/pull/12808#discussion_r2059724419


##########
bigquery/src/main/java/org/apache/iceberg/gcp/bigquery/BigQueryMetastoreCatalog.java:
##########
@@ -0,0 +1,402 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.iceberg.gcp.bigquery;
+
+import com.google.api.services.bigquery.model.Dataset;
+import com.google.api.services.bigquery.model.DatasetList.Datasets;
+import com.google.api.services.bigquery.model.DatasetReference;
+import com.google.api.services.bigquery.model.ExternalCatalogDatasetOptions;
+import com.google.api.services.bigquery.model.TableReference;
+import com.google.cloud.ServiceOptions;
+import com.google.cloud.bigquery.BigQueryOptions;
+import java.io.IOException;
+import java.security.GeneralSecurityException;
+import java.util.List;
+import java.util.Locale;
+import java.util.Map;
+import java.util.Objects;
+import java.util.Set;
+import org.apache.hadoop.conf.Configurable;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.iceberg.BaseMetastoreCatalog;
+import org.apache.iceberg.CatalogProperties;
+import org.apache.iceberg.CatalogUtil;
+import org.apache.iceberg.TableMetadata;
+import org.apache.iceberg.TableOperations;
+import org.apache.iceberg.catalog.Namespace;
+import org.apache.iceberg.catalog.SupportsNamespaces;
+import org.apache.iceberg.catalog.TableIdentifier;
+import org.apache.iceberg.exceptions.AlreadyExistsException;
+import org.apache.iceberg.exceptions.NoSuchNamespaceException;
+import org.apache.iceberg.exceptions.NoSuchTableException;
+import org.apache.iceberg.exceptions.ServiceFailureException;
+import org.apache.iceberg.exceptions.ValidationException;
+import org.apache.iceberg.io.FileIO;
+import org.apache.iceberg.relocated.com.google.common.annotations.VisibleForTesting;
+import org.apache.iceberg.relocated.com.google.common.base.Preconditions;
+import org.apache.iceberg.relocated.com.google.common.base.Strings;
+import org.apache.iceberg.relocated.com.google.common.collect.ImmutableList;
+import org.apache.iceberg.relocated.com.google.common.collect.ImmutableMap;
+import org.apache.iceberg.relocated.com.google.common.collect.Maps;
+import org.apache.iceberg.util.LocationUtil;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/** Iceberg Bigquery Metastore Catalog implementation. */
+public final class BigQueryMetastoreCatalog extends BaseMetastoreCatalog
+    implements SupportsNamespaces, Configurable {
+
+  // User provided properties.
+  public static final String PROPERTIES_KEY_GCP_PROJECT = "gcp-project";
+  public static final String PROPERTIES_KEY_GCP_LOCATION = "gcp-location";
+  public static final String PROPERTIES_KEY_FILTER_UNSUPPORTED_TABLES = "filter-unsupported-tables";
+  public static final String PROPERTIES_KEY_TESTING_ENABLED = "testing-enabled";
+
+  public static final String HIVE_METASTORE_WAREHOUSE_DIR = "hive.metastore.warehouse.dir";
+
+  private static final Logger LOG = LoggerFactory.getLogger(BigQueryMetastoreCatalog.class);
+
+  private static final String DEFAULT_GCP_LOCATION = "us";
+
+  private String catalogPluginName;
+  private Map<String, String> catalogProperties;
+  private FileIO fileIO;
+  private Configuration conf;
+  private String projectId;
+  private String location;
+  private BigQueryMetastoreClient client;
+  private boolean filterUnsupportedTables;
+
+  // Must have a no-arg constructor to be dynamically loaded
+  // initialize(String name, Map<String, String> properties) will be called to complete
+  // initialization
+  public BigQueryMetastoreCatalog() {}
+
+  @Override
+  public void initialize(String inputName, Map<String, String> properties) {
+    if (!properties.containsKey(PROPERTIES_KEY_GCP_PROJECT)) {
+      throw new ValidationException("GCP project must be specified");
+    }
+    projectId = properties.get(PROPERTIES_KEY_GCP_PROJECT);
+    location = properties.getOrDefault(PROPERTIES_KEY_GCP_LOCATION, DEFAULT_GCP_LOCATION);
+    boolean testingEnabled =
+        Boolean.parseBoolean(properties.getOrDefault(PROPERTIES_KEY_TESTING_ENABLED, "false"));
+
+    BigQueryOptions options =
+        BigQueryOptions.newBuilder()
+            .setProjectId(projectId)
+            .setLocation(location)
+            .setRetrySettings(ServiceOptions.getDefaultRetrySettings())
+            .build();
+
+    try {
+      if (testingEnabled) {
+        client = new FakeBigQueryMetastoreClient(options);

Review Comment:
   I want to keep this flag because I can run catalog tests against BigQuery Prod service just disabling flag.



-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org

For queries about this service, please contact Infrastructure at:
users@infra.apache.org


---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For additional commands, e-mail: issues-help@iceberg.apache.org


Re: [PR] Catalog: Add BigQuery Metastore Catalog Support [iceberg]

Posted by "nastra (via GitHub)" <gi...@apache.org>.
nastra commented on code in PR #12808:
URL: https://github.com/apache/iceberg/pull/12808#discussion_r2059732820


##########
bigquery/src/main/java/org/apache/iceberg/gcp/bigquery/BigQueryMetastoreCatalog.java:
##########
@@ -0,0 +1,402 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.iceberg.gcp.bigquery;
+
+import com.google.api.services.bigquery.model.Dataset;
+import com.google.api.services.bigquery.model.DatasetList.Datasets;
+import com.google.api.services.bigquery.model.DatasetReference;
+import com.google.api.services.bigquery.model.ExternalCatalogDatasetOptions;
+import com.google.api.services.bigquery.model.TableReference;
+import com.google.cloud.ServiceOptions;
+import com.google.cloud.bigquery.BigQueryOptions;
+import java.io.IOException;
+import java.security.GeneralSecurityException;
+import java.util.List;
+import java.util.Locale;
+import java.util.Map;
+import java.util.Objects;
+import java.util.Set;
+import org.apache.hadoop.conf.Configurable;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.iceberg.BaseMetastoreCatalog;
+import org.apache.iceberg.CatalogProperties;
+import org.apache.iceberg.CatalogUtil;
+import org.apache.iceberg.TableMetadata;
+import org.apache.iceberg.TableOperations;
+import org.apache.iceberg.catalog.Namespace;
+import org.apache.iceberg.catalog.SupportsNamespaces;
+import org.apache.iceberg.catalog.TableIdentifier;
+import org.apache.iceberg.exceptions.AlreadyExistsException;
+import org.apache.iceberg.exceptions.NoSuchNamespaceException;
+import org.apache.iceberg.exceptions.NoSuchTableException;
+import org.apache.iceberg.exceptions.ServiceFailureException;
+import org.apache.iceberg.exceptions.ValidationException;
+import org.apache.iceberg.io.FileIO;
+import org.apache.iceberg.relocated.com.google.common.annotations.VisibleForTesting;
+import org.apache.iceberg.relocated.com.google.common.base.Preconditions;
+import org.apache.iceberg.relocated.com.google.common.base.Strings;
+import org.apache.iceberg.relocated.com.google.common.collect.ImmutableList;
+import org.apache.iceberg.relocated.com.google.common.collect.ImmutableMap;
+import org.apache.iceberg.relocated.com.google.common.collect.Maps;
+import org.apache.iceberg.util.LocationUtil;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/** Iceberg Bigquery Metastore Catalog implementation. */
+public final class BigQueryMetastoreCatalog extends BaseMetastoreCatalog
+    implements SupportsNamespaces, Configurable {
+
+  // User provided properties.
+  public static final String PROPERTIES_KEY_GCP_PROJECT = "gcp-project";
+  public static final String PROPERTIES_KEY_GCP_LOCATION = "gcp-location";
+  public static final String PROPERTIES_KEY_FILTER_UNSUPPORTED_TABLES = "filter-unsupported-tables";
+  public static final String PROPERTIES_KEY_TESTING_ENABLED = "testing-enabled";
+
+  public static final String HIVE_METASTORE_WAREHOUSE_DIR = "hive.metastore.warehouse.dir";
+
+  private static final Logger LOG = LoggerFactory.getLogger(BigQueryMetastoreCatalog.class);
+
+  private static final String DEFAULT_GCP_LOCATION = "us";
+
+  private String catalogPluginName;
+  private Map<String, String> catalogProperties;
+  private FileIO fileIO;
+  private Configuration conf;
+  private String projectId;
+  private String location;
+  private BigQueryMetastoreClient client;
+  private boolean filterUnsupportedTables;
+
+  // Must have a no-arg constructor to be dynamically loaded
+  // initialize(String name, Map<String, String> properties) will be called to complete
+  // initialization
+  public BigQueryMetastoreCatalog() {}
+
+  @Override
+  public void initialize(String inputName, Map<String, String> properties) {
+    if (!properties.containsKey(PROPERTIES_KEY_GCP_PROJECT)) {
+      throw new ValidationException("GCP project must be specified");
+    }
+    projectId = properties.get(PROPERTIES_KEY_GCP_PROJECT);
+    location = properties.getOrDefault(PROPERTIES_KEY_GCP_LOCATION, DEFAULT_GCP_LOCATION);
+    boolean testingEnabled =
+        Boolean.parseBoolean(properties.getOrDefault(PROPERTIES_KEY_TESTING_ENABLED, "false"));
+
+    BigQueryOptions options =
+        BigQueryOptions.newBuilder()
+            .setProjectId(projectId)
+            .setLocation(location)
+            .setRetrySettings(ServiceOptions.getDefaultRetrySettings())
+            .build();
+
+    try {
+      if (testingEnabled) {
+        client = new FakeBigQueryMetastoreClient(options);

Review Comment:
   see my comment in https://github.com/apache/iceberg/pull/12808/files#r2059731731 on this



-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org

For queries about this service, please contact Infrastructure at:
users@infra.apache.org


---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For additional commands, e-mail: issues-help@iceberg.apache.org


Re: [PR] Catalog: Add BigQuery Metastore Catalog Support [iceberg]

Posted by "nastra (via GitHub)" <gi...@apache.org>.
nastra commented on code in PR #12808:
URL: https://github.com/apache/iceberg/pull/12808#discussion_r2059745760


##########
bigquery/src/test/java/org/apache/iceberg/gcp/bigquery/TestBigQueryCatalog.java:
##########
@@ -0,0 +1,673 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.iceberg.gcp.bigquery;
+
+import static org.apache.iceberg.CatalogUtil.ICEBERG_CATALOG_TYPE;
+import static org.apache.iceberg.CatalogUtil.ICEBERG_CATALOG_TYPE_BIGQUERY;
+import static org.apache.iceberg.gcp.bigquery.BigQueryMetastoreCatalog.PROJECT_ID;
+import static org.apache.iceberg.gcp.bigquery.BigQueryMetastoreCatalog.TESTING_ENABLED;
+import static org.apache.iceberg.types.Types.NestedField.optional;
+import static org.assertj.core.api.Assertions.assertThat;
+import static org.assertj.core.api.Assertions.assertThatThrownBy;
+
+import java.io.File;
+import java.util.List;
+import java.util.Map;
+import org.apache.iceberg.CatalogProperties;
+import org.apache.iceberg.CatalogUtil;
+import org.apache.iceberg.PartitionSpec;
+import org.apache.iceberg.Schema;
+import org.apache.iceberg.SortOrder;
+import org.apache.iceberg.Table;
+import org.apache.iceberg.TableUtil;
+import org.apache.iceberg.Transaction;
+import org.apache.iceberg.catalog.CatalogTests;
+import org.apache.iceberg.catalog.Namespace;
+import org.apache.iceberg.catalog.TableIdentifier;
+import org.apache.iceberg.exceptions.NoSuchNamespaceException;
+import org.apache.iceberg.exceptions.NoSuchTableException;
+import org.apache.iceberg.expressions.Expressions;
+import org.apache.iceberg.relocated.com.google.common.collect.ImmutableMap;
+import org.apache.iceberg.types.Types;
+import org.junit.jupiter.api.AfterEach;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Disabled;
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.io.TempDir;
+import org.junit.jupiter.params.ParameterizedTest;
+import org.junit.jupiter.params.provider.ValueSource;
+
+public class TestBigQueryCatalog extends CatalogTests<BigQueryMetastoreCatalog> {
+
+  private static final String GCP_PROJECT = "project-id";
+  private static final String CATALOG_ID = "catalog-name";
+
+  // another schema that is not the same
+  protected static final Schema OTHER_SCHEMA_OPTIONAL =
+      new Schema(optional(1, "some_id", Types.IntegerType.get()));
+
+  // Schema passed to create tables
+  protected static final Schema SCHEMA_OPTIONAL =
+      new Schema(
+          optional(3, "id", Types.IntegerType.get(), "unique ID 🤪"),
+          optional(4, "data", Types.StringType.get()));
+
+  // This is the actual schema for the table, with column IDs reassigned
+  protected static final Schema REPLACE_SCHEMA_OPTIONAL =
+      new Schema(
+          optional(2, "id", Types.IntegerType.get(), "unique ID 🤪"),
+          optional(3, "data", Types.StringType.get()));
+
+  protected static final PartitionSpec REPLACE_SPEC_OPTIONAL =
+      PartitionSpec.builderFor(REPLACE_SCHEMA_OPTIONAL).bucket("id", 16).withSpecId(1).build();
+
+  protected static final SortOrder REPLACE_WRITE_ORDER_OPTIONAL =
+      SortOrder.builderFor(REPLACE_SCHEMA_OPTIONAL)
+          .asc(Expressions.bucket("id", 16))
+          .asc("id")
+          .build();
+
+  @TempDir private File tempFolder;
+  private BigQueryMetastoreCatalog catalog;
+  private String warehouseLocation;
+
+  @BeforeEach
+  public void before() throws Exception {
+    catalog = initCatalog(CATALOG_ID, ImmutableMap.of());
+  }
+
+  @AfterEach
+  public void after() throws Exception {
+    // Drop all tables in all datasets
+    List<Namespace> namespaces = catalog().listNamespaces(Namespace.empty());
+    for (Namespace namespace : namespaces) {
+      List<TableIdentifier> tables = catalog().listTables(namespace);
+      for (TableIdentifier table : tables) {
+        try {
+          catalog().dropTable(table, true); // drop with purge
+        } catch (NoSuchTableException e) {
+          // Table might be dropped by a previous iteration, ignore
+        }
+      }
+
+      // Drop all datasets (except any initial ones)
+      try {
+        catalog().dropNamespace(namespace);
+      } catch (NoSuchNamespaceException e) {
+        // Namespace might be dropped by a previous iteration, ignore
+      }
+    }
+  }
+
+  @Override
+  protected boolean requiresNamespaceCreate() {
+    return true;
+  }
+
+  @Override
+  protected boolean supportsNamesWithSlashes() {
+    return false;
+  }
+
+  @Override
+  protected boolean supportsNamesWithDot() {
+    return false;
+  }
+
+  @Override
+  public BigQueryMetastoreCatalog catalog() {
+    return catalog;
+  }
+
+  @Override
+  protected BigQueryMetastoreCatalog initCatalog(
+      String catalogName, Map<String, String> additionalProperties) {
+
+    warehouseLocation = tempFolder.toPath().resolve("hive-warehouse").toString();
+
+    Map<String, String> properties =
+        Map.of(
+            ICEBERG_CATALOG_TYPE,
+            ICEBERG_CATALOG_TYPE_BIGQUERY,
+            PROJECT_ID,
+            GCP_PROJECT,
+            CatalogProperties.WAREHOUSE_LOCATION,
+            warehouseLocation,
+            TESTING_ENABLED,
+            "true",
+            CatalogProperties.TABLE_DEFAULT_PREFIX + "default-key1",
+            "catalog-default-key1",
+            CatalogProperties.TABLE_DEFAULT_PREFIX + "default-key2",
+            "catalog-default-key2",
+            CatalogProperties.TABLE_DEFAULT_PREFIX + "override-key3",
+            "catalog-default-key3",
+            CatalogProperties.TABLE_OVERRIDE_PREFIX + "override-key3",
+            "catalog-override-key3",
+            CatalogProperties.TABLE_OVERRIDE_PREFIX + "override-key4",
+            "catalog-override-key4");
+
+    BigQueryMetastoreCatalog tmpCatalog =
+        (BigQueryMetastoreCatalog)
+            CatalogUtil.buildIcebergCatalog(
+                catalogName,
+                ImmutableMap.<String, String>builder()
+                    .putAll(properties)
+                    .putAll(additionalProperties)
+                    .build(),
+                null);
+
+    return tmpCatalog;
+  }
+
+  @Test
+  public void testRenameTable() {
+    if (requiresNamespaceCreate()) {
+      catalog.createNamespace(NS);
+    }
+
+    assertThat(catalog.tableExists(TABLE))
+        .as("Source table should not exist before create")
+        .isFalse();
+
+    catalog.buildTable(TABLE, SCHEMA).create();
+    assertThat(catalog.tableExists(TABLE)).as("Table should exist after create").isTrue();
+
+    assertThat(catalog.tableExists(RENAMED_TABLE))
+        .as("Destination table should not exist before rename")
+        .isFalse();
+
+    assertThatThrownBy(() -> catalog.renameTable(TABLE, RENAMED_TABLE))
+        .isInstanceOf(UnsupportedOperationException.class)
+        .hasMessageContaining("Table rename operation is unsupported");
+
+    catalog.dropTable(TABLE);
+  }
+
+  @Disabled(
+      "BigQuery does not allow adding a REQUIRED column to an existing table schema. "
+          + "When this operation is attempted, an exception gets thrown.")
+  @Test
+  public void testOverrideTablePropertiesReplaceTransaction() {}

Review Comment:
   given that the tests here that live in the Iceberg codebase are always running against the Fake client, I would suggest to remove all of those `Disabled` tests, since they all pass against the Fake client. If you want to run the catalog tests against an actual client, you can then basically just create a new `TestRealBigQueryCatalog` that initializes the correct client where you disable the tests that don't work



-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org

For queries about this service, please contact Infrastructure at:
users@infra.apache.org


---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For additional commands, e-mail: issues-help@iceberg.apache.org


Re: [PR] Catalog: Add BigQuery Metastore Catalog Support [iceberg]

Posted by "talatuyarer (via GitHub)" <gi...@apache.org>.
talatuyarer commented on code in PR #12808:
URL: https://github.com/apache/iceberg/pull/12808#discussion_r2059828506


##########
bigquery/src/main/java/org/apache/iceberg/gcp/bigquery/BigQueryMetastoreCatalog.java:
##########
@@ -0,0 +1,402 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.iceberg.gcp.bigquery;
+
+import com.google.api.services.bigquery.model.Dataset;
+import com.google.api.services.bigquery.model.DatasetList.Datasets;
+import com.google.api.services.bigquery.model.DatasetReference;
+import com.google.api.services.bigquery.model.ExternalCatalogDatasetOptions;
+import com.google.api.services.bigquery.model.TableReference;
+import com.google.cloud.ServiceOptions;
+import com.google.cloud.bigquery.BigQueryOptions;
+import java.io.IOException;
+import java.security.GeneralSecurityException;
+import java.util.List;
+import java.util.Locale;
+import java.util.Map;
+import java.util.Objects;
+import java.util.Set;
+import org.apache.hadoop.conf.Configurable;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.iceberg.BaseMetastoreCatalog;
+import org.apache.iceberg.CatalogProperties;
+import org.apache.iceberg.CatalogUtil;
+import org.apache.iceberg.TableMetadata;
+import org.apache.iceberg.TableOperations;
+import org.apache.iceberg.catalog.Namespace;
+import org.apache.iceberg.catalog.SupportsNamespaces;
+import org.apache.iceberg.catalog.TableIdentifier;
+import org.apache.iceberg.exceptions.AlreadyExistsException;
+import org.apache.iceberg.exceptions.NoSuchNamespaceException;
+import org.apache.iceberg.exceptions.NoSuchTableException;
+import org.apache.iceberg.exceptions.ServiceFailureException;
+import org.apache.iceberg.exceptions.ValidationException;
+import org.apache.iceberg.io.FileIO;
+import org.apache.iceberg.relocated.com.google.common.annotations.VisibleForTesting;
+import org.apache.iceberg.relocated.com.google.common.base.Preconditions;
+import org.apache.iceberg.relocated.com.google.common.base.Strings;
+import org.apache.iceberg.relocated.com.google.common.collect.ImmutableList;
+import org.apache.iceberg.relocated.com.google.common.collect.ImmutableMap;
+import org.apache.iceberg.relocated.com.google.common.collect.Maps;
+import org.apache.iceberg.util.LocationUtil;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/** Iceberg Bigquery Metastore Catalog implementation. */
+public final class BigQueryMetastoreCatalog extends BaseMetastoreCatalog
+    implements SupportsNamespaces, Configurable {
+
+  // User provided properties.
+  public static final String PROPERTIES_KEY_GCP_PROJECT = "gcp-project";
+  public static final String PROPERTIES_KEY_GCP_LOCATION = "gcp-location";
+  public static final String PROPERTIES_KEY_FILTER_UNSUPPORTED_TABLES = "filter-unsupported-tables";
+  public static final String PROPERTIES_KEY_TESTING_ENABLED = "testing-enabled";
+
+  public static final String HIVE_METASTORE_WAREHOUSE_DIR = "hive.metastore.warehouse.dir";
+
+  private static final Logger LOG = LoggerFactory.getLogger(BigQueryMetastoreCatalog.class);
+
+  private static final String DEFAULT_GCP_LOCATION = "us";
+
+  private String catalogPluginName;
+  private Map<String, String> catalogProperties;
+  private FileIO fileIO;
+  private Configuration conf;
+  private String projectId;
+  private String location;
+  private BigQueryMetastoreClient client;
+  private boolean filterUnsupportedTables;
+
+  // Must have a no-arg constructor to be dynamically loaded
+  // initialize(String name, Map<String, String> properties) will be called to complete
+  // initialization
+  public BigQueryMetastoreCatalog() {}
+
+  @Override
+  public void initialize(String inputName, Map<String, String> properties) {
+    if (!properties.containsKey(PROPERTIES_KEY_GCP_PROJECT)) {
+      throw new ValidationException("GCP project must be specified");
+    }
+    projectId = properties.get(PROPERTIES_KEY_GCP_PROJECT);
+    location = properties.getOrDefault(PROPERTIES_KEY_GCP_LOCATION, DEFAULT_GCP_LOCATION);
+    boolean testingEnabled =
+        Boolean.parseBoolean(properties.getOrDefault(PROPERTIES_KEY_TESTING_ENABLED, "false"));
+
+    BigQueryOptions options =
+        BigQueryOptions.newBuilder()
+            .setProjectId(projectId)
+            .setLocation(location)
+            .setRetrySettings(ServiceOptions.getDefaultRetrySettings())
+            .build();
+
+    try {
+      if (testingEnabled) {
+        client = new FakeBigQueryMetastoreClient(options);

Review Comment:
   Done



##########
bigquery/src/main/java/org/apache/iceberg/gcp/bigquery/BigQueryMetastoreClient.java:
##########
@@ -0,0 +1,121 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.iceberg.gcp.bigquery;
+
+import com.google.api.services.bigquery.model.Dataset;
+import com.google.api.services.bigquery.model.DatasetList.Datasets;
+import com.google.api.services.bigquery.model.DatasetReference;
+import com.google.api.services.bigquery.model.Table;
+import com.google.api.services.bigquery.model.TableList.Tables;
+import com.google.api.services.bigquery.model.TableReference;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+
+/**
+ * A client of Google BigQuery Metastore functions over the BigQuery service. Uses the Google
+ * BigQuery API.
+ */
+public interface BigQueryMetastoreClient {
+
+  /**
+   * Creates and returns a new dataset.
+   *
+   * @param dataset the dataset to create
+   */
+  Dataset create(Dataset dataset);
+
+  /**
+   * Returns a dataset.
+   *
+   * @param datasetReference full dataset reference
+   */
+  Dataset load(DatasetReference datasetReference);
+
+  /**
+   * Deletes a dataset.
+   *
+   * @param datasetReference full dataset reference
+   */
+  void delete(DatasetReference datasetReference);
+
+  /**
+   * Updates parameters of a dataset or adds them if did not exist, leaving already-existing ones
+   * intact.
+   *
+   * @param datasetReference full dataset reference
+   * @param parameters metadata parameters to add
+   * @return dataset after patch
+   */
+  Dataset setDatasetParameters(DatasetReference datasetReference, Map<String, String> parameters);

Review Comment:
   Done



-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org

For queries about this service, please contact Infrastructure at:
users@infra.apache.org


---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For additional commands, e-mail: issues-help@iceberg.apache.org


Re: [PR] Catalog: Add BigQuery Metastore Catalog Support [iceberg]

Posted by "talatuyarer (via GitHub)" <gi...@apache.org>.
talatuyarer commented on code in PR #12808:
URL: https://github.com/apache/iceberg/pull/12808#discussion_r2059829007


##########
build.gradle:
##########
@@ -657,6 +657,47 @@ project(':iceberg-delta-lake') {
   }
 }
 
+project(':iceberg-bigquery') {
+  test {
+    useJUnitPlatform()
+  }
+
+  dependencies {
+    implementation project(path: ':iceberg-bundled-guava', configuration: 'shadow')
+    implementation project(':iceberg-gcp')
+
+    implementation platform(libs.google.libraries.bom)
+    compileOnly "com.google.cloud:google-cloud-storage"
+    implementation "com.google.cloud:google-cloud-bigquery"
+    implementation "com.google.cloud:google-cloud-core"
+
+    api project(':iceberg-api')
+    implementation project(':iceberg-common')
+    implementation project(':iceberg-core')
+
+    compileOnly libs.hadoop3.common
+    compileOnly (libs.hive4.metastore) {
+      exclude group: 'org.apache.avro', module: 'avro'
+      exclude group: 'org.slf4j', module: 'slf4j-log4j12'
+      exclude group: 'org.pentaho'
+      exclude group: 'org.apache.hbase'
+      exclude group: 'org.apache.logging.log4j'
+      exclude group: 'co.cask.tephra'
+      exclude group: 'com.google.code.findbugs', module: 'jsr305'
+      exclude group: 'org.eclipse.jetty.aggregate', module: 'jetty-all'
+      exclude group: 'org.eclipse.jetty.orbit', module: 'javax.servlet'
+      exclude group: 'org.apache.parquet', module: 'parquet-hadoop-bundle'
+      exclude group: 'com.tdunning', module: 'json'
+      exclude group: 'javax.transaction', module: 'transaction-api'
+      exclude group: 'com.zaxxer', module: 'HikariCP'
+      exclude group: 'com.google.code.gson', module: 'gson'
+  }

Review Comment:
   Good Catch!



-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org

For queries about this service, please contact Infrastructure at:
users@infra.apache.org


---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For additional commands, e-mail: issues-help@iceberg.apache.org


Re: [PR] Catalog: Add BigQuery Metastore Catalog Support [iceberg]

Posted by "talatuyarer (via GitHub)" <gi...@apache.org>.
talatuyarer commented on code in PR #12808:
URL: https://github.com/apache/iceberg/pull/12808#discussion_r2064334228


##########
bigquery/src/main/java/org/apache/iceberg/gcp/bigquery/BigQueryMetastoreCatalog.java:
##########
@@ -0,0 +1,394 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.iceberg.gcp.bigquery;
+
+import com.google.api.services.bigquery.model.Dataset;
+import com.google.api.services.bigquery.model.DatasetList.Datasets;
+import com.google.api.services.bigquery.model.DatasetReference;
+import com.google.api.services.bigquery.model.ExternalCatalogDatasetOptions;
+import com.google.api.services.bigquery.model.TableReference;
+import com.google.cloud.ServiceOptions;
+import com.google.cloud.bigquery.BigQueryOptions;
+import java.io.IOException;
+import java.security.GeneralSecurityException;
+import java.util.List;
+import java.util.Locale;
+import java.util.Map;
+import java.util.Objects;
+import java.util.Set;
+import org.apache.hadoop.conf.Configurable;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.iceberg.BaseMetastoreCatalog;
+import org.apache.iceberg.CatalogProperties;
+import org.apache.iceberg.CatalogUtil;
+import org.apache.iceberg.TableMetadata;
+import org.apache.iceberg.TableOperations;
+import org.apache.iceberg.catalog.Namespace;
+import org.apache.iceberg.catalog.SupportsNamespaces;
+import org.apache.iceberg.catalog.TableIdentifier;
+import org.apache.iceberg.exceptions.AlreadyExistsException;
+import org.apache.iceberg.exceptions.NoSuchNamespaceException;
+import org.apache.iceberg.exceptions.NoSuchTableException;
+import org.apache.iceberg.exceptions.ServiceFailureException;
+import org.apache.iceberg.exceptions.ValidationException;
+import org.apache.iceberg.io.FileIO;
+import org.apache.iceberg.relocated.com.google.common.annotations.VisibleForTesting;
+import org.apache.iceberg.relocated.com.google.common.base.Preconditions;
+import org.apache.iceberg.relocated.com.google.common.base.Strings;
+import org.apache.iceberg.relocated.com.google.common.collect.ImmutableList;
+import org.apache.iceberg.relocated.com.google.common.collect.ImmutableMap;
+import org.apache.iceberg.relocated.com.google.common.collect.Maps;
+import org.apache.iceberg.util.LocationUtil;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/** Iceberg Bigquery Metastore Catalog implementation. */
+public class BigQueryMetastoreCatalog extends BaseMetastoreCatalog
+    implements SupportsNamespaces, Configurable {
+
+  // User provided properties.
+  public static final String PROJECT_ID = "gcp-project";
+  public static final String GCP_LOCATION = "gcp-location";
+  public static final String LIST_ALL_TABLES = "list-all-tables";
+
+  public static final String HIVE_METASTORE_WAREHOUSE_DIR = "hive.metastore.warehouse.dir";
+
+  private static final Logger LOG = LoggerFactory.getLogger(BigQueryMetastoreCatalog.class);
+
+  private static final String DEFAULT_GCP_LOCATION = "us";
+
+  private String catalogName;
+  private Map<String, String> catalogProperties;
+  private FileIO fileIO;
+  private Configuration conf;
+  private String projectId;
+  private String location;
+  private BigQueryMetastoreClient client;
+  private boolean listAllTables;
+
+  // Must have a no-arg constructor to be dynamically loaded
+  // initialize(String name, Map<String, String> properties) will be called to complete
+  // initialization
+  public BigQueryMetastoreCatalog() {}
+
+  @Override
+  public void initialize(String inputName, Map<String, String> properties) {
+    Preconditions.checkArgument(
+        properties.containsKey(PROJECT_ID), "GCP project must be specified");
+
+    projectId = properties.get(PROJECT_ID);
+    location = properties.getOrDefault(GCP_LOCATION, DEFAULT_GCP_LOCATION);
+
+    BigQueryOptions options =
+        BigQueryOptions.newBuilder()
+            .setProjectId(projectId)
+            .setLocation(location)
+            .setRetrySettings(ServiceOptions.getDefaultRetrySettings())
+            .build();
+
+    try {
+      client = new BigQueryMetastoreClientImpl(options);
+    } catch (IOException e) {
+      throw new ServiceFailureException(e, "Creating BigQuery client failed");
+    } catch (GeneralSecurityException e) {
+      throw new ValidationException(e, "Creating BigQuery client failed due to a security issue");
+    }
+
+    initialize(inputName, properties, projectId, location, client);
+  }
+
+  @VisibleForTesting
+  void initialize(
+      String inputName,
+      Map<String, String> properties,
+      String initialProjectId,
+      String initialLocation,
+      BigQueryMetastoreClient bigQueryMetaStoreClient) {
+    Preconditions.checkArgument(bigQueryMetaStoreClient != null, "BigQuery client can not be null");
+    this.catalogName = inputName;
+    this.catalogProperties = ImmutableMap.copyOf(properties);
+    this.projectId = initialProjectId;
+    this.location = initialLocation;
+    this.client = bigQueryMetaStoreClient;
+
+    if (this.conf == null) {
+      LOG.warn("No configuration was set, using the default environment Configuration");
+      this.conf = new Configuration();
+    }
+
+    LOG.info("Using BigQuery Metastore Iceberg Catalog: {}", inputName);
+
+    if (properties.containsKey(CatalogProperties.WAREHOUSE_LOCATION)) {
+      this.conf.set(
+          HIVE_METASTORE_WAREHOUSE_DIR,
+          LocationUtil.stripTrailingSlash(properties.get(CatalogProperties.WAREHOUSE_LOCATION)));
+    }
+
+    String fileIoImpl =
+        properties.getOrDefault(
+            CatalogProperties.FILE_IO_IMPL, "org.apache.iceberg.hadoop.HadoopFileIO");
+    this.fileIO = CatalogUtil.loadFileIO(fileIoImpl, properties, conf);
+
+    this.listAllTables = Boolean.parseBoolean(properties.getOrDefault(LIST_ALL_TABLES, "true"));
+  }
+
+  @Override
+  protected TableOperations newTableOps(TableIdentifier identifier) {
+    return new BigQueryTableOperations(client, fileIO, toTableReference(identifier), conf);
+  }
+
+  @Override
+  protected String defaultWarehouseLocation(TableIdentifier identifier) {
+    String locationUri = null;
+    DatasetReference datasetReference = toDatasetReference(identifier.namespace());
+    Dataset dataset = client.load(datasetReference);
+    if (dataset != null && dataset.getExternalCatalogDatasetOptions() != null) {
+      locationUri = dataset.getExternalCatalogDatasetOptions().getDefaultStorageLocationUri();
+    }
+
+    return String.format(
+        "%s/%s",
+        Strings.isNullOrEmpty(locationUri)
+            ? createDefaultStorageLocationUri(datasetReference.getDatasetId())
+            : LocationUtil.stripTrailingSlash(locationUri),
+        identifier.name());
+  }
+
+  @Override
+  public List<TableIdentifier> listTables(Namespace namespace) {
+    validateNamespace(namespace);
+
+    return client.list(toDatasetReference(namespace), listAllTables).stream()
+        .map(
+            table -> TableIdentifier.of(namespace.level(0), table.getTableReference().getTableId()))
+        .collect(ImmutableList.toImmutableList());
+  }
+
+  @Override
+  public boolean dropTable(TableIdentifier identifier, boolean purge) {
+    try {
+      TableOperations ops = newTableOps(identifier);
+      TableMetadata lastMetadata = ops.current();
+
+      client.delete(toTableReference(identifier));
+
+      if (purge && lastMetadata != null) {
+        CatalogUtil.dropTableData(ops.io(), lastMetadata);
+      }
+    } catch (NoSuchTableException e) {
+      return false;
+    }
+
+    return true;
+  }
+
+  @Override
+  public void renameTable(TableIdentifier from, TableIdentifier to) {
+    validateNamespace(from.namespace());
+    validateNamespace(to.namespace());
+
+    if (!namespaceExists(to.namespace())) {
+      throw new NoSuchNamespaceException("Namespace does not exist: %s", to.namespace());
+    }
+
+    if (!namespaceExists(from.namespace())) {
+      throw new NoSuchNamespaceException("Namespace does not exist: %s", from.namespace());
+    }
+
+    // Check if the source table exists
+    if (!tableExists(from)) {
+      throw new NoSuchTableException("Table does not exist: %s", from.name());
+    }
+
+    if (!from.namespace().equals(to.namespace())) {
+      throw new ValidationException("New table name must be in the same namespace");
+    }
+
+    // Check if the destination table already exists
+    if (tableExists(to)) {
+      throw new AlreadyExistsException("Table already exists: %s", to.name());
+    }
+
+    // TODO: Enable once supported by BigQuery API.
+    throw new UnsupportedOperationException("Table rename operation is unsupported.");
+  }
+
+  @Override
+  public void createNamespace(Namespace namespace, Map<String, String> metadata) {
+    Dataset builder = new Dataset();
+    DatasetReference datasetReference = toDatasetReference(namespace);
+    builder.setLocation(this.location);
+    builder.setDatasetReference(datasetReference);
+    builder.setExternalCatalogDatasetOptions(
+        BigQueryMetastoreUtils.createExternalCatalogDatasetOptions(
+            createDefaultStorageLocationUri(datasetReference.getDatasetId()), metadata));
+
+    client.create(builder);
+  }
+
+  @Override
+  public List<Namespace> listNamespaces() {
+    try {
+      return listNamespaces(Namespace.empty());
+    } catch (NoSuchNamespaceException e) {
+      return ImmutableList.of();
+    }
+  }
+
+  /**
+   * Since this catalog only supports one-level namespaces, it always returns an empty list unless
+   * passed an empty namespace to list all namespaces within the catalog.
+   */
+  @Override
+  public List<Namespace> listNamespaces(Namespace namespace) {
+    List<Datasets> allDatasets = client.list(projectId);
+
+    ImmutableList<Namespace> namespaces =
+        allDatasets.stream()
+            .map(BigQueryMetastoreCatalog::toNamespace)
+            .collect(ImmutableList.toImmutableList());
+    if (namespaces.isEmpty()) {
+      throw new NoSuchNamespaceException("Namespace does not exist: %s", namespace);
+    }
+
+    return namespaces;
+  }
+
+  @Override
+  public boolean dropNamespace(Namespace namespace) {
+    try {
+      client.delete(toDatasetReference(namespace));
+      // We don't delete the data folder for safety, which aligns with Hive Metastore's default
+      // behavior.
+      // We can support database or catalog level config controlling file deletion in the future.
+      return true;
+    } catch (NoSuchNamespaceException e) {
+      return false;
+    }
+  }
+
+  @Override
+  public boolean setProperties(Namespace namespace, Map<String, String> properties) {
+    Dataset dataset = client.load(toDatasetReference(namespace));
+
+    ExternalCatalogDatasetOptions existingOptions = dataset.getExternalCatalogDatasetOptions();
+    Map<String, String> existingParameters =
+        existingOptions != null ? existingOptions.getParameters() : null;
+
+    Map<String, String> newParameters = Maps.newHashMap();
+    if (existingParameters != null) {
+      newParameters.putAll(existingParameters);
+    }
+
+    newParameters.putAll(properties);
+
+    if (Objects.equals(existingParameters, newParameters)) {
+      // No change in parameters detected
+      return false;
+    }
+
+    client.setParameters(toDatasetReference(namespace), properties);
+    return true;
+  }
+
+  @Override
+  public boolean removeProperties(Namespace namespace, Set<String> properties) {
+    client.removeParameters(toDatasetReference(namespace), properties);
+    return true;
+  }
+
+  @SuppressWarnings("FormatStringAnnotation")
+  @Override
+  public Map<String, String> loadNamespaceMetadata(Namespace namespace) {
+    try {
+      return toMetadata(client.load(toDatasetReference(namespace)));
+    } catch (IllegalArgumentException e) {
+      throw new NoSuchNamespaceException(e.getMessage());
+    }
+  }
+
+  @Override
+  public String name() {
+    return catalogName;
+  }
+
+  @Override
+  protected Map<String, String> properties() {
+    return catalogProperties == null ? ImmutableMap.of() : catalogProperties;
+  }
+
+  @Override
+  public void setConf(Configuration conf) {
+    this.conf = new Configuration(conf);
+  }
+
+  @Override
+  public Configuration getConf() {
+    return this.conf;
+  }
+
+  private String createDefaultStorageLocationUri(String dbId) {
+    String warehouseLocation = conf.get(HIVE_METASTORE_WAREHOUSE_DIR);
+    Preconditions.checkNotNull(warehouseLocation, "Data warehouse location is not set");
+    return String.format("%s/%s.db", LocationUtil.stripTrailingSlash(warehouseLocation), dbId);
+  }
+
+  private static Namespace toNamespace(Datasets dataset) {

Review Comment:
   I can not use this without static where i used. 



-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org

For queries about this service, please contact Infrastructure at:
users@infra.apache.org


---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For additional commands, e-mail: issues-help@iceberg.apache.org


Re: [PR] Catalog: Add BigQuery Metastore Catalog Support [iceberg]

Posted by "nastra (via GitHub)" <gi...@apache.org>.
nastra commented on code in PR #12808:
URL: https://github.com/apache/iceberg/pull/12808#discussion_r2063939711


##########
bigquery/src/test/java/org/apache/iceberg/gcp/bigquery/TestBigQueryCatalog.java:
##########
@@ -0,0 +1,191 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.iceberg.gcp.bigquery;
+
+import static org.apache.iceberg.CatalogUtil.ICEBERG_CATALOG_TYPE;
+import static org.apache.iceberg.CatalogUtil.ICEBERG_CATALOG_TYPE_BIGQUERY;
+import static org.apache.iceberg.gcp.bigquery.BigQueryMetastoreCatalog.PROJECT_ID;
+import static org.assertj.core.api.Assertions.assertThat;
+import static org.assertj.core.api.Assertions.assertThatThrownBy;
+
+import java.io.File;
+import java.util.List;
+import java.util.Map;
+import org.apache.iceberg.CatalogProperties;
+import org.apache.iceberg.PartitionSpec;
+import org.apache.iceberg.TableUtil;
+import org.apache.iceberg.catalog.CatalogTests;
+import org.apache.iceberg.catalog.Namespace;
+import org.apache.iceberg.catalog.TableIdentifier;
+import org.apache.iceberg.exceptions.NoSuchNamespaceException;
+import org.apache.iceberg.exceptions.NoSuchTableException;
+import org.apache.iceberg.relocated.com.google.common.collect.ImmutableMap;
+import org.junit.jupiter.api.AfterEach;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Disabled;
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.io.TempDir;
+import org.junit.jupiter.params.ParameterizedTest;
+import org.junit.jupiter.params.provider.ValueSource;
+
+public class TestBigQueryCatalog extends CatalogTests<BigQueryMetastoreCatalog> {
+  @TempDir private File tempFolder;
+  private BigQueryMetastoreCatalog catalog;
+  private String warehouseLocation;

Review Comment:
   can be inlined



-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org

For queries about this service, please contact Infrastructure at:
users@infra.apache.org


---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For additional commands, e-mail: issues-help@iceberg.apache.org


Re: [PR] Catalog: Add BigQuery Metastore Catalog Support [iceberg]

Posted by "talatuyarer (via GitHub)" <gi...@apache.org>.
talatuyarer commented on code in PR #12808:
URL: https://github.com/apache/iceberg/pull/12808#discussion_r2067872740


##########
bigquery/src/main/java/org/apache/iceberg/gcp/bigquery/BigQueryMetastoreCatalog.java:
##########
@@ -0,0 +1,405 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.iceberg.gcp.bigquery;
+
+import com.google.api.services.bigquery.model.Dataset;
+import com.google.api.services.bigquery.model.DatasetList.Datasets;
+import com.google.api.services.bigquery.model.DatasetReference;
+import com.google.api.services.bigquery.model.ExternalCatalogDatasetOptions;
+import com.google.api.services.bigquery.model.TableReference;
+import com.google.cloud.ServiceOptions;
+import com.google.cloud.bigquery.BigQueryOptions;
+import java.io.IOException;
+import java.security.GeneralSecurityException;
+import java.util.List;
+import java.util.Locale;
+import java.util.Map;
+import java.util.Objects;
+import java.util.Set;
+import org.apache.hadoop.conf.Configurable;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.iceberg.BaseMetastoreCatalog;
+import org.apache.iceberg.CatalogProperties;
+import org.apache.iceberg.CatalogUtil;
+import org.apache.iceberg.TableMetadata;
+import org.apache.iceberg.TableOperations;
+import org.apache.iceberg.catalog.Namespace;
+import org.apache.iceberg.catalog.SupportsNamespaces;
+import org.apache.iceberg.catalog.TableIdentifier;
+import org.apache.iceberg.exceptions.AlreadyExistsException;
+import org.apache.iceberg.exceptions.NoSuchNamespaceException;
+import org.apache.iceberg.exceptions.NoSuchTableException;
+import org.apache.iceberg.exceptions.ServiceFailureException;
+import org.apache.iceberg.exceptions.ValidationException;
+import org.apache.iceberg.io.FileIO;
+import org.apache.iceberg.relocated.com.google.common.annotations.VisibleForTesting;
+import org.apache.iceberg.relocated.com.google.common.base.Preconditions;
+import org.apache.iceberg.relocated.com.google.common.base.Strings;
+import org.apache.iceberg.relocated.com.google.common.collect.ImmutableList;
+import org.apache.iceberg.relocated.com.google.common.collect.ImmutableMap;
+import org.apache.iceberg.relocated.com.google.common.collect.Maps;
+import org.apache.iceberg.util.LocationUtil;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/** Iceberg Bigquery Metastore Catalog implementation. */
+public class BigQueryMetastoreCatalog extends BaseMetastoreCatalog
+    implements SupportsNamespaces, Configurable {
+
+  // User provided properties.
+  public static final String PROJECT_ID = "gcp-project";
+  public static final String GCP_LOCATION = "gcp-location";
+  public static final String LIST_ALL_TABLES = "list-all-tables";
+
+  public static final String HIVE_METASTORE_WAREHOUSE_DIR = "hive.metastore.warehouse.dir";
+
+  private static final Logger LOG = LoggerFactory.getLogger(BigQueryMetastoreCatalog.class);
+
+  private static final String DEFAULT_GCP_LOCATION = "us";
+
+  private String catalogName;
+  private Map<String, String> catalogProperties;
+  private FileIO fileIO;
+  private Configuration conf;
+  private String projectId;
+  private String location;
+  private BigQueryMetastoreClient client;
+  private boolean listAllTables;
+
+  // Must have a no-arg constructor to be dynamically loaded
+  // initialize(String name, Map<String, String> properties) will be called to complete
+  // initialization
+  public BigQueryMetastoreCatalog() {}
+
+  @Override
+  public void initialize(String inputName, Map<String, String> properties) {
+    Preconditions.checkArgument(
+        properties.containsKey(PROJECT_ID),
+        "Invalid GCP project: %s must be specified",
+        PROJECT_ID);
+
+    projectId = properties.get(PROJECT_ID);
+    location = properties.getOrDefault(GCP_LOCATION, DEFAULT_GCP_LOCATION);
+
+    BigQueryOptions options =
+        BigQueryOptions.newBuilder()
+            .setProjectId(projectId)
+            .setLocation(location)
+            .setRetrySettings(ServiceOptions.getDefaultRetrySettings())
+            .build();
+
+    try {
+      client = new BigQueryMetastoreClientImpl(options);
+    } catch (IOException e) {
+      throw new ServiceFailureException(e, "Creating BigQuery client failed");
+    } catch (GeneralSecurityException e) {
+      throw new ValidationException(e, "Creating BigQuery client failed due to a security issue");
+    }
+
+    initialize(inputName, properties, projectId, location, client);
+  }
+
+  @VisibleForTesting
+  void initialize(
+      String inputName,
+      Map<String, String> properties,
+      String initialProjectId,
+      String initialLocation,
+      BigQueryMetastoreClient bigQueryMetaStoreClient) {
+    Preconditions.checkArgument(bigQueryMetaStoreClient != null, "Invalid BigQuery client: null");
+    this.catalogName = inputName;
+    this.catalogProperties = ImmutableMap.copyOf(properties);
+    this.projectId = initialProjectId;
+    this.location = initialLocation;
+    this.client = bigQueryMetaStoreClient;
+
+    if (this.conf == null) {
+      LOG.warn("No configuration was set, using the default environment Configuration");
+      this.conf = new Configuration();
+    }
+
+    LOG.info("Using BigQuery Metastore Iceberg Catalog: {}", inputName);
+
+    if (properties.containsKey(CatalogProperties.WAREHOUSE_LOCATION)) {
+      this.conf.set(
+          HIVE_METASTORE_WAREHOUSE_DIR,
+          LocationUtil.stripTrailingSlash(properties.get(CatalogProperties.WAREHOUSE_LOCATION)));
+    }
+
+    String fileIoImpl =

Review Comment:
   Done



-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org

For queries about this service, please contact Infrastructure at:
users@infra.apache.org


---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For additional commands, e-mail: issues-help@iceberg.apache.org


Re: [PR] Catalog: Add BigQuery Metastore Catalog Support [iceberg]

Posted by "danielcweeks (via GitHub)" <gi...@apache.org>.
danielcweeks commented on code in PR #12808:
URL: https://github.com/apache/iceberg/pull/12808#discussion_r2067550907


##########
bigquery/src/main/java/org/apache/iceberg/gcp/bigquery/BigQueryMetastoreCatalog.java:
##########
@@ -0,0 +1,405 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.iceberg.gcp.bigquery;
+
+import com.google.api.services.bigquery.model.Dataset;
+import com.google.api.services.bigquery.model.DatasetList.Datasets;
+import com.google.api.services.bigquery.model.DatasetReference;
+import com.google.api.services.bigquery.model.ExternalCatalogDatasetOptions;
+import com.google.api.services.bigquery.model.TableReference;
+import com.google.cloud.ServiceOptions;
+import com.google.cloud.bigquery.BigQueryOptions;
+import java.io.IOException;
+import java.security.GeneralSecurityException;
+import java.util.List;
+import java.util.Locale;
+import java.util.Map;
+import java.util.Objects;
+import java.util.Set;
+import org.apache.hadoop.conf.Configurable;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.iceberg.BaseMetastoreCatalog;
+import org.apache.iceberg.CatalogProperties;
+import org.apache.iceberg.CatalogUtil;
+import org.apache.iceberg.TableMetadata;
+import org.apache.iceberg.TableOperations;
+import org.apache.iceberg.catalog.Namespace;
+import org.apache.iceberg.catalog.SupportsNamespaces;
+import org.apache.iceberg.catalog.TableIdentifier;
+import org.apache.iceberg.exceptions.AlreadyExistsException;
+import org.apache.iceberg.exceptions.NoSuchNamespaceException;
+import org.apache.iceberg.exceptions.NoSuchTableException;
+import org.apache.iceberg.exceptions.ServiceFailureException;
+import org.apache.iceberg.exceptions.ValidationException;
+import org.apache.iceberg.io.FileIO;
+import org.apache.iceberg.relocated.com.google.common.annotations.VisibleForTesting;
+import org.apache.iceberg.relocated.com.google.common.base.Preconditions;
+import org.apache.iceberg.relocated.com.google.common.base.Strings;
+import org.apache.iceberg.relocated.com.google.common.collect.ImmutableList;
+import org.apache.iceberg.relocated.com.google.common.collect.ImmutableMap;
+import org.apache.iceberg.relocated.com.google.common.collect.Maps;
+import org.apache.iceberg.util.LocationUtil;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/** Iceberg Bigquery Metastore Catalog implementation. */
+public class BigQueryMetastoreCatalog extends BaseMetastoreCatalog
+    implements SupportsNamespaces, Configurable {
+
+  // User provided properties.
+  public static final String PROJECT_ID = "gcp-project";
+  public static final String GCP_LOCATION = "gcp-location";
+  public static final String LIST_ALL_TABLES = "list-all-tables";
+
+  public static final String HIVE_METASTORE_WAREHOUSE_DIR = "hive.metastore.warehouse.dir";
+
+  private static final Logger LOG = LoggerFactory.getLogger(BigQueryMetastoreCatalog.class);
+
+  private static final String DEFAULT_GCP_LOCATION = "us";
+
+  private String catalogName;
+  private Map<String, String> catalogProperties;
+  private FileIO fileIO;
+  private Configuration conf;
+  private String projectId;
+  private String location;
+  private BigQueryMetastoreClient client;
+  private boolean listAllTables;
+
+  // Must have a no-arg constructor to be dynamically loaded
+  // initialize(String name, Map<String, String> properties) will be called to complete
+  // initialization
+  public BigQueryMetastoreCatalog() {}
+
+  @Override
+  public void initialize(String inputName, Map<String, String> properties) {
+    Preconditions.checkArgument(
+        properties.containsKey(PROJECT_ID),
+        "Invalid GCP project: %s must be specified",
+        PROJECT_ID);
+
+    projectId = properties.get(PROJECT_ID);
+    location = properties.getOrDefault(GCP_LOCATION, DEFAULT_GCP_LOCATION);
+
+    BigQueryOptions options =
+        BigQueryOptions.newBuilder()
+            .setProjectId(projectId)
+            .setLocation(location)
+            .setRetrySettings(ServiceOptions.getDefaultRetrySettings())
+            .build();
+
+    try {
+      client = new BigQueryMetastoreClientImpl(options);
+    } catch (IOException e) {
+      throw new ServiceFailureException(e, "Creating BigQuery client failed");
+    } catch (GeneralSecurityException e) {
+      throw new ValidationException(e, "Creating BigQuery client failed due to a security issue");
+    }
+
+    initialize(inputName, properties, projectId, location, client);
+  }
+
+  @VisibleForTesting
+  void initialize(
+      String inputName,
+      Map<String, String> properties,
+      String initialProjectId,
+      String initialLocation,
+      BigQueryMetastoreClient bigQueryMetaStoreClient) {
+    Preconditions.checkArgument(bigQueryMetaStoreClient != null, "Invalid BigQuery client: null");
+    this.catalogName = inputName;
+    this.catalogProperties = ImmutableMap.copyOf(properties);
+    this.projectId = initialProjectId;
+    this.location = initialLocation;
+    this.client = bigQueryMetaStoreClient;
+
+    if (this.conf == null) {
+      LOG.warn("No configuration was set, using the default environment Configuration");
+      this.conf = new Configuration();
+    }
+
+    LOG.info("Using BigQuery Metastore Iceberg Catalog: {}", inputName);
+
+    if (properties.containsKey(CatalogProperties.WAREHOUSE_LOCATION)) {
+      this.conf.set(
+          HIVE_METASTORE_WAREHOUSE_DIR,
+          LocationUtil.stripTrailingSlash(properties.get(CatalogProperties.WAREHOUSE_LOCATION)));
+    }
+
+    String fileIoImpl =
+        properties.getOrDefault(
+            CatalogProperties.FILE_IO_IMPL, "org.apache.iceberg.hadoop.HadoopFileIO");
+    this.fileIO = CatalogUtil.loadFileIO(fileIoImpl, properties, conf);
+
+    this.listAllTables = Boolean.parseBoolean(properties.getOrDefault(LIST_ALL_TABLES, "true"));
+  }
+
+  @Override
+  protected TableOperations newTableOps(TableIdentifier identifier) {
+    return new BigQueryTableOperations(client, fileIO, toTableReference(identifier), conf);
+  }
+
+  @Override
+  protected String defaultWarehouseLocation(TableIdentifier identifier) {
+    String locationUri = null;
+    DatasetReference datasetReference = toDatasetReference(identifier.namespace());
+    Dataset dataset = client.load(datasetReference);
+    if (dataset != null && dataset.getExternalCatalogDatasetOptions() != null) {
+      locationUri = dataset.getExternalCatalogDatasetOptions().getDefaultStorageLocationUri();
+    }
+
+    return String.format(
+        "%s/%s",
+        Strings.isNullOrEmpty(locationUri)
+            ? createDefaultStorageLocationUri(datasetReference.getDatasetId())
+            : LocationUtil.stripTrailingSlash(locationUri),
+        identifier.name());
+  }
+
+  @Override
+  public List<TableIdentifier> listTables(Namespace namespace) {
+    validateNamespace(namespace);
+
+    return client.list(toDatasetReference(namespace), listAllTables).stream()
+        .map(
+            table -> TableIdentifier.of(namespace.level(0), table.getTableReference().getTableId()))
+        .collect(ImmutableList.toImmutableList());
+  }
+
+  @Override
+  public boolean dropTable(TableIdentifier identifier, boolean purge) {
+    try {
+      TableOperations ops = newTableOps(identifier);
+      TableMetadata lastMetadata = ops.current();
+
+      client.delete(toTableReference(identifier));
+
+      if (purge && lastMetadata != null) {
+        CatalogUtil.dropTableData(ops.io(), lastMetadata);
+      }
+    } catch (NoSuchTableException e) {
+      return false;
+    }
+
+    return true;
+  }
+
+  @Override
+  public void renameTable(TableIdentifier from, TableIdentifier to) {
+    validateNamespace(from.namespace());
+    validateNamespace(to.namespace());
+
+    if (!namespaceExists(to.namespace())) {
+      throw new NoSuchNamespaceException(
+          "Cannot rename %s to %s. Namespace does not exist: %s",
+          from.name(), to.name(), to.namespace());
+    }
+
+    if (!namespaceExists(from.namespace())) {
+      throw new NoSuchNamespaceException(
+          "Cannot rename %s to %s. Namespace does not exist: %s",
+          from.name(), to.name(), from.namespace());
+    }
+
+    if (!tableExists(from)) {
+      throw new NoSuchTableException(
+          "Cannot rename %s to %s. Table does not exist: %s", from.name(), to.name(), from.name());
+    }
+
+    if (!from.namespace().equals(to.namespace())) {
+      throw new ValidationException(
+          "Cannot rename %s to %s. New table name must be in the same namespace",
+          from.name(), to.name());
+    }
+
+    if (tableExists(to)) {
+      throw new AlreadyExistsException(
+          "Cannot rename %s to %s. Table already exists: %s", from.name(), to.name(), to.name());
+    }
+
+    // TODO: Enable once supported by BigQuery API.
+    throw new UnsupportedOperationException("Table rename operation is unsupported.");

Review Comment:
   I don't we need to include all the validation at this time if rename is unsupported.  Just throw this immediately.



-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org

For queries about this service, please contact Infrastructure at:
users@infra.apache.org


---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For additional commands, e-mail: issues-help@iceberg.apache.org


Re: [PR] Catalog: Add BigQuery Metastore Catalog Support [iceberg]

Posted by "nastra (via GitHub)" <gi...@apache.org>.
nastra commented on code in PR #12808:
URL: https://github.com/apache/iceberg/pull/12808#discussion_r2067912014


##########
bigquery/src/main/java/org/apache/iceberg/gcp/bigquery/BigQueryMetastoreCatalog.java:
##########
@@ -0,0 +1,371 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.iceberg.gcp.bigquery;
+
+import com.google.api.services.bigquery.model.Dataset;
+import com.google.api.services.bigquery.model.DatasetList.Datasets;
+import com.google.api.services.bigquery.model.DatasetReference;
+import com.google.api.services.bigquery.model.ExternalCatalogDatasetOptions;
+import com.google.api.services.bigquery.model.TableReference;
+import com.google.cloud.ServiceOptions;
+import com.google.cloud.bigquery.BigQueryOptions;
+import java.io.IOException;
+import java.security.GeneralSecurityException;
+import java.util.List;
+import java.util.Locale;
+import java.util.Map;
+import java.util.Objects;
+import java.util.Set;
+import org.apache.hadoop.conf.Configurable;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.iceberg.BaseMetastoreCatalog;
+import org.apache.iceberg.CatalogProperties;
+import org.apache.iceberg.CatalogUtil;
+import org.apache.iceberg.TableMetadata;
+import org.apache.iceberg.TableOperations;
+import org.apache.iceberg.catalog.Namespace;
+import org.apache.iceberg.catalog.SupportsNamespaces;
+import org.apache.iceberg.catalog.TableIdentifier;
+import org.apache.iceberg.exceptions.NoSuchNamespaceException;
+import org.apache.iceberg.exceptions.NoSuchTableException;
+import org.apache.iceberg.exceptions.ServiceFailureException;
+import org.apache.iceberg.exceptions.ValidationException;
+import org.apache.iceberg.io.FileIO;
+import org.apache.iceberg.relocated.com.google.common.annotations.VisibleForTesting;
+import org.apache.iceberg.relocated.com.google.common.base.Preconditions;
+import org.apache.iceberg.relocated.com.google.common.base.Strings;
+import org.apache.iceberg.relocated.com.google.common.collect.ImmutableList;
+import org.apache.iceberg.relocated.com.google.common.collect.ImmutableMap;
+import org.apache.iceberg.relocated.com.google.common.collect.Maps;
+import org.apache.iceberg.util.LocationUtil;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/** Iceberg Bigquery Metastore Catalog implementation. */
+public class BigQueryMetastoreCatalog extends BaseMetastoreCatalog
+    implements SupportsNamespaces, Configurable {
+
+  // User provided properties.
+  public static final String PROJECT_ID = "gcp.bigquery.project-id";
+  public static final String GCP_LOCATION = "gcp.bigquery.location";
+  public static final String LIST_ALL_TABLES = "gcp.bigquery.list-all-tables";
+
+  private static final Logger LOG = LoggerFactory.getLogger(BigQueryMetastoreCatalog.class);
+
+  private static final String DEFAULT_GCP_LOCATION = "us";
+
+  private String catalogName;
+  private Map<String, String> catalogProperties;
+  private FileIO fileIO;
+  private Configuration conf;
+  private String projectId;
+  private String projectLocation;
+  private BigQueryMetastoreClient client;
+  private boolean listAllTables;
+  private String warehouseLocation;
+
+  // Must have a no-arg constructor to be dynamically loaded
+  // initialize(String name, Map<String, String> properties) will be called to complete
+  // initialization
+  public BigQueryMetastoreCatalog() {}
+
+  @Override
+  public void initialize(String name, Map<String, String> properties) {
+    Preconditions.checkArgument(
+        properties.containsKey(PROJECT_ID),
+        "Invalid GCP project: %s must be specified",
+        PROJECT_ID);
+
+    this.projectId = properties.get(PROJECT_ID);
+    this.projectLocation = properties.getOrDefault(GCP_LOCATION, DEFAULT_GCP_LOCATION);
+
+    BigQueryOptions options =
+        BigQueryOptions.newBuilder()
+            .setProjectId(projectId)
+            .setLocation(projectLocation)
+            .setRetrySettings(ServiceOptions.getDefaultRetrySettings())
+            .build();
+
+    try {
+      client = new BigQueryMetastoreClientImpl(options);
+    } catch (IOException e) {
+      throw new ServiceFailureException(e, "Creating BigQuery client failed");
+    } catch (GeneralSecurityException e) {
+      throw new ValidationException(e, "Creating BigQuery client failed due to a security issue");
+    }
+
+    initialize(name, properties, projectId, projectLocation, client);
+  }
+
+  @VisibleForTesting
+  void initialize(
+      String inputName,
+      Map<String, String> properties,
+      String initialProjectId,
+      String initialLocation,
+      BigQueryMetastoreClient bigQueryMetaStoreClient) {
+    Preconditions.checkArgument(bigQueryMetaStoreClient != null, "Invalid BigQuery client: null");
+    this.catalogName = inputName;
+    this.catalogProperties = ImmutableMap.copyOf(properties);
+    this.projectId = initialProjectId;
+    this.projectLocation = initialLocation;
+    this.client = bigQueryMetaStoreClient;
+
+    if (this.conf == null) {
+      LOG.warn("No configuration was set, using the default environment Configuration");
+      this.conf = new Configuration();
+    }
+
+    LOG.info("Using BigQuery Metastore Iceberg Catalog: {}", inputName);
+
+    if (properties.containsKey(CatalogProperties.WAREHOUSE_LOCATION)) {
+      this.warehouseLocation =
+          LocationUtil.stripTrailingSlash(properties.get(CatalogProperties.WAREHOUSE_LOCATION));
+    }
+
+    this.fileIO =
+        CatalogUtil.loadFileIO(
+            properties.getOrDefault(
+                CatalogProperties.FILE_IO_IMPL, "org.apache.iceberg.io.ResolvingFileIO"),
+            properties,
+            conf);
+
+    this.listAllTables = Boolean.parseBoolean(properties.getOrDefault(LIST_ALL_TABLES, "true"));
+  }
+
+  @Override
+  protected TableOperations newTableOps(TableIdentifier identifier) {
+    return new BigQueryTableOperations(client, fileIO, toTableReference(identifier), conf);
+  }
+
+  @Override
+  protected String defaultWarehouseLocation(TableIdentifier identifier) {
+    String locationUri = null;
+    DatasetReference datasetReference = toDatasetReference(identifier.namespace());
+    Dataset dataset = client.load(datasetReference);
+    if (dataset != null && dataset.getExternalCatalogDatasetOptions() != null) {
+      locationUri = dataset.getExternalCatalogDatasetOptions().getDefaultStorageLocationUri();
+    }
+
+    return String.format(
+        "%s/%s",
+        Strings.isNullOrEmpty(locationUri)
+            ? createDefaultStorageLocationUri(datasetReference.getDatasetId())
+            : LocationUtil.stripTrailingSlash(locationUri),
+        identifier.name());
+  }
+
+  @Override
+  public List<TableIdentifier> listTables(Namespace namespace) {
+    validateNamespace(namespace);
+
+    return client.list(toDatasetReference(namespace), listAllTables).stream()
+        .map(
+            table -> TableIdentifier.of(namespace.level(0), table.getTableReference().getTableId()))
+        .collect(ImmutableList.toImmutableList());
+  }
+
+  @Override
+  public boolean dropTable(TableIdentifier identifier, boolean purge) {
+    try {
+      TableOperations ops = newTableOps(identifier);
+      TableMetadata lastMetadata = ops.current();
+
+      client.delete(toTableReference(identifier));
+
+      if (purge && lastMetadata != null) {
+        CatalogUtil.dropTableData(ops.io(), lastMetadata);
+      }
+    } catch (NoSuchTableException e) {
+      return false;
+    }
+
+    return true;
+  }
+
+  @Override
+  public void renameTable(TableIdentifier from, TableIdentifier to) {
+    // TODO: Enable once supported by BigQuery API.
+    throw new UnsupportedOperationException("Table rename operation is unsupported.");
+  }
+
+  @Override
+  public void createNamespace(Namespace namespace, Map<String, String> metadata) {
+    Dataset builder = new Dataset();
+    DatasetReference datasetReference = toDatasetReference(namespace);
+    builder.setLocation(this.projectLocation);
+    builder.setDatasetReference(datasetReference);
+    builder.setExternalCatalogDatasetOptions(
+        BigQueryMetastoreUtils.createExternalCatalogDatasetOptions(
+            createDefaultStorageLocationUri(datasetReference.getDatasetId()), metadata));
+
+    client.create(builder);
+  }
+
+  @Override
+  public List<Namespace> listNamespaces() {
+    try {
+      return listNamespaces(Namespace.empty());
+    } catch (NoSuchNamespaceException e) {
+      return ImmutableList.of();
+    }
+  }
+
+  /**
+   * Since this catalog only supports one-level namespaces, it always returns an empty list unless
+   * passed an empty namespace to list all namespaces within the catalog.
+   */
+  @Override
+  public List<Namespace> listNamespaces(Namespace namespace) {
+    List<Datasets> allDatasets = client.list(projectId);
+
+    ImmutableList<Namespace> namespaces =
+        allDatasets.stream().map(this::toNamespace).collect(ImmutableList.toImmutableList());
+    if (namespaces.isEmpty()) {
+      throw new NoSuchNamespaceException("Namespace does not exist: %s", namespace);
+    }
+
+    return namespaces;
+  }
+
+  @Override
+  public boolean dropNamespace(Namespace namespace) {
+    try {
+      client.delete(toDatasetReference(namespace));
+      // We don't delete the data folder for safety, which aligns with Hive Metastore's default
+      // behavior.
+      // We can support database or catalog level config controlling file deletion in the future.
+      return true;
+    } catch (NoSuchNamespaceException e) {
+      return false;
+    }
+  }
+
+  @Override
+  public boolean setProperties(Namespace namespace, Map<String, String> properties) {
+    Dataset dataset = client.load(toDatasetReference(namespace));
+
+    ExternalCatalogDatasetOptions existingOptions = dataset.getExternalCatalogDatasetOptions();
+    Map<String, String> existingParameters =
+        existingOptions != null ? existingOptions.getParameters() : null;
+
+    Map<String, String> newParameters = Maps.newHashMap();
+    if (existingParameters != null) {
+      newParameters.putAll(existingParameters);
+    }
+
+    newParameters.putAll(properties);
+
+    if (Objects.equals(existingParameters, newParameters)) {
+      // No change in parameters detected
+      return false;
+    }
+
+    client.setParameters(toDatasetReference(namespace), properties);
+    return true;
+  }
+
+  @Override
+  public boolean removeProperties(Namespace namespace, Set<String> properties) {
+    client.removeParameters(toDatasetReference(namespace), properties);
+    return true;
+  }
+
+  @SuppressWarnings("FormatStringAnnotation")

Review Comment:
   can be removed and the exception throwing can be updated to `NoSuchNamespaceException("%s", e.getMessage())`



-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org

For queries about this service, please contact Infrastructure at:
users@infra.apache.org


---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For additional commands, e-mail: issues-help@iceberg.apache.org


Re: [PR] Catalog: Add BigQuery Metastore Catalog Support [iceberg]

Posted by "talatuyarer (via GitHub)" <gi...@apache.org>.
talatuyarer commented on code in PR #12808:
URL: https://github.com/apache/iceberg/pull/12808#discussion_r2070954082


##########
bigquery/src/main/java/org/apache/iceberg/gcp/bigquery/BigQueryMetastoreCatalog.java:
##########
@@ -0,0 +1,370 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.iceberg.gcp.bigquery;
+
+import com.google.api.services.bigquery.model.Dataset;
+import com.google.api.services.bigquery.model.DatasetList.Datasets;
+import com.google.api.services.bigquery.model.DatasetReference;
+import com.google.api.services.bigquery.model.ExternalCatalogDatasetOptions;
+import com.google.api.services.bigquery.model.TableReference;
+import com.google.cloud.ServiceOptions;
+import com.google.cloud.bigquery.BigQueryOptions;
+import java.io.IOException;
+import java.security.GeneralSecurityException;
+import java.util.List;
+import java.util.Locale;
+import java.util.Map;
+import java.util.Objects;
+import java.util.Set;
+import org.apache.hadoop.conf.Configurable;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.iceberg.BaseMetastoreCatalog;
+import org.apache.iceberg.CatalogProperties;
+import org.apache.iceberg.CatalogUtil;
+import org.apache.iceberg.TableMetadata;
+import org.apache.iceberg.TableOperations;
+import org.apache.iceberg.catalog.Namespace;
+import org.apache.iceberg.catalog.SupportsNamespaces;
+import org.apache.iceberg.catalog.TableIdentifier;
+import org.apache.iceberg.exceptions.NoSuchNamespaceException;
+import org.apache.iceberg.exceptions.NoSuchTableException;
+import org.apache.iceberg.exceptions.ServiceFailureException;
+import org.apache.iceberg.exceptions.ValidationException;
+import org.apache.iceberg.io.FileIO;
+import org.apache.iceberg.relocated.com.google.common.annotations.VisibleForTesting;
+import org.apache.iceberg.relocated.com.google.common.base.Preconditions;
+import org.apache.iceberg.relocated.com.google.common.base.Strings;
+import org.apache.iceberg.relocated.com.google.common.collect.ImmutableList;
+import org.apache.iceberg.relocated.com.google.common.collect.ImmutableMap;
+import org.apache.iceberg.relocated.com.google.common.collect.Maps;
+import org.apache.iceberg.util.LocationUtil;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/** Iceberg Bigquery Metastore Catalog implementation. */
+public class BigQueryMetastoreCatalog extends BaseMetastoreCatalog
+    implements SupportsNamespaces, Configurable {
+
+  // User provided properties.
+  public static final String PROJECT_ID = "gcp.bigquery.project-id";
+  public static final String GCP_LOCATION = "gcp.bigquery.location";
+  public static final String LIST_ALL_TABLES = "gcp.bigquery.list-all-tables";
+
+  private static final Logger LOG = LoggerFactory.getLogger(BigQueryMetastoreCatalog.class);
+
+  private static final String DEFAULT_GCP_LOCATION = "us";
+
+  private String catalogName;
+  private Map<String, String> catalogProperties;
+  private FileIO fileIO;
+  private Configuration conf;
+  private String projectId;
+  private String projectLocation;
+  private BigQueryMetastoreClient client;
+  private boolean listAllTables;
+  private String warehouseLocation;
+
+  // Must have a no-arg constructor to be dynamically loaded
+  // initialize(String name, Map<String, String> properties) will be called to complete
+  // initialization

Review Comment:
   I removed these comments. They can not be Javadoc.



-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org

For queries about this service, please contact Infrastructure at:
users@infra.apache.org


---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For additional commands, e-mail: issues-help@iceberg.apache.org


Re: [PR] Catalog: Add BigQuery Metastore Catalog Support [iceberg]

Posted by "danielcweeks (via GitHub)" <gi...@apache.org>.
danielcweeks commented on code in PR #12808:
URL: https://github.com/apache/iceberg/pull/12808#discussion_r2072205218


##########
bigquery/src/main/java/org/apache/iceberg/gcp/bigquery/BigQueryTableOperations.java:
##########
@@ -0,0 +1,296 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.iceberg.gcp.bigquery;
+
+import com.google.api.services.bigquery.model.ExternalCatalogTableOptions;
+import com.google.api.services.bigquery.model.Table;
+import com.google.api.services.bigquery.model.TableReference;
+import java.util.Locale;
+import java.util.Map;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hive.common.StatsSetupConst;
+import org.apache.iceberg.BaseMetastoreOperations;
+import org.apache.iceberg.BaseMetastoreTableOperations;
+import org.apache.iceberg.SnapshotSummary;
+import org.apache.iceberg.TableMetadata;
+import org.apache.iceberg.TableProperties;
+import org.apache.iceberg.exceptions.AlreadyExistsException;
+import org.apache.iceberg.exceptions.CommitFailedException;
+import org.apache.iceberg.exceptions.CommitStateUnknownException;
+import org.apache.iceberg.exceptions.NoSuchTableException;
+import org.apache.iceberg.exceptions.ValidationException;
+import org.apache.iceberg.hadoop.ConfigProperties;
+import org.apache.iceberg.io.FileIO;
+import org.apache.iceberg.relocated.com.google.common.collect.Maps;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/** Handles BigQuery metastore table operations. */
+public final class BigQueryTableOperations extends BaseMetastoreTableOperations {
+
+  private static final Logger LOG = LoggerFactory.getLogger(BigQueryTableOperations.class);
+
+  private static final String TABLE_PROPERTIES_BQ_CONNECTION = "bq_connection";
+
+  private final BigQueryMetastoreClient client;
+  private final FileIO fileIO;
+  private final TableReference tableReference;
+  private final Configuration conf;
+
+  BigQueryTableOperations(
+      BigQueryMetastoreClient client,
+      FileIO fileIO,
+      TableReference tableReference,
+      Configuration conf) {
+    this.client = client;
+    this.fileIO = fileIO;
+    this.tableReference = tableReference;
+    this.conf = conf;
+  }
+
+  // The doRefresh method should provide implementation on how to get the metadata location.
+  @Override
+  public void doRefresh() {
+    // Must default to null.
+    String metadataLocation = null;
+    try {
+      metadataLocation =
+          loadMetadataLocationOrThrow(
+              client.load(this.tableReference).getExternalCatalogTableOptions());
+    } catch (NoSuchTableException e) {
+      if (currentMetadataLocation() != null) {
+        // Re-throws the exception because the table must exist in this case.
+        throw e;
+      }
+    }
+    refreshFromMetadataLocation(metadataLocation);
+  }
+
+  // The doCommit method should provide implementation on how to update with metadata location
+  // atomically
+  @Override
+  public void doCommit(TableMetadata base, TableMetadata metadata) {
+    String newMetadataLocation =
+        base == null && metadata.metadataFileLocation() != null
+            ? metadata.metadataFileLocation()
+            : writeNewMetadata(metadata, currentVersion() + 1);
+    BaseMetastoreOperations.CommitStatus commitStatus =
+        BaseMetastoreOperations.CommitStatus.FAILURE;
+    try {
+      if (base == null) {
+        createTable(newMetadataLocation, metadata);
+      } else {
+        updateTable(base.metadataFileLocation(), newMetadataLocation, metadata);
+      }
+      commitStatus = BaseMetastoreOperations.CommitStatus.SUCCESS;
+    } catch (CommitFailedException | CommitStateUnknownException e) {
+      throw e;
+    } catch (Throwable e) {
+      LOG.error("Exception thrown on commit: ", e);
+      if (e instanceof AlreadyExistsException) {
+        throw e;
+      }
+      commitStatus =
+          BaseMetastoreOperations.CommitStatus.valueOf(
+              checkCommitStatus(newMetadataLocation, metadata).name());
+      if (commitStatus == BaseMetastoreOperations.CommitStatus.FAILURE) {
+        throw new CommitFailedException(e, "Failed to commit");
+      }
+      if (commitStatus == BaseMetastoreOperations.CommitStatus.UNKNOWN) {
+        throw new CommitStateUnknownException(e);
+      }
+    } finally {
+      try {
+        if (commitStatus == BaseMetastoreOperations.CommitStatus.FAILURE) {
+          LOG.warn("Failed to commit updates to table {}", tableName());
+          io().deleteFile(newMetadataLocation);
+        }
+      } catch (RuntimeException e) {
+        LOG.error(
+            "Failed to cleanup metadata file at {} for table {}",
+            newMetadataLocation,
+            tableName(),
+            e);
+      }
+    }
+  }
+
+  @Override
+  public String tableName() {
+    return String.format("%s.%s", tableReference.getDatasetId(), tableReference.getTableId());
+  }
+
+  @Override
+  public FileIO io() {
+    return fileIO;
+  }
+
+  private void createTable(String newMetadataLocation, TableMetadata metadata) {
+    LOG.debug("Creating a new Iceberg table: {}", tableName());
+    Table tableBuilder = makeNewTable(metadata, newMetadataLocation);
+    tableBuilder.setTableReference(this.tableReference);
+    addConnectionIfProvided(tableBuilder, metadata.properties());
+
+    client.create(tableBuilder);

Review Comment:
   Do we have any checks to ensure the namespace exists or that the proper exception will be thrown?



-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org

For queries about this service, please contact Infrastructure at:
users@infra.apache.org


---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For additional commands, e-mail: issues-help@iceberg.apache.org


Re: [PR] Catalog: Add BigQuery Metastore Catalog Support [iceberg]

Posted by "talatuyarer (via GitHub)" <gi...@apache.org>.
talatuyarer commented on code in PR #12808:
URL: https://github.com/apache/iceberg/pull/12808#discussion_r2073786108


##########
bigquery/src/main/java/org/apache/iceberg/gcp/bigquery/BigQueryTableOperations.java:
##########
@@ -0,0 +1,296 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.iceberg.gcp.bigquery;
+
+import com.google.api.services.bigquery.model.ExternalCatalogTableOptions;
+import com.google.api.services.bigquery.model.Table;
+import com.google.api.services.bigquery.model.TableReference;
+import java.util.Locale;
+import java.util.Map;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hive.common.StatsSetupConst;
+import org.apache.iceberg.BaseMetastoreOperations;
+import org.apache.iceberg.BaseMetastoreTableOperations;
+import org.apache.iceberg.SnapshotSummary;
+import org.apache.iceberg.TableMetadata;
+import org.apache.iceberg.TableProperties;
+import org.apache.iceberg.exceptions.AlreadyExistsException;
+import org.apache.iceberg.exceptions.CommitFailedException;
+import org.apache.iceberg.exceptions.CommitStateUnknownException;
+import org.apache.iceberg.exceptions.NoSuchTableException;
+import org.apache.iceberg.exceptions.ValidationException;
+import org.apache.iceberg.hadoop.ConfigProperties;
+import org.apache.iceberg.io.FileIO;
+import org.apache.iceberg.relocated.com.google.common.collect.Maps;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/** Handles BigQuery metastore table operations. */
+public final class BigQueryTableOperations extends BaseMetastoreTableOperations {
+
+  private static final Logger LOG = LoggerFactory.getLogger(BigQueryTableOperations.class);
+
+  private static final String TABLE_PROPERTIES_BQ_CONNECTION = "bq_connection";
+
+  private final BigQueryMetastoreClient client;
+  private final FileIO fileIO;
+  private final TableReference tableReference;
+  private final Configuration conf;
+
+  BigQueryTableOperations(
+      BigQueryMetastoreClient client,
+      FileIO fileIO,
+      TableReference tableReference,
+      Configuration conf) {
+    this.client = client;
+    this.fileIO = fileIO;
+    this.tableReference = tableReference;
+    this.conf = conf;
+  }
+
+  // The doRefresh method should provide implementation on how to get the metadata location.
+  @Override
+  public void doRefresh() {
+    // Must default to null.
+    String metadataLocation = null;
+    try {
+      metadataLocation =
+          loadMetadataLocationOrThrow(
+              client.load(this.tableReference).getExternalCatalogTableOptions());
+    } catch (NoSuchTableException e) {
+      if (currentMetadataLocation() != null) {
+        // Re-throws the exception because the table must exist in this case.
+        throw e;
+      }
+    }
+    refreshFromMetadataLocation(metadataLocation);
+  }
+
+  // The doCommit method should provide implementation on how to update with metadata location
+  // atomically
+  @Override
+  public void doCommit(TableMetadata base, TableMetadata metadata) {
+    String newMetadataLocation =
+        base == null && metadata.metadataFileLocation() != null
+            ? metadata.metadataFileLocation()
+            : writeNewMetadata(metadata, currentVersion() + 1);
+    BaseMetastoreOperations.CommitStatus commitStatus =
+        BaseMetastoreOperations.CommitStatus.FAILURE;
+    try {
+      if (base == null) {
+        createTable(newMetadataLocation, metadata);
+      } else {
+        updateTable(base.metadataFileLocation(), newMetadataLocation, metadata);
+      }
+      commitStatus = BaseMetastoreOperations.CommitStatus.SUCCESS;
+    } catch (CommitFailedException | CommitStateUnknownException e) {
+      throw e;
+    } catch (Throwable e) {
+      LOG.error("Exception thrown on commit: ", e);
+      if (e instanceof AlreadyExistsException) {
+        throw e;
+      }
+      commitStatus =
+          BaseMetastoreOperations.CommitStatus.valueOf(
+              checkCommitStatus(newMetadataLocation, metadata).name());
+      if (commitStatus == BaseMetastoreOperations.CommitStatus.FAILURE) {
+        throw new CommitFailedException(e, "Failed to commit");
+      }
+      if (commitStatus == BaseMetastoreOperations.CommitStatus.UNKNOWN) {
+        throw new CommitStateUnknownException(e);
+      }
+    } finally {
+      try {
+        if (commitStatus == BaseMetastoreOperations.CommitStatus.FAILURE) {
+          LOG.warn("Failed to commit updates to table {}", tableName());
+          io().deleteFile(newMetadataLocation);
+        }
+      } catch (RuntimeException e) {
+        LOG.error(
+            "Failed to cleanup metadata file at {} for table {}",
+            newMetadataLocation,
+            tableName(),
+            e);
+      }
+    }
+  }
+
+  @Override
+  public String tableName() {
+    return String.format("%s.%s", tableReference.getDatasetId(), tableReference.getTableId());
+  }
+
+  @Override
+  public FileIO io() {
+    return fileIO;
+  }
+
+  private void createTable(String newMetadataLocation, TableMetadata metadata) {
+    LOG.debug("Creating a new Iceberg table: {}", tableName());
+    Table tableBuilder = makeNewTable(metadata, newMetadataLocation);
+    tableBuilder.setTableReference(this.tableReference);
+    addConnectionIfProvided(tableBuilder, metadata.properties());
+
+    client.create(tableBuilder);
+  }
+
+  private void addConnectionIfProvided(Table tableBuilder, Map<String, String> metadataProperties) {
+    if (metadataProperties.containsKey(TABLE_PROPERTIES_BQ_CONNECTION)) {
+      tableBuilder
+          .getExternalCatalogTableOptions()
+          .setConnectionId(metadataProperties.get(TABLE_PROPERTIES_BQ_CONNECTION));
+    }
+  }
+
+  /** Update table properties with concurrent update detection using etag. */
+  private void updateTable(
+      String oldMetadataLocation, String newMetadataLocation, TableMetadata metadata) {
+    Table table = client.load(this.tableReference);
+    if (table.getEtag().isEmpty()) {
+      throw new ValidationException(
+          "Etag of legacy table %s is empty, manually update the table via the BigQuery API or"
+              + " recreate and retry",
+          tableName());
+    }
+    ExternalCatalogTableOptions options = table.getExternalCatalogTableOptions();
+    addConnectionIfProvided(table, metadata.properties());
+
+    // If `metadataLocationFromMetastore` is different from metadata location of base, it means
+    // someone has updated metadata location in metastore, which is a conflict update.
+    String metadataLocationFromMetastore =
+        options.getParameters().getOrDefault(METADATA_LOCATION_PROP, "");
+    if (!metadataLocationFromMetastore.isEmpty()
+        && !metadataLocationFromMetastore.equals(oldMetadataLocation)) {
+      throw new CommitFailedException(
+          "Cannot commit base metadata location '%s' is not same as the current table metadata location '%s' for"
+              + " %s.%s",
+          oldMetadataLocation,
+          metadataLocationFromMetastore,
+          tableReference.getDatasetId(),
+          tableReference.getTableId());
+    }
+
+    options.setParameters(buildTableParameters(newMetadataLocation, metadata));
+    try {
+      client.update(tableReference, table);
+    } catch (ValidationException e) {
+      if (e.getMessage().toLowerCase(Locale.ENGLISH).contains("etag mismatch")) {
+        throw new CommitFailedException(
+            "Updating table failed due to conflict updates (etag mismatch). Retry the update");
+      }
+
+      throw e;
+    }
+  }
+
+  // To make the table queryable from Hive, the user would likely be setting the HIVE_ENGINE_ENABLED
+  // parameter.
+  //
+  // TODO: We need to make a decision on how to make the table queryable from Hive.
+  // (could be a server side change or a client side change - that's TBD).
+  private Table makeNewTable(TableMetadata metadata, String metadataFileLocation) {
+    return new Table()
+        .setExternalCatalogTableOptions(
+            BigQueryMetastoreUtils.createExternalCatalogTableOptions(
+                metadata.location(),
+                buildTableParameters(metadataFileLocation, metadata),
+                hiveEngineEnabled(metadata)));

Review Comment:
   We plan to support https://iceberg.apache.org/docs/latest/hive/#create-external-table-overlaying-an-existing-iceberg-table for our users. BQMS is not just BigQuery also become a catalog implementation for other engines. 



-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org

For queries about this service, please contact Infrastructure at:
users@infra.apache.org


---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For additional commands, e-mail: issues-help@iceberg.apache.org


Re: [PR] Catalog: Add BigQuery Metastore Catalog Support [iceberg]

Posted by "ebyhr (via GitHub)" <gi...@apache.org>.
ebyhr commented on code in PR #12808:
URL: https://github.com/apache/iceberg/pull/12808#discussion_r2074868327


##########
bigquery/src/main/java/org/apache/iceberg/gcp/bigquery/BigQueryMetastoreClientImpl.java:
##########
@@ -0,0 +1,579 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.iceberg.gcp.bigquery;
+
+import com.google.api.client.googleapis.javanet.GoogleNetHttpTransport;
+import com.google.api.client.http.HttpHeaders;
+import com.google.api.client.http.HttpResponse;
+import com.google.api.client.http.HttpResponseException;
+import com.google.api.client.http.HttpStatusCodes;
+import com.google.api.client.json.gson.GsonFactory;
+import com.google.api.client.util.Data;
+import com.google.api.services.bigquery.Bigquery;
+import com.google.api.services.bigquery.BigqueryScopes;
+import com.google.api.services.bigquery.model.Dataset;
+import com.google.api.services.bigquery.model.DatasetList;
+import com.google.api.services.bigquery.model.DatasetList.Datasets;
+import com.google.api.services.bigquery.model.DatasetReference;
+import com.google.api.services.bigquery.model.ExternalCatalogDatasetOptions;
+import com.google.api.services.bigquery.model.ExternalCatalogTableOptions;
+import com.google.api.services.bigquery.model.Table;
+import com.google.api.services.bigquery.model.TableList;
+import com.google.api.services.bigquery.model.TableList.Tables;
+import com.google.api.services.bigquery.model.TableReference;
+import com.google.api.services.bigquery.model.TableSchema;
+import com.google.auth.http.HttpCredentialsAdapter;
+import com.google.auth.oauth2.GoogleCredentials;
+import com.google.cloud.BaseServiceException;
+import com.google.cloud.ExceptionHandler;
+import com.google.cloud.bigquery.BigQueryErrorMessages;
+import com.google.cloud.bigquery.BigQueryOptions;
+import com.google.cloud.bigquery.BigQueryRetryConfig;
+import com.google.cloud.bigquery.BigQueryRetryHelper;
+import java.io.IOException;
+import java.nio.charset.StandardCharsets;
+import java.security.GeneralSecurityException;
+import java.util.List;
+import java.util.Locale;
+import java.util.Map;
+import java.util.Set;
+import java.util.stream.Collectors;
+import java.util.stream.Stream;
+import org.apache.iceberg.BaseMetastoreTableOperations;
+import org.apache.iceberg.exceptions.AlreadyExistsException;
+import org.apache.iceberg.exceptions.BadRequestException;
+import org.apache.iceberg.exceptions.ForbiddenException;
+import org.apache.iceberg.exceptions.NoSuchIcebergTableException;
+import org.apache.iceberg.exceptions.NoSuchNamespaceException;
+import org.apache.iceberg.exceptions.NoSuchTableException;
+import org.apache.iceberg.exceptions.NotAuthorizedException;
+import org.apache.iceberg.exceptions.RuntimeIOException;
+import org.apache.iceberg.exceptions.ServiceFailureException;
+import org.apache.iceberg.exceptions.ServiceUnavailableException;
+import org.apache.iceberg.exceptions.ValidationException;
+import org.apache.iceberg.relocated.com.google.common.base.Preconditions;
+import org.apache.iceberg.relocated.com.google.common.collect.Lists;
+import org.apache.iceberg.relocated.com.google.common.collect.Maps;
+
+/** A client of Google Bigquery Metastore functions over the BigQuery service. */
+public final class BigQueryMetastoreClientImpl implements BigQueryMetastoreClient {
+
+  private final Bigquery client;
+  private final BigQueryOptions bigqueryOptions;
+
+  public static final ExceptionHandler.Interceptor EXCEPTION_HANDLER_INTERCEPTOR =
+      new ExceptionHandler.Interceptor() {
+
+        @Override
+        public RetryResult afterEval(Exception exception, RetryResult retryResult) {
+          return ExceptionHandler.Interceptor.RetryResult.CONTINUE_EVALUATION;
+        }
+
+        @Override
+        public RetryResult beforeEval(Exception exception) {
+          if (exception instanceof BaseServiceException) {
+            boolean retriable = ((BaseServiceException) exception).isRetryable();
+            return retriable
+                ? ExceptionHandler.Interceptor.RetryResult.RETRY
+                : ExceptionHandler.Interceptor.RetryResult.CONTINUE_EVALUATION;
+          }
+
+          return ExceptionHandler.Interceptor.RetryResult.CONTINUE_EVALUATION;
+        }
+      };
+
+  // Retry config with error messages and regex for rate limit exceeded errors.
+  private static final BigQueryRetryConfig DEFAULT_RETRY_CONFIG =
+      BigQueryRetryConfig.newBuilder()
+          .retryOnMessage(BigQueryErrorMessages.RATE_LIMIT_EXCEEDED_MSG)
+          .retryOnMessage(BigQueryErrorMessages.JOB_RATE_LIMIT_EXCEEDED_MSG)
+          .retryOnRegEx(BigQueryErrorMessages.RetryRegExPatterns.RATE_LIMIT_EXCEEDED_REGEX)
+          .build();
+
+  public static final ExceptionHandler BIGQUERY_EXCEPTION_HANDLER =
+      ExceptionHandler.newBuilder()
+          .abortOn(RuntimeException.class)
+          // Retry on connection failures due to transient network issues.
+          .retryOn(java.net.ConnectException.class)
+          // Retry to recover from temporary DNS resolution failures.
+          .retryOn(java.net.UnknownHostException.class)
+          .addInterceptors(EXCEPTION_HANDLER_INTERCEPTOR)
+          .build();
+
+  /** Constructs a client of the Google BigQuery service. */
+  public BigQueryMetastoreClientImpl(BigQueryOptions options)
+      throws IOException, GeneralSecurityException {
+    // Initialize client that will be used to send requests. This client only needs to be created
+    // once, and can be reused for multiple requests
+    HttpCredentialsAdapter httpCredentialsAdapter =
+        new HttpCredentialsAdapter(
+            GoogleCredentials.getApplicationDefault().createScoped(BigqueryScopes.all()));

Review Comment:
   This credential relies on environment variables by default, right? How can we connect to multiple BigQuery projects  from a single machine? Why not add a dedicated property with `GoogleCredentials#fromStream` or something?  



-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org

For queries about this service, please contact Infrastructure at:
users@infra.apache.org


---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For additional commands, e-mail: issues-help@iceberg.apache.org


Re: [PR] Catalog: Add BigQuery Metastore Catalog Support [iceberg]

Posted by "nastra (via GitHub)" <gi...@apache.org>.
nastra commented on code in PR #12808:
URL: https://github.com/apache/iceberg/pull/12808#discussion_r2066011789


##########
bigquery/src/main/java/org/apache/iceberg/gcp/bigquery/BigQueryMetastoreClientImpl.java:
##########
@@ -0,0 +1,594 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.iceberg.gcp.bigquery;
+
+import com.google.api.client.googleapis.javanet.GoogleNetHttpTransport;
+import com.google.api.client.http.HttpHeaders;
+import com.google.api.client.http.HttpResponse;
+import com.google.api.client.http.HttpResponseException;
+import com.google.api.client.http.HttpStatusCodes;
+import com.google.api.client.json.gson.GsonFactory;
+import com.google.api.client.util.Data;
+import com.google.api.services.bigquery.Bigquery;
+import com.google.api.services.bigquery.BigqueryScopes;
+import com.google.api.services.bigquery.model.Dataset;
+import com.google.api.services.bigquery.model.DatasetList;
+import com.google.api.services.bigquery.model.DatasetList.Datasets;
+import com.google.api.services.bigquery.model.DatasetReference;
+import com.google.api.services.bigquery.model.ExternalCatalogDatasetOptions;
+import com.google.api.services.bigquery.model.ExternalCatalogTableOptions;
+import com.google.api.services.bigquery.model.Table;
+import com.google.api.services.bigquery.model.TableList;
+import com.google.api.services.bigquery.model.TableList.Tables;
+import com.google.api.services.bigquery.model.TableReference;
+import com.google.api.services.bigquery.model.TableSchema;
+import com.google.auth.http.HttpCredentialsAdapter;
+import com.google.auth.oauth2.GoogleCredentials;
+import com.google.cloud.BaseServiceException;
+import com.google.cloud.ExceptionHandler;
+import com.google.cloud.bigquery.BigQueryErrorMessages;
+import com.google.cloud.bigquery.BigQueryOptions;
+import com.google.cloud.bigquery.BigQueryRetryConfig;
+import com.google.cloud.bigquery.BigQueryRetryHelper;
+import java.io.IOException;
+import java.nio.charset.StandardCharsets;
+import java.security.GeneralSecurityException;
+import java.util.List;
+import java.util.Locale;
+import java.util.Map;
+import java.util.Set;
+import java.util.stream.Collectors;
+import java.util.stream.Stream;
+import org.apache.iceberg.BaseMetastoreTableOperations;
+import org.apache.iceberg.exceptions.AlreadyExistsException;
+import org.apache.iceberg.exceptions.BadRequestException;
+import org.apache.iceberg.exceptions.ForbiddenException;
+import org.apache.iceberg.exceptions.NoSuchIcebergTableException;
+import org.apache.iceberg.exceptions.NoSuchNamespaceException;
+import org.apache.iceberg.exceptions.NoSuchTableException;
+import org.apache.iceberg.exceptions.NotAuthorizedException;
+import org.apache.iceberg.exceptions.RuntimeIOException;
+import org.apache.iceberg.exceptions.ServiceFailureException;
+import org.apache.iceberg.exceptions.ServiceUnavailableException;
+import org.apache.iceberg.exceptions.ValidationException;
+import org.apache.iceberg.relocated.com.google.common.base.Preconditions;
+import org.apache.iceberg.relocated.com.google.common.collect.Lists;
+import org.apache.iceberg.relocated.com.google.common.collect.Maps;
+
+/** A client of Google Bigquery Metastore functions over the BigQuery service. */
+public final class BigQueryMetastoreClientImpl implements BigQueryMetastoreClient {
+
+  public static final ExceptionHandler.Interceptor EXCEPTION_HANDLER_INTERCEPTOR =
+      new ExceptionHandler.Interceptor() {
+
+        private static final long serialVersionUID = -8429573486870467828L;
+
+        @Override
+        public RetryResult afterEval(Exception exception, RetryResult retryResult) {
+          return ExceptionHandler.Interceptor.RetryResult.CONTINUE_EVALUATION;
+        }
+
+        @Override
+        public RetryResult beforeEval(Exception exception) {
+          if (exception instanceof BaseServiceException) {
+            boolean retriable = ((BaseServiceException) exception).isRetryable();
+            return retriable
+                ? ExceptionHandler.Interceptor.RetryResult.RETRY
+                : ExceptionHandler.Interceptor.RetryResult.CONTINUE_EVALUATION;
+          }
+
+          return ExceptionHandler.Interceptor.RetryResult.CONTINUE_EVALUATION;
+        }
+      };
+
+  private final Bigquery client;
+  private final BigQueryOptions bigqueryOptions;
+
+  // Retry config with error messages and regex for rate limit exceeded errors.
+  private static final BigQueryRetryConfig DEFAULT_RETRY_CONFIG =
+      BigQueryRetryConfig.newBuilder()
+          .retryOnMessage(BigQueryErrorMessages.RATE_LIMIT_EXCEEDED_MSG)
+          .retryOnMessage(BigQueryErrorMessages.JOB_RATE_LIMIT_EXCEEDED_MSG)
+          .retryOnRegEx(BigQueryErrorMessages.RetryRegExPatterns.RATE_LIMIT_EXCEEDED_REGEX)
+          .build();
+
+  public static final ExceptionHandler BIGQUERY_EXCEPTION_HANDLER =
+      ExceptionHandler.newBuilder()
+          .abortOn(RuntimeException.class)
+          .retryOn(
+              java.net.ConnectException
+                  .class) // Retry on connection failures due to transient network issues.
+          .retryOn(
+              java.net.UnknownHostException
+                  .class) // Retry to recover from temporary DNS resolution failures.
+          .addInterceptors(EXCEPTION_HANDLER_INTERCEPTOR)
+          .build();
+
+  /** Constructs a client of the Google BigQuery service. */
+  public BigQueryMetastoreClientImpl(BigQueryOptions options)
+      throws IOException, GeneralSecurityException {
+    // Initialize client that will be used to send requests. This client only needs to be created
+    // once, and can be reused for multiple requests
+    HttpCredentialsAdapter httpCredentialsAdapter =
+        new HttpCredentialsAdapter(
+            GoogleCredentials.getApplicationDefault().createScoped(BigqueryScopes.all()));
+    this.client =
+        new Bigquery.Builder(
+                GoogleNetHttpTransport.newTrustedTransport(),
+                GsonFactory.getDefaultInstance(),
+                httpRequest -> {
+                  httpCredentialsAdapter.initialize(httpRequest);
+                  // Instead of throwing exceptions, analyze the HttpResponse object and inspect its
+                  // status code. This will allow BigQuery API errors to be converted into Iceberg
+                  // exceptions.
+                  httpRequest.setThrowExceptionOnExecuteError(false);
+                })
+            .setApplicationName("BigQuery Metastore Iceberg Catalog Plugin")
+            .build();
+    this.bigqueryOptions = options;
+  }
+
+  @Override
+  public Dataset create(Dataset dataset) {
+    Dataset response = null;
+    try {
+      response =
+          BigQueryRetryHelper.runWithRetries(
+              () -> internalCreate(dataset),
+              bigqueryOptions.getRetrySettings(),
+              BIGQUERY_EXCEPTION_HANDLER,
+              bigqueryOptions.getClock(),
+              DEFAULT_RETRY_CONFIG);
+    } catch (BigQueryRetryHelper.BigQueryRetryHelperException e) {
+      handleBigQueryRetryException(e);
+    }
+    return response;
+  }
+
+  @SuppressWarnings("FormatStringAnnotation")
+  private Dataset internalCreate(Dataset dataset) {
+    try {
+      HttpResponse response =
+          client
+              .datasets()
+              .insert(dataset.getDatasetReference().getProjectId(), dataset)
+              .executeUnparsed();
+      return convertExceptionIfUnsuccessful(response).parseAs(Dataset.class);
+    } catch (IOException e) {
+      throw new RuntimeIOException(e);
+    } catch (AlreadyExistsException e) {
+      throw new AlreadyExistsException("Namespace already exists: " + dataset.getId());
+    }
+  }
+
+  @Override
+  @SuppressWarnings("FormatStringAnnotation")
+  public Dataset load(DatasetReference datasetReference) {
+    try {
+      HttpResponse response =
+          client
+              .datasets()
+              .get(datasetReference.getProjectId(), datasetReference.getDatasetId())
+              .executeUnparsed();
+      if (response.getStatusCode() == HttpStatusCodes.STATUS_CODE_NOT_FOUND) {
+        throw new NoSuchNamespaceException(
+            "Namespace does not exist: " + datasetReference.getDatasetId());
+      }
+
+      return convertExceptionIfUnsuccessful(response).parseAs(Dataset.class);
+    } catch (IOException e) {
+      throw new RuntimeIOException(e);
+    }
+  }
+
+  @Override
+  @SuppressWarnings("FormatStringAnnotation")

Review Comment:
   same as above



-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org

For queries about this service, please contact Infrastructure at:
users@infra.apache.org


---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For additional commands, e-mail: issues-help@iceberg.apache.org


Re: [PR] Catalog: Add BigQuery Metastore Catalog Support [iceberg]

Posted by "nastra (via GitHub)" <gi...@apache.org>.
nastra commented on code in PR #12808:
URL: https://github.com/apache/iceberg/pull/12808#discussion_r2066093734


##########
bigquery/src/test/java/org/apache/iceberg/gcp/bigquery/BigQueryTableOperationsTest.java:
##########
@@ -0,0 +1,274 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.iceberg.gcp.bigquery;
+
+import static org.apache.iceberg.gcp.bigquery.BigQueryMetastoreCatalog.PROJECT_ID;
+import static org.apache.iceberg.types.Types.NestedField.required;
+import static org.assertj.core.api.Assertions.assertThat;
+import static org.assertj.core.api.Assertions.assertThatThrownBy;
+import static org.mockito.Mockito.any;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.reset;
+import static org.mockito.Mockito.times;
+import static org.mockito.Mockito.verify;
+import static org.mockito.Mockito.when;
+
+import com.google.api.services.bigquery.model.Dataset;
+import com.google.api.services.bigquery.model.DatasetReference;
+import com.google.api.services.bigquery.model.ExternalCatalogDatasetOptions;
+import com.google.api.services.bigquery.model.ExternalCatalogTableOptions;
+import com.google.api.services.bigquery.model.StorageDescriptor;
+import com.google.api.services.bigquery.model.Table;
+import com.google.api.services.bigquery.model.TableReference;
+import java.io.File;
+import java.io.IOException;
+import java.util.Collections;
+import java.util.Optional;
+import org.apache.commons.io.FileUtils;
+import org.apache.commons.io.filefilter.TrueFileFilter;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.iceberg.CatalogProperties;
+import org.apache.iceberg.PartitionSpec;
+import org.apache.iceberg.Schema;
+import org.apache.iceberg.catalog.TableIdentifier;
+import org.apache.iceberg.exceptions.CommitFailedException;
+import org.apache.iceberg.exceptions.NoSuchTableException;
+import org.apache.iceberg.exceptions.ValidationException;
+import org.apache.iceberg.relocated.com.google.common.collect.ImmutableMap;
+import org.apache.iceberg.types.Types;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.io.TempDir;
+import org.mockito.ArgumentCaptor;
+
+public class BigQueryTableOperationsTest {
+
+  @TempDir private File tempFolder;
+  public static final String METADATA_LOCATION_PROP = "metadata_location";
+  private static final String GCP_PROJECT = "my-project";
+  private static final String GCP_REGION = "us";
+  private static final String DATASET_ID = "db";
+  private static final String TABLE_ID = "tbl";
+  private static final TableIdentifier SPARK_TABLE_ID = TableIdentifier.of(DATASET_ID, TABLE_ID);
+
+  private static final TableReference TABLE_REFERENCE =
+      new TableReference().setProjectId(GCP_PROJECT).setDatasetId(DATASET_ID).setTableId(TABLE_ID);
+
+  private final BigQueryMetastoreClient bigQueryMetaStoreClient =
+      mock(BigQueryMetastoreClient.class);
+
+  private BigQueryMetastoreCatalog bigQueryMetastoreCatalog;
+  private BigQueryTableOperations tableOps;
+
+  @BeforeEach
+  public void before() {
+    this.bigQueryMetastoreCatalog = new BigQueryMetastoreCatalog();
+    this.bigQueryMetastoreCatalog.setConf(new Configuration());
+    String warehouseLocation = tempFolder.toPath().resolve("hive-warehouse").toString();
+
+    bigQueryMetastoreCatalog.initialize(
+        "CATALOG_ID",
+        ImmutableMap.of(
+            PROJECT_ID,
+            GCP_PROJECT,
+            CatalogProperties.WAREHOUSE_LOCATION,
+            warehouseLocation,
+            CatalogProperties.FILE_IO_IMPL,
+            "org.apache.iceberg.hadoop.HadoopFileIO"),
+        GCP_PROJECT,
+        GCP_REGION,
+        bigQueryMetaStoreClient);
+    this.tableOps = (BigQueryTableOperations) bigQueryMetastoreCatalog.newTableOps(SPARK_TABLE_ID);
+  }
+
+  @Test
+  public void testFetchLatestMetadataFromBigQuery() throws Exception {

Review Comment:
   existing tests use `test` as a prefix in test methods but for new test classes we typically try to avoid using `test` in the prefix as it's a legacy thing from old JUnit days that isn't needed anymore



-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org

For queries about this service, please contact Infrastructure at:
users@infra.apache.org


---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For additional commands, e-mail: issues-help@iceberg.apache.org


Re: [PR] Catalog: Add BigQuery Metastore Catalog Support [iceberg]

Posted by "talatuyarer (via GitHub)" <gi...@apache.org>.
talatuyarer commented on code in PR #12808:
URL: https://github.com/apache/iceberg/pull/12808#discussion_r2051086168


##########
bigquery/src/test/java/org/apache/iceberg/gcp/bigquery/BigQueryMetastoreTestUtils.java:
##########
@@ -0,0 +1,86 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.iceberg.gcp.bigquery;
+
+import static org.apache.iceberg.types.Types.NestedField.required;
+import static org.assertj.core.api.Assertions.assertThat;
+
+import com.google.api.services.bigquery.model.ExternalCatalogTableOptions;
+import com.google.api.services.bigquery.model.StorageDescriptor;
+import com.google.api.services.bigquery.model.Table;
+import com.google.api.services.bigquery.model.TableReference;
+import java.io.File;
+import java.io.IOException;
+import java.util.Collections;
+import java.util.Optional;
+import org.apache.commons.io.FileUtils;
+import org.apache.commons.io.filefilter.TrueFileFilter;
+import org.apache.iceberg.Schema;
+import org.apache.iceberg.catalog.TableIdentifier;
+import org.apache.iceberg.types.Types;
+
+/** Test utility methods for BigQuery Metastore Iceberg catalog. */
+public final class BigQueryMetastoreTestUtils {
+  private BigQueryMetastoreTestUtils() {}
+
+  public static final String METADATA_LOCATION_PROP = "metadata_location";
+
+  public static Schema getTestSchema() {
+    return new Schema(
+        required(1, "id", Types.IntegerType.get(), "unique ID"),
+        required(2, "data", Types.StringType.get()));
+  }
+
+  public static Table createTestTable(
+      File tempFolder,
+      BigQueryMetastoreCatalog bigQueryMetastoreCatalog,
+      TableReference tableReference)
+      throws IOException {
+    Schema schema = getTestSchema();
+    TableIdentifier tableIdentifier =
+        TableIdentifier.of(tableReference.getDatasetId(), tableReference.getTableId());
+    String tableDir = tempFolder.toPath().resolve(tableReference.getTableId()).toString();
+
+    bigQueryMetastoreCatalog
+        .buildTable(tableIdentifier, schema)
+        .withLocation(tableDir)
+        .createTransaction()
+        .commitTransaction();
+
+    Optional<String> metadataLocation = getIcebergMetadataFilePath(tableDir);
+    assertThat(metadataLocation.isPresent());

Review Comment:
   I am checking is metadataLocation is exists or not. It is optional. isPresent will return True then assertThat works. 



-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org

For queries about this service, please contact Infrastructure at:
users@infra.apache.org


---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For additional commands, e-mail: issues-help@iceberg.apache.org


Re: [PR] Catalog: Add BigQuery Metastore Catalog Support [iceberg]

Posted by "nastra (via GitHub)" <gi...@apache.org>.
nastra commented on PR #12808:
URL: https://github.com/apache/iceberg/pull/12808#issuecomment-3077400196

   @locchipinti yes this will be shipped with the next release


-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org

For queries about this service, please contact Infrastructure at:
users@infra.apache.org


---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For additional commands, e-mail: issues-help@iceberg.apache.org


Re: [PR] Catalog: Add BigQuery Metastore Catalog Support [iceberg]

Posted by "nastra (via GitHub)" <gi...@apache.org>.
nastra commented on code in PR #12808:
URL: https://github.com/apache/iceberg/pull/12808#discussion_r2066077490


##########
bigquery/src/test/java/org/apache/iceberg/gcp/bigquery/BigQueryTableOperationsTest.java:
##########
@@ -0,0 +1,274 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.iceberg.gcp.bigquery;
+
+import static org.apache.iceberg.gcp.bigquery.BigQueryMetastoreCatalog.PROJECT_ID;
+import static org.apache.iceberg.types.Types.NestedField.required;
+import static org.assertj.core.api.Assertions.assertThat;
+import static org.assertj.core.api.Assertions.assertThatThrownBy;
+import static org.mockito.Mockito.any;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.reset;
+import static org.mockito.Mockito.times;
+import static org.mockito.Mockito.verify;
+import static org.mockito.Mockito.when;
+
+import com.google.api.services.bigquery.model.Dataset;
+import com.google.api.services.bigquery.model.DatasetReference;
+import com.google.api.services.bigquery.model.ExternalCatalogDatasetOptions;
+import com.google.api.services.bigquery.model.ExternalCatalogTableOptions;
+import com.google.api.services.bigquery.model.StorageDescriptor;
+import com.google.api.services.bigquery.model.Table;
+import com.google.api.services.bigquery.model.TableReference;
+import java.io.File;
+import java.io.IOException;
+import java.util.Collections;
+import java.util.Optional;
+import org.apache.commons.io.FileUtils;
+import org.apache.commons.io.filefilter.TrueFileFilter;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.iceberg.CatalogProperties;
+import org.apache.iceberg.PartitionSpec;
+import org.apache.iceberg.Schema;
+import org.apache.iceberg.catalog.TableIdentifier;
+import org.apache.iceberg.exceptions.CommitFailedException;
+import org.apache.iceberg.exceptions.NoSuchTableException;
+import org.apache.iceberg.exceptions.ValidationException;
+import org.apache.iceberg.relocated.com.google.common.collect.ImmutableMap;
+import org.apache.iceberg.types.Types;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.io.TempDir;
+import org.mockito.ArgumentCaptor;
+
+public class BigQueryTableOperationsTest {
+
+  @TempDir private File tempFolder;
+  public static final String METADATA_LOCATION_PROP = "metadata_location";

Review Comment:
   this is already defined in `BaseMetastoreTableOperations` so can be reused from there



-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org

For queries about this service, please contact Infrastructure at:
users@infra.apache.org


---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For additional commands, e-mail: issues-help@iceberg.apache.org


Re: [PR] Catalog: Add BigQuery Metastore Catalog Support [iceberg]

Posted by "talatuyarer (via GitHub)" <gi...@apache.org>.
talatuyarer commented on code in PR #12808:
URL: https://github.com/apache/iceberg/pull/12808#discussion_r2067872969


##########
bigquery/src/main/java/org/apache/iceberg/gcp/bigquery/BigQueryMetastoreClientImpl.java:
##########
@@ -0,0 +1,594 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.iceberg.gcp.bigquery;
+
+import com.google.api.client.googleapis.javanet.GoogleNetHttpTransport;
+import com.google.api.client.http.HttpHeaders;
+import com.google.api.client.http.HttpResponse;
+import com.google.api.client.http.HttpResponseException;
+import com.google.api.client.http.HttpStatusCodes;
+import com.google.api.client.json.gson.GsonFactory;
+import com.google.api.client.util.Data;
+import com.google.api.services.bigquery.Bigquery;
+import com.google.api.services.bigquery.BigqueryScopes;
+import com.google.api.services.bigquery.model.Dataset;
+import com.google.api.services.bigquery.model.DatasetList;
+import com.google.api.services.bigquery.model.DatasetList.Datasets;
+import com.google.api.services.bigquery.model.DatasetReference;
+import com.google.api.services.bigquery.model.ExternalCatalogDatasetOptions;
+import com.google.api.services.bigquery.model.ExternalCatalogTableOptions;
+import com.google.api.services.bigquery.model.Table;
+import com.google.api.services.bigquery.model.TableList;
+import com.google.api.services.bigquery.model.TableList.Tables;
+import com.google.api.services.bigquery.model.TableReference;
+import com.google.api.services.bigquery.model.TableSchema;
+import com.google.auth.http.HttpCredentialsAdapter;
+import com.google.auth.oauth2.GoogleCredentials;
+import com.google.cloud.BaseServiceException;
+import com.google.cloud.ExceptionHandler;
+import com.google.cloud.bigquery.BigQueryErrorMessages;
+import com.google.cloud.bigquery.BigQueryOptions;
+import com.google.cloud.bigquery.BigQueryRetryConfig;
+import com.google.cloud.bigquery.BigQueryRetryHelper;
+import java.io.IOException;
+import java.nio.charset.StandardCharsets;
+import java.security.GeneralSecurityException;
+import java.util.List;
+import java.util.Locale;
+import java.util.Map;
+import java.util.Set;
+import java.util.stream.Collectors;
+import java.util.stream.Stream;
+import org.apache.iceberg.BaseMetastoreTableOperations;
+import org.apache.iceberg.exceptions.AlreadyExistsException;
+import org.apache.iceberg.exceptions.BadRequestException;
+import org.apache.iceberg.exceptions.ForbiddenException;
+import org.apache.iceberg.exceptions.NoSuchIcebergTableException;
+import org.apache.iceberg.exceptions.NoSuchNamespaceException;
+import org.apache.iceberg.exceptions.NoSuchTableException;
+import org.apache.iceberg.exceptions.NotAuthorizedException;
+import org.apache.iceberg.exceptions.RuntimeIOException;
+import org.apache.iceberg.exceptions.ServiceFailureException;
+import org.apache.iceberg.exceptions.ServiceUnavailableException;
+import org.apache.iceberg.exceptions.ValidationException;
+import org.apache.iceberg.relocated.com.google.common.base.Preconditions;
+import org.apache.iceberg.relocated.com.google.common.collect.Lists;
+import org.apache.iceberg.relocated.com.google.common.collect.Maps;
+
+/** A client of Google Bigquery Metastore functions over the BigQuery service. */
+public final class BigQueryMetastoreClientImpl implements BigQueryMetastoreClient {
+
+  public static final ExceptionHandler.Interceptor EXCEPTION_HANDLER_INTERCEPTOR =
+      new ExceptionHandler.Interceptor() {
+
+        private static final long serialVersionUID = -8429573486870467828L;
+
+        @Override
+        public RetryResult afterEval(Exception exception, RetryResult retryResult) {
+          return ExceptionHandler.Interceptor.RetryResult.CONTINUE_EVALUATION;
+        }
+
+        @Override
+        public RetryResult beforeEval(Exception exception) {
+          if (exception instanceof BaseServiceException) {
+            boolean retriable = ((BaseServiceException) exception).isRetryable();
+            return retriable
+                ? ExceptionHandler.Interceptor.RetryResult.RETRY
+                : ExceptionHandler.Interceptor.RetryResult.CONTINUE_EVALUATION;
+          }
+
+          return ExceptionHandler.Interceptor.RetryResult.CONTINUE_EVALUATION;
+        }
+      };
+
+  private final Bigquery client;
+  private final BigQueryOptions bigqueryOptions;
+
+  // Retry config with error messages and regex for rate limit exceeded errors.
+  private static final BigQueryRetryConfig DEFAULT_RETRY_CONFIG =
+      BigQueryRetryConfig.newBuilder()
+          .retryOnMessage(BigQueryErrorMessages.RATE_LIMIT_EXCEEDED_MSG)
+          .retryOnMessage(BigQueryErrorMessages.JOB_RATE_LIMIT_EXCEEDED_MSG)
+          .retryOnRegEx(BigQueryErrorMessages.RetryRegExPatterns.RATE_LIMIT_EXCEEDED_REGEX)
+          .build();
+
+  public static final ExceptionHandler BIGQUERY_EXCEPTION_HANDLER =
+      ExceptionHandler.newBuilder()
+          .abortOn(RuntimeException.class)
+          .retryOn(
+              java.net.ConnectException
+                  .class) // Retry on connection failures due to transient network issues.

Review Comment:
   Done



-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org

For queries about this service, please contact Infrastructure at:
users@infra.apache.org


---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For additional commands, e-mail: issues-help@iceberg.apache.org


Re: [PR] Catalog: Add BigQuery Metastore Catalog Support [iceberg]

Posted by "talatuyarer (via GitHub)" <gi...@apache.org>.
talatuyarer commented on code in PR #12808:
URL: https://github.com/apache/iceberg/pull/12808#discussion_r2067868284


##########
bigquery/src/main/java/org/apache/iceberg/gcp/bigquery/BigQueryMetastoreCatalog.java:
##########
@@ -0,0 +1,405 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.iceberg.gcp.bigquery;
+
+import com.google.api.services.bigquery.model.Dataset;
+import com.google.api.services.bigquery.model.DatasetList.Datasets;
+import com.google.api.services.bigquery.model.DatasetReference;
+import com.google.api.services.bigquery.model.ExternalCatalogDatasetOptions;
+import com.google.api.services.bigquery.model.TableReference;
+import com.google.cloud.ServiceOptions;
+import com.google.cloud.bigquery.BigQueryOptions;
+import java.io.IOException;
+import java.security.GeneralSecurityException;
+import java.util.List;
+import java.util.Locale;
+import java.util.Map;
+import java.util.Objects;
+import java.util.Set;
+import org.apache.hadoop.conf.Configurable;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.iceberg.BaseMetastoreCatalog;
+import org.apache.iceberg.CatalogProperties;
+import org.apache.iceberg.CatalogUtil;
+import org.apache.iceberg.TableMetadata;
+import org.apache.iceberg.TableOperations;
+import org.apache.iceberg.catalog.Namespace;
+import org.apache.iceberg.catalog.SupportsNamespaces;
+import org.apache.iceberg.catalog.TableIdentifier;
+import org.apache.iceberg.exceptions.AlreadyExistsException;
+import org.apache.iceberg.exceptions.NoSuchNamespaceException;
+import org.apache.iceberg.exceptions.NoSuchTableException;
+import org.apache.iceberg.exceptions.ServiceFailureException;
+import org.apache.iceberg.exceptions.ValidationException;
+import org.apache.iceberg.io.FileIO;
+import org.apache.iceberg.relocated.com.google.common.annotations.VisibleForTesting;
+import org.apache.iceberg.relocated.com.google.common.base.Preconditions;
+import org.apache.iceberg.relocated.com.google.common.base.Strings;
+import org.apache.iceberg.relocated.com.google.common.collect.ImmutableList;
+import org.apache.iceberg.relocated.com.google.common.collect.ImmutableMap;
+import org.apache.iceberg.relocated.com.google.common.collect.Maps;
+import org.apache.iceberg.util.LocationUtil;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/** Iceberg Bigquery Metastore Catalog implementation. */
+public class BigQueryMetastoreCatalog extends BaseMetastoreCatalog
+    implements SupportsNamespaces, Configurable {
+
+  // User provided properties.
+  public static final String PROJECT_ID = "gcp-project";
+  public static final String GCP_LOCATION = "gcp-location";
+  public static final String LIST_ALL_TABLES = "list-all-tables";
+
+  public static final String HIVE_METASTORE_WAREHOUSE_DIR = "hive.metastore.warehouse.dir";
+
+  private static final Logger LOG = LoggerFactory.getLogger(BigQueryMetastoreCatalog.class);
+
+  private static final String DEFAULT_GCP_LOCATION = "us";
+
+  private String catalogName;
+  private Map<String, String> catalogProperties;
+  private FileIO fileIO;
+  private Configuration conf;
+  private String projectId;
+  private String location;

Review Comment:
   Done



-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org

For queries about this service, please contact Infrastructure at:
users@infra.apache.org


---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For additional commands, e-mail: issues-help@iceberg.apache.org


Re: [PR] Catalog: Add BigQuery Metastore Catalog Support [iceberg]

Posted by "talatuyarer (via GitHub)" <gi...@apache.org>.
talatuyarer commented on code in PR #12808:
URL: https://github.com/apache/iceberg/pull/12808#discussion_r2067872433


##########
bigquery/src/main/java/org/apache/iceberg/gcp/bigquery/BigQueryMetastoreCatalog.java:
##########
@@ -0,0 +1,405 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.iceberg.gcp.bigquery;
+
+import com.google.api.services.bigquery.model.Dataset;
+import com.google.api.services.bigquery.model.DatasetList.Datasets;
+import com.google.api.services.bigquery.model.DatasetReference;
+import com.google.api.services.bigquery.model.ExternalCatalogDatasetOptions;
+import com.google.api.services.bigquery.model.TableReference;
+import com.google.cloud.ServiceOptions;
+import com.google.cloud.bigquery.BigQueryOptions;
+import java.io.IOException;
+import java.security.GeneralSecurityException;
+import java.util.List;
+import java.util.Locale;
+import java.util.Map;
+import java.util.Objects;
+import java.util.Set;
+import org.apache.hadoop.conf.Configurable;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.iceberg.BaseMetastoreCatalog;
+import org.apache.iceberg.CatalogProperties;
+import org.apache.iceberg.CatalogUtil;
+import org.apache.iceberg.TableMetadata;
+import org.apache.iceberg.TableOperations;
+import org.apache.iceberg.catalog.Namespace;
+import org.apache.iceberg.catalog.SupportsNamespaces;
+import org.apache.iceberg.catalog.TableIdentifier;
+import org.apache.iceberg.exceptions.AlreadyExistsException;
+import org.apache.iceberg.exceptions.NoSuchNamespaceException;
+import org.apache.iceberg.exceptions.NoSuchTableException;
+import org.apache.iceberg.exceptions.ServiceFailureException;
+import org.apache.iceberg.exceptions.ValidationException;
+import org.apache.iceberg.io.FileIO;
+import org.apache.iceberg.relocated.com.google.common.annotations.VisibleForTesting;
+import org.apache.iceberg.relocated.com.google.common.base.Preconditions;
+import org.apache.iceberg.relocated.com.google.common.base.Strings;
+import org.apache.iceberg.relocated.com.google.common.collect.ImmutableList;
+import org.apache.iceberg.relocated.com.google.common.collect.ImmutableMap;
+import org.apache.iceberg.relocated.com.google.common.collect.Maps;
+import org.apache.iceberg.util.LocationUtil;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/** Iceberg Bigquery Metastore Catalog implementation. */
+public class BigQueryMetastoreCatalog extends BaseMetastoreCatalog
+    implements SupportsNamespaces, Configurable {
+
+  // User provided properties.
+  public static final String PROJECT_ID = "gcp-project";
+  public static final String GCP_LOCATION = "gcp-location";
+  public static final String LIST_ALL_TABLES = "list-all-tables";
+
+  public static final String HIVE_METASTORE_WAREHOUSE_DIR = "hive.metastore.warehouse.dir";

Review Comment:
   Ok I made this as class variable. Removed conf set and get. 



-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org

For queries about this service, please contact Infrastructure at:
users@infra.apache.org


---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For additional commands, e-mail: issues-help@iceberg.apache.org


Re: [PR] Catalog: Add BigQuery Metastore Catalog Support [iceberg]

Posted by "talatuyarer (via GitHub)" <gi...@apache.org>.
talatuyarer commented on code in PR #12808:
URL: https://github.com/apache/iceberg/pull/12808#discussion_r2067930498


##########
bigquery/src/main/java/org/apache/iceberg/gcp/bigquery/BigQueryMetastoreCatalog.java:
##########
@@ -0,0 +1,371 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.iceberg.gcp.bigquery;
+
+import com.google.api.services.bigquery.model.Dataset;
+import com.google.api.services.bigquery.model.DatasetList.Datasets;
+import com.google.api.services.bigquery.model.DatasetReference;
+import com.google.api.services.bigquery.model.ExternalCatalogDatasetOptions;
+import com.google.api.services.bigquery.model.TableReference;
+import com.google.cloud.ServiceOptions;
+import com.google.cloud.bigquery.BigQueryOptions;
+import java.io.IOException;
+import java.security.GeneralSecurityException;
+import java.util.List;
+import java.util.Locale;
+import java.util.Map;
+import java.util.Objects;
+import java.util.Set;
+import org.apache.hadoop.conf.Configurable;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.iceberg.BaseMetastoreCatalog;
+import org.apache.iceberg.CatalogProperties;
+import org.apache.iceberg.CatalogUtil;
+import org.apache.iceberg.TableMetadata;
+import org.apache.iceberg.TableOperations;
+import org.apache.iceberg.catalog.Namespace;
+import org.apache.iceberg.catalog.SupportsNamespaces;
+import org.apache.iceberg.catalog.TableIdentifier;
+import org.apache.iceberg.exceptions.NoSuchNamespaceException;
+import org.apache.iceberg.exceptions.NoSuchTableException;
+import org.apache.iceberg.exceptions.ServiceFailureException;
+import org.apache.iceberg.exceptions.ValidationException;
+import org.apache.iceberg.io.FileIO;
+import org.apache.iceberg.relocated.com.google.common.annotations.VisibleForTesting;
+import org.apache.iceberg.relocated.com.google.common.base.Preconditions;
+import org.apache.iceberg.relocated.com.google.common.base.Strings;
+import org.apache.iceberg.relocated.com.google.common.collect.ImmutableList;
+import org.apache.iceberg.relocated.com.google.common.collect.ImmutableMap;
+import org.apache.iceberg.relocated.com.google.common.collect.Maps;
+import org.apache.iceberg.util.LocationUtil;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/** Iceberg Bigquery Metastore Catalog implementation. */
+public class BigQueryMetastoreCatalog extends BaseMetastoreCatalog
+    implements SupportsNamespaces, Configurable {
+
+  // User provided properties.
+  public static final String PROJECT_ID = "gcp.bigquery.project-id";
+  public static final String GCP_LOCATION = "gcp.bigquery.location";
+  public static final String LIST_ALL_TABLES = "gcp.bigquery.list-all-tables";
+
+  private static final Logger LOG = LoggerFactory.getLogger(BigQueryMetastoreCatalog.class);
+
+  private static final String DEFAULT_GCP_LOCATION = "us";
+
+  private String catalogName;
+  private Map<String, String> catalogProperties;
+  private FileIO fileIO;
+  private Configuration conf;
+  private String projectId;
+  private String projectLocation;
+  private BigQueryMetastoreClient client;
+  private boolean listAllTables;
+  private String warehouseLocation;
+
+  // Must have a no-arg constructor to be dynamically loaded
+  // initialize(String name, Map<String, String> properties) will be called to complete
+  // initialization
+  public BigQueryMetastoreCatalog() {}
+
+  @Override
+  public void initialize(String name, Map<String, String> properties) {
+    Preconditions.checkArgument(
+        properties.containsKey(PROJECT_ID),
+        "Invalid GCP project: %s must be specified",
+        PROJECT_ID);
+
+    this.projectId = properties.get(PROJECT_ID);
+    this.projectLocation = properties.getOrDefault(GCP_LOCATION, DEFAULT_GCP_LOCATION);
+
+    BigQueryOptions options =
+        BigQueryOptions.newBuilder()
+            .setProjectId(projectId)
+            .setLocation(projectLocation)
+            .setRetrySettings(ServiceOptions.getDefaultRetrySettings())
+            .build();
+
+    try {
+      client = new BigQueryMetastoreClientImpl(options);
+    } catch (IOException e) {
+      throw new ServiceFailureException(e, "Creating BigQuery client failed");
+    } catch (GeneralSecurityException e) {
+      throw new ValidationException(e, "Creating BigQuery client failed due to a security issue");
+    }
+
+    initialize(name, properties, projectId, projectLocation, client);
+  }
+
+  @VisibleForTesting
+  void initialize(
+      String inputName,
+      Map<String, String> properties,
+      String initialProjectId,
+      String initialLocation,
+      BigQueryMetastoreClient bigQueryMetaStoreClient) {
+    Preconditions.checkArgument(bigQueryMetaStoreClient != null, "Invalid BigQuery client: null");
+    this.catalogName = inputName;
+    this.catalogProperties = ImmutableMap.copyOf(properties);
+    this.projectId = initialProjectId;
+    this.projectLocation = initialLocation;
+    this.client = bigQueryMetaStoreClient;
+
+    if (this.conf == null) {
+      LOG.warn("No configuration was set, using the default environment Configuration");
+      this.conf = new Configuration();
+    }
+
+    LOG.info("Using BigQuery Metastore Iceberg Catalog: {}", inputName);
+
+    if (properties.containsKey(CatalogProperties.WAREHOUSE_LOCATION)) {
+      this.warehouseLocation =
+          LocationUtil.stripTrailingSlash(properties.get(CatalogProperties.WAREHOUSE_LOCATION));
+    }
+
+    this.fileIO =
+        CatalogUtil.loadFileIO(
+            properties.getOrDefault(
+                CatalogProperties.FILE_IO_IMPL, "org.apache.iceberg.io.ResolvingFileIO"),
+            properties,
+            conf);
+
+    this.listAllTables = Boolean.parseBoolean(properties.getOrDefault(LIST_ALL_TABLES, "true"));
+  }
+
+  @Override
+  protected TableOperations newTableOps(TableIdentifier identifier) {
+    return new BigQueryTableOperations(client, fileIO, toTableReference(identifier), conf);
+  }
+
+  @Override
+  protected String defaultWarehouseLocation(TableIdentifier identifier) {
+    String locationUri = null;
+    DatasetReference datasetReference = toDatasetReference(identifier.namespace());
+    Dataset dataset = client.load(datasetReference);
+    if (dataset != null && dataset.getExternalCatalogDatasetOptions() != null) {
+      locationUri = dataset.getExternalCatalogDatasetOptions().getDefaultStorageLocationUri();
+    }
+
+    return String.format(
+        "%s/%s",
+        Strings.isNullOrEmpty(locationUri)
+            ? createDefaultStorageLocationUri(datasetReference.getDatasetId())
+            : LocationUtil.stripTrailingSlash(locationUri),
+        identifier.name());
+  }
+
+  @Override
+  public List<TableIdentifier> listTables(Namespace namespace) {
+    validateNamespace(namespace);
+
+    return client.list(toDatasetReference(namespace), listAllTables).stream()
+        .map(
+            table -> TableIdentifier.of(namespace.level(0), table.getTableReference().getTableId()))
+        .collect(ImmutableList.toImmutableList());
+  }
+
+  @Override
+  public boolean dropTable(TableIdentifier identifier, boolean purge) {
+    try {
+      TableOperations ops = newTableOps(identifier);
+      TableMetadata lastMetadata = ops.current();
+
+      client.delete(toTableReference(identifier));
+
+      if (purge && lastMetadata != null) {
+        CatalogUtil.dropTableData(ops.io(), lastMetadata);
+      }
+    } catch (NoSuchTableException e) {
+      return false;
+    }
+
+    return true;
+  }
+
+  @Override
+  public void renameTable(TableIdentifier from, TableIdentifier to) {
+    // TODO: Enable once supported by BigQuery API.
+    throw new UnsupportedOperationException("Table rename operation is unsupported.");
+  }
+
+  @Override
+  public void createNamespace(Namespace namespace, Map<String, String> metadata) {
+    Dataset builder = new Dataset();
+    DatasetReference datasetReference = toDatasetReference(namespace);
+    builder.setLocation(this.projectLocation);
+    builder.setDatasetReference(datasetReference);
+    builder.setExternalCatalogDatasetOptions(
+        BigQueryMetastoreUtils.createExternalCatalogDatasetOptions(
+            createDefaultStorageLocationUri(datasetReference.getDatasetId()), metadata));
+
+    client.create(builder);
+  }
+
+  @Override
+  public List<Namespace> listNamespaces() {
+    try {
+      return listNamespaces(Namespace.empty());
+    } catch (NoSuchNamespaceException e) {
+      return ImmutableList.of();
+    }
+  }
+
+  /**
+   * Since this catalog only supports one-level namespaces, it always returns an empty list unless
+   * passed an empty namespace to list all namespaces within the catalog.
+   */
+  @Override
+  public List<Namespace> listNamespaces(Namespace namespace) {
+    List<Datasets> allDatasets = client.list(projectId);
+
+    ImmutableList<Namespace> namespaces =
+        allDatasets.stream().map(this::toNamespace).collect(ImmutableList.toImmutableList());
+    if (namespaces.isEmpty()) {
+      throw new NoSuchNamespaceException("Namespace does not exist: %s", namespace);
+    }
+
+    return namespaces;
+  }
+
+  @Override
+  public boolean dropNamespace(Namespace namespace) {
+    try {
+      client.delete(toDatasetReference(namespace));
+      // We don't delete the data folder for safety, which aligns with Hive Metastore's default
+      // behavior.
+      // We can support database or catalog level config controlling file deletion in the future.
+      return true;
+    } catch (NoSuchNamespaceException e) {
+      return false;
+    }
+  }
+
+  @Override
+  public boolean setProperties(Namespace namespace, Map<String, String> properties) {
+    Dataset dataset = client.load(toDatasetReference(namespace));
+
+    ExternalCatalogDatasetOptions existingOptions = dataset.getExternalCatalogDatasetOptions();
+    Map<String, String> existingParameters =
+        existingOptions != null ? existingOptions.getParameters() : null;
+
+    Map<String, String> newParameters = Maps.newHashMap();
+    if (existingParameters != null) {
+      newParameters.putAll(existingParameters);
+    }
+
+    newParameters.putAll(properties);
+
+    if (Objects.equals(existingParameters, newParameters)) {
+      // No change in parameters detected
+      return false;
+    }
+
+    client.setParameters(toDatasetReference(namespace), properties);
+    return true;
+  }
+
+  @Override
+  public boolean removeProperties(Namespace namespace, Set<String> properties) {
+    client.removeParameters(toDatasetReference(namespace), properties);
+    return true;
+  }
+
+  @SuppressWarnings("FormatStringAnnotation")

Review Comment:
   I did not think about this trick. Thank you. I removed all. 



-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org

For queries about this service, please contact Infrastructure at:
users@infra.apache.org


---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For additional commands, e-mail: issues-help@iceberg.apache.org


Re: [PR] Catalog: Add BigQuery Metastore Catalog Support [iceberg]

Posted by "talatuyarer (via GitHub)" <gi...@apache.org>.
talatuyarer commented on code in PR #12808:
URL: https://github.com/apache/iceberg/pull/12808#discussion_r2067928742


##########
bigquery/src/test/java/org/apache/iceberg/gcp/bigquery/TestBigQueryTableOperations.java:
##########
@@ -0,0 +1,291 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.iceberg.gcp.bigquery;
+
+import static org.apache.iceberg.gcp.bigquery.BigQueryMetastoreCatalog.PROJECT_ID;
+import static org.apache.iceberg.types.Types.NestedField.required;
+import static org.assertj.core.api.Assertions.assertThat;
+import static org.assertj.core.api.Assertions.assertThatThrownBy;
+import static org.mockito.Mockito.any;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.reset;
+import static org.mockito.Mockito.times;
+import static org.mockito.Mockito.verify;
+import static org.mockito.Mockito.when;
+
+import com.google.api.services.bigquery.model.Dataset;
+import com.google.api.services.bigquery.model.DatasetReference;
+import com.google.api.services.bigquery.model.ExternalCatalogDatasetOptions;
+import com.google.api.services.bigquery.model.ExternalCatalogTableOptions;
+import com.google.api.services.bigquery.model.StorageDescriptor;
+import com.google.api.services.bigquery.model.Table;
+import com.google.api.services.bigquery.model.TableReference;
+import java.io.File;
+import java.io.IOException;
+import java.util.Collections;
+import java.util.Optional;
+import org.apache.commons.io.FileUtils;
+import org.apache.commons.io.filefilter.TrueFileFilter;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.iceberg.CatalogProperties;
+import org.apache.iceberg.PartitionSpec;
+import org.apache.iceberg.Schema;
+import org.apache.iceberg.catalog.TableIdentifier;
+import org.apache.iceberg.exceptions.CommitFailedException;
+import org.apache.iceberg.exceptions.NoSuchTableException;
+import org.apache.iceberg.exceptions.ValidationException;
+import org.apache.iceberg.relocated.com.google.common.collect.ImmutableMap;
+import org.apache.iceberg.types.Types;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.io.TempDir;
+import org.mockito.ArgumentCaptor;
+
+public class TestBigQueryTableOperations {
+
+  @TempDir private File tempFolder;
+  public static final String METADATA_LOCATION_PROP = "metadata_location";
+  private static final String GCP_PROJECT = "my-project";
+  private static final String GCP_REGION = "us";
+  private static final String NS = "db";
+  private static final String TABLE = "tbl";
+  private static final TableIdentifier IDENTIFIER = TableIdentifier.of(NS, TABLE);
+
+  private static final TableReference TABLE_REFERENCE =
+      new TableReference().setProjectId(GCP_PROJECT).setDatasetId(NS).setTableId(TABLE);
+
+  private static final Schema SCHEMA =
+      new Schema(
+          required(1, "id", Types.IntegerType.get(), "unique ID"),
+          required(2, "data", Types.StringType.get()));
+
+  private final BigQueryMetastoreClient client = mock(BigQueryMetastoreClient.class);
+
+  private BigQueryMetastoreCatalog catalog;
+  private BigQueryTableOperations tableOps;
+
+  @BeforeEach
+  public void before() {
+    this.catalog = new BigQueryMetastoreCatalog();
+    this.catalog.setConf(new Configuration());
+    String warehouseLocation = tempFolder.toPath().resolve("hive-warehouse").toString();
+
+    catalog.initialize(
+        "CATALOG_ID",
+        ImmutableMap.of(
+            PROJECT_ID,
+            GCP_PROJECT,
+            CatalogProperties.WAREHOUSE_LOCATION,
+            warehouseLocation,
+            CatalogProperties.FILE_IO_IMPL,
+            "org.apache.iceberg.hadoop.HadoopFileIO"),
+        GCP_PROJECT,
+        GCP_REGION,
+        client);
+    this.tableOps = (BigQueryTableOperations) catalog.newTableOps(IDENTIFIER);
+  }
+
+  @Test
+  public void fetchLatestMetadataFromBigQuery() throws Exception {
+    Table createdTable = createTestTable();
+    reset(client);
+    when(client.load(TABLE_REFERENCE)).thenReturn(createdTable);
+
+    tableOps.refresh();
+    assertThat(
+            createdTable
+                .getExternalCatalogTableOptions()
+                .getParameters()
+                .getOrDefault(METADATA_LOCATION_PROP, ""))
+        .isEqualTo(tableOps.currentMetadataLocation());
+
+    reset(client);
+    when(client.load(TABLE_REFERENCE))
+        .thenThrow(new NoSuchTableException("error message getTable"));
+    // Refresh fails when table is not found but metadata already presents.
+    assertThatThrownBy(() -> tableOps.refresh())
+        .isInstanceOf(NoSuchTableException.class)
+        .hasMessageContaining("error message getTable");
+  }
+
+  @Test
+  public void loadNonIcebergTableFails() {
+    when(client.load(TABLE_REFERENCE)).thenReturn(new Table().setTableReference(TABLE_REFERENCE));
+
+    assertThatThrownBy(() -> tableOps.refresh())
+        .isInstanceOf(ValidationException.class)
+        .hasMessageContaining("metadata location not found");
+  }
+
+  @Test
+  public void loadNoOpWhenMetadataAndTableNotFound() {
+    when(client.load(TABLE_REFERENCE))
+        .thenThrow(new NoSuchTableException("error message getTable"));
+    // Table not found won't cause errors when the metadata is null.
+    assertThat(tableOps.currentMetadataLocation()).isNull();

Review Comment:
   Yes I added. While applying changes Looks like I deleted but i did not added :) 



-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org

For queries about this service, please contact Infrastructure at:
users@infra.apache.org


---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For additional commands, e-mail: issues-help@iceberg.apache.org


Re: [PR] Catalog: Add BigQuery Metastore Catalog Support [iceberg]

Posted by "nastra (via GitHub)" <gi...@apache.org>.
nastra commented on code in PR #12808:
URL: https://github.com/apache/iceberg/pull/12808#discussion_r2055561741


##########
bigquery/src/main/java/org/apache/iceberg/gcp/bigquery/BigQueryMetastoreClient.java:
##########
@@ -0,0 +1,121 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.iceberg.gcp.bigquery;
+
+import com.google.api.services.bigquery.model.Dataset;
+import com.google.api.services.bigquery.model.DatasetList.Datasets;
+import com.google.api.services.bigquery.model.DatasetReference;
+import com.google.api.services.bigquery.model.Table;
+import com.google.api.services.bigquery.model.TableList.Tables;
+import com.google.api.services.bigquery.model.TableReference;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+
+/**
+ * A client of Google BigQuery Metastore functions over the BigQuery service. Uses the Google
+ * BigQuery API.
+ */
+public interface BigQueryMetastoreClient {
+
+  /**
+   * Creates and returns a new dataset.
+   *
+   * @param dataset the dataset to create
+   */
+  Dataset createDataset(Dataset dataset);
+
+  /**
+   * Returns a dataset.
+   *
+   * @param datasetReference full dataset reference
+   */
+  Dataset getDataset(DatasetReference datasetReference);
+
+  /**
+   * Deletes a dataset.
+   *
+   * @param datasetReference full dataset reference
+   */
+  void deleteDataset(DatasetReference datasetReference);
+
+  /**
+   * Updates parameters of a dataset or adds them if did not exist, leaving already-existing ones
+   * intact.
+   *
+   * @param datasetReference full dataset reference
+   * @param parameters metadata parameters to add
+   * @return dataset after patch
+   */
+  Dataset setDatasetParameters(DatasetReference datasetReference, Map<String, String> parameters);
+
+  /**
+   * Removes given set of keys of parameters of a dataset. Ignores keys that do not exist already.
+   *
+   * @param datasetReference full dataset reference
+   * @param parameters metadata parameter keys to remove
+   * @return dataset after patch
+   */
+  Dataset removeDatasetParameters(DatasetReference datasetReference, Set<String> parameters);
+
+  /**
+   * Lists datasets under a given project
+   *
+   * @param projectId the identifier of the project to list datasets under
+   */
+  List<Datasets> listDatasets(String projectId);
+
+  /**
+   * Creates and returns a new table.
+   *
+   * @param table body of the table to create
+   */
+  Table createTable(Table table);
+
+  /**
+   * Returns a table.
+   *
+   * @param tableReference full table reference
+   */
+  Table getTable(TableReference tableReference);
+
+  /**
+   * Updates the catalog table options of an Iceberg table and returns the updated table.
+   *
+   * @param tableReference full table reference
+   * @param table to patch
+   */
+  Table patchTable(TableReference tableReference, Table table);

Review Comment:
   what does patching a table mean? Should this maybe be called `updateTable`?



-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org

For queries about this service, please contact Infrastructure at:
users@infra.apache.org


---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For additional commands, e-mail: issues-help@iceberg.apache.org


Re: [PR] Catalog: Add BigQuery Metastore Catalog Support [iceberg]

Posted by "talatuyarer (via GitHub)" <gi...@apache.org>.
talatuyarer commented on code in PR #12808:
URL: https://github.com/apache/iceberg/pull/12808#discussion_r2057079908


##########
bigquery/src/main/java/org/apache/iceberg/gcp/bigquery/BigQueryMetastoreCatalog.java:
##########
@@ -0,0 +1,400 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.iceberg.gcp.bigquery;
+
+import com.google.api.services.bigquery.model.Dataset;
+import com.google.api.services.bigquery.model.DatasetList.Datasets;
+import com.google.api.services.bigquery.model.DatasetReference;
+import com.google.api.services.bigquery.model.ExternalCatalogDatasetOptions;
+import com.google.api.services.bigquery.model.TableReference;
+import com.google.cloud.ServiceOptions;
+import com.google.cloud.bigquery.BigQueryOptions;
+import java.io.IOException;
+import java.security.GeneralSecurityException;
+import java.util.List;
+import java.util.Locale;
+import java.util.Map;
+import java.util.Objects;
+import java.util.Set;
+import org.apache.hadoop.conf.Configurable;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.iceberg.BaseMetastoreCatalog;
+import org.apache.iceberg.CatalogProperties;
+import org.apache.iceberg.CatalogUtil;
+import org.apache.iceberg.TableMetadata;
+import org.apache.iceberg.TableOperations;
+import org.apache.iceberg.catalog.Namespace;
+import org.apache.iceberg.catalog.SupportsNamespaces;
+import org.apache.iceberg.catalog.TableIdentifier;
+import org.apache.iceberg.exceptions.AlreadyExistsException;
+import org.apache.iceberg.exceptions.NoSuchNamespaceException;
+import org.apache.iceberg.exceptions.NoSuchTableException;
+import org.apache.iceberg.exceptions.ServiceFailureException;
+import org.apache.iceberg.exceptions.ValidationException;
+import org.apache.iceberg.io.FileIO;
+import org.apache.iceberg.relocated.com.google.common.annotations.VisibleForTesting;
+import org.apache.iceberg.relocated.com.google.common.base.Preconditions;
+import org.apache.iceberg.relocated.com.google.common.base.Strings;
+import org.apache.iceberg.relocated.com.google.common.collect.ImmutableList;
+import org.apache.iceberg.relocated.com.google.common.collect.ImmutableMap;
+import org.apache.iceberg.relocated.com.google.common.collect.Maps;
+import org.apache.iceberg.util.LocationUtil;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/** Iceberg Bigquery Metastore Catalog implementation. */
+public final class BigQueryMetastoreCatalog extends BaseMetastoreCatalog
+    implements SupportsNamespaces, Configurable {
+
+  // User provided properties.
+  public static final String PROJECT_ID = "gcp-project";
+  public static final String GCP_LOCATION = "gcp-location";
+  public static final String FILTER_UNSUPPORTED_TABLES = "filter-unsupported-tables";
+  public static final String TESTING_ENABLED = "testing-enabled";
+
+  public static final String HIVE_METASTORE_WAREHOUSE_DIR = "hive.metastore.warehouse.dir";
+
+  private static final Logger LOG = LoggerFactory.getLogger(BigQueryMetastoreCatalog.class);
+
+  private static final String DEFAULT_GCP_LOCATION = "us";
+
+  private String catalogName;
+  private Map<String, String> catalogProperties;
+  private FileIO fileIO;
+  private Configuration conf;
+  private String projectId;
+  private String location;
+  private BigQueryMetastoreClient client;
+  private boolean filterUnsupportedTables;
+
+  // Must have a no-arg constructor to be dynamically loaded
+  // initialize(String name, Map<String, String> properties) will be called to complete
+  // initialization
+  public BigQueryMetastoreCatalog() {}
+
+  @Override
+  public void initialize(String inputName, Map<String, String> properties) {
+    Preconditions.checkArgument(
+        properties.containsKey(PROJECT_ID), "GCP project must be specified");
+
+    projectId = properties.get(PROJECT_ID);
+    location = properties.getOrDefault(GCP_LOCATION, DEFAULT_GCP_LOCATION);
+    boolean testingEnabled =
+        Boolean.parseBoolean(properties.getOrDefault(TESTING_ENABLED, "false"));
+
+    BigQueryOptions options =
+        BigQueryOptions.newBuilder()
+            .setProjectId(projectId)
+            .setLocation(location)
+            .setRetrySettings(ServiceOptions.getDefaultRetrySettings())
+            .build();
+
+    try {
+      if (testingEnabled) {
+        client = new FakeBigQueryMetastoreClient(options);
+
+      } else {
+        client = new BigQueryMetastoreClientImpl(options);
+      }
+    } catch (IOException e) {
+      throw new ServiceFailureException(e, "Creating BigQuery client failed");
+    } catch (GeneralSecurityException e) {
+      throw new ValidationException(e, "Creating BigQuery client failed due to a security issue");
+    }
+    initialize(inputName, properties, projectId, location, client);
+  }
+
+  @VisibleForTesting
+  void initialize(
+      String inputName,
+      Map<String, String> properties,
+      String initialProjectId,
+      String initialLocation,
+      BigQueryMetastoreClient bigQueryMetaStoreClient) {
+    Preconditions.checkArgument(bigQueryMetaStoreClient != null, "BigQuery client can not be null");
+    this.catalogName = inputName;
+    this.catalogProperties = ImmutableMap.copyOf(properties);
+    this.projectId = initialProjectId;
+    this.location = initialLocation;
+    this.client = bigQueryMetaStoreClient;
+
+    if (this.conf == null) {
+      LOG.warn("No configuration was set, using the default environment Configuration");
+      this.conf = new Configuration();
+    }
+
+    LOG.info("Using BigQuery Metastore Iceberg Catalog: {}", inputName);
+
+    if (properties.containsKey(CatalogProperties.WAREHOUSE_LOCATION)) {
+      // Iceberg always removes trailing slash to avoid paths like "<folder>//data" in file systems
+      // like s3.
+      this.conf.set(
+          HIVE_METASTORE_WAREHOUSE_DIR,
+          LocationUtil.stripTrailingSlash(properties.get(CatalogProperties.WAREHOUSE_LOCATION)));
+    }
+
+    String fileIoImpl =
+        properties.getOrDefault(
+            CatalogProperties.FILE_IO_IMPL, "org.apache.iceberg.hadoop.HadoopFileIO");
+    this.fileIO = CatalogUtil.loadFileIO(fileIoImpl, properties, conf);
+
+    this.filterUnsupportedTables =
+        Boolean.parseBoolean(properties.getOrDefault(FILTER_UNSUPPORTED_TABLES, "false"));
+  }
+
+  @Override
+  protected TableOperations newTableOps(TableIdentifier identifier) {
+    return new BigQueryTableOperations(
+        client,
+        fileIO,
+        projectId,
+        // Sometimes extensions have the namespace contain the table name too, so we are forced to
+        // allow invalid namespace and just take the first part here like other catalog
+        // implementations do.
+        identifier.namespace().level(0),
+        identifier.name(),
+        conf);
+  }
+
+  @Override
+  protected String defaultWarehouseLocation(TableIdentifier identifier) {
+    String locationUri = null;
+    DatasetReference datasetReference = toDatasetReference(identifier.namespace());
+    Dataset dataset = client.getDataset(datasetReference);
+    if (dataset != null && dataset.getExternalCatalogDatasetOptions() != null) {
+      locationUri = dataset.getExternalCatalogDatasetOptions().getDefaultStorageLocationUri();
+    }
+    return String.format(
+        "%s/%s",
+        Strings.isNullOrEmpty(locationUri)
+            ? datasetReference.getDatasetId()
+            : LocationUtil.stripTrailingSlash(locationUri),
+        identifier.name());
+  }
+
+  @Override
+  public List<TableIdentifier> listTables(Namespace namespace) {
+    validateNamespace(namespace);
+
+    return client.listTables(toDatasetReference(namespace), filterUnsupportedTables).stream()
+        .map(
+            table -> TableIdentifier.of(namespace.level(0), table.getTableReference().getTableId()))
+        .collect(ImmutableList.toImmutableList());
+  }
+
+  @Override
+  public boolean dropTable(TableIdentifier identifier, boolean purge) {
+    try {
+      TableOperations ops = newTableOps(identifier);
+      TableMetadata lastMetadata = ops.current();
+
+      client.deleteTable(toTableReference(identifier));
+
+      if (purge && lastMetadata != null) {
+        CatalogUtil.dropTableData(ops.io(), lastMetadata);
+      }
+    } catch (NoSuchTableException e) { // Not catching a NoSuchIcebergTableException on purpose
+      return false; // The documentation says just return false in this case
+    }
+    return true;
+  }
+
+  @Override
+  public void renameTable(TableIdentifier from, TableIdentifier to) {
+    validateNamespace(from.namespace());
+    validateNamespace(to.namespace());
+
+    try {
+      client.getDataset(toDatasetReference(to.namespace()));
+      client.getDataset(toDatasetReference(from.namespace()));
+    } catch (NoSuchNamespaceException e) {
+      throw e;
+    }
+
+    TableReference fromTableRef = toTableReference(from);
+    TableReference toTableRef = toTableReference(to);
+
+    // Check if the source table exists
+    try {
+      client.getTable(fromTableRef);
+    } catch (NoSuchTableException e) {
+      throw new NoSuchTableException("Table does not exist: %s", from.name());
+    }
+
+    // Check if the destination table already exists
+    try {
+      client.getTable(toTableRef);
+      // If getTable succeeds, the destination table exists
+      throw new AlreadyExistsException("Table already exists: %s", to.name());
+    } catch (NoSuchTableException e) {
+      // Destination table does not exist, proceed with throwing exception
+    }
+
+    if (!from.namespace().equals(to.namespace())) {
+      throw new ValidationException("New table name must be in the same namespace");
+    }
+
+    // TODO(b/354981675): Enable once supported by the API.
+    throw new UnsupportedOperationException("Table rename operation is unsupported.");
+  }
+
+  @Override
+  public void createNamespace(Namespace namespace, Map<String, String> metadata) {
+    Dataset builder = new Dataset();
+    DatasetReference datasetReference = toDatasetReference(namespace);
+    builder.setLocation(this.location);
+    builder.setDatasetReference(datasetReference);
+    builder.setExternalCatalogDatasetOptions(
+        BigQueryMetastoreUtils.createExternalCatalogDatasetOptions(
+            datasetReference.getDatasetId(), metadata));
+
+    client.createDataset(builder);
+  }
+
+  /**
+   * Since this catalog only supports one-level namespaces, it always returns an empty list unless
+   * passed an empty namespace to list all namespaces within the catalog.
+   */
+  @Override
+  public List<Namespace> listNamespaces(Namespace namespace) {
+    if (namespace.levels().length != 0) {
+      // BQMS does not support namespaces under database or tables, returns empty.
+      // It is called when dropping a namespace to make sure it's empty (listTables is called as
+      // well), returns empty to unblock deletion.
+      return ImmutableList.of();
+    }
+    return client.listDatasets(projectId).stream()
+        .map(BigQueryMetastoreCatalog::getNamespace)
+        .collect(ImmutableList.toImmutableList());
+  }
+
+  @Override
+  public boolean dropNamespace(Namespace namespace) {
+    try {
+      client.deleteDataset(toDatasetReference(namespace));
+      // We don't delete the data folder for safety, which aligns with Hive Metastore's default
+      // behavior.
+      // We can support database or catalog level config controlling file deletion in the future.
+      return true;
+    } catch (NoSuchNamespaceException e) {
+      return false;
+    }
+  }
+
+  @Override
+  public boolean setProperties(Namespace namespace, Map<String, String> properties) {
+    Dataset dataset = client.getDataset(toDatasetReference(namespace));
+
+    ExternalCatalogDatasetOptions existingOptions = dataset.getExternalCatalogDatasetOptions();
+    Map<String, String> existingParameters =
+        existingOptions != null ? existingOptions.getParameters() : null;
+
+    Map<String, String> newParameters = Maps.newHashMap();
+    if (existingParameters != null) {
+      newParameters.putAll(existingParameters);
+    }
+    newParameters.putAll(properties);
+
+    if (Objects.equals(existingParameters, newParameters)) {
+      // No change in parameters detected
+      return false;
+    }
+
+    client.setDatasetParameters(toDatasetReference(namespace), properties);
+    return true;
+  }
+
+  @Override
+  public boolean removeProperties(Namespace namespace, Set<String> properties) {
+    client.removeDatasetParameters(toDatasetReference(namespace), properties);
+    return true;
+  }
+
+  @SuppressWarnings("FormatStringAnnotation")
+  @Override
+  public Map<String, String> loadNamespaceMetadata(Namespace namespace) {
+    try {
+      return getMetadata(client.getDataset(toDatasetReference(namespace)));
+    } catch (IllegalArgumentException e) {
+      throw new NoSuchNamespaceException(e.getMessage());
+    }
+  }
+
+  @Override
+  public String name() {
+    return catalogName;
+  }
+
+  @Override
+  protected Map<String, String> properties() {
+    return catalogProperties == null ? ImmutableMap.of() : catalogProperties;
+  }
+
+  @Override
+  public void setConf(Configuration conf) {
+    this.conf = new Configuration(conf);
+  }
+
+  @Override
+  public Configuration getConf() {
+    return this.conf;
+  }
+
+  private static Namespace getNamespace(Datasets dataset) {
+    return Namespace.of(dataset.getDatasetReference().getDatasetId());
+  }
+
+  private DatasetReference toDatasetReference(Namespace namespace) {
+    validateNamespace(namespace);
+    return new DatasetReference().setProjectId(projectId).setDatasetId(namespace.level(0));
+  }
+
+  private TableReference toTableReference(TableIdentifier tableIdentifier) {
+    DatasetReference datasetReference = toDatasetReference(tableIdentifier.namespace());
+    return new TableReference()
+        .setProjectId(datasetReference.getProjectId())
+        .setDatasetId(datasetReference.getDatasetId())
+        .setTableId(tableIdentifier.name());
+  }
+
+  private static Map<String, String> getMetadata(Dataset dataset) {
+    ExternalCatalogDatasetOptions options = dataset.getExternalCatalogDatasetOptions();
+    Map<String, String> metadata = Maps.newHashMap();
+    if (options != null) {
+      if (options.getParameters() != null) {
+        metadata.putAll(options.getParameters());
+      }
+      if (!Strings.isNullOrEmpty(options.getDefaultStorageLocationUri())) {
+        metadata.put("location", options.getDefaultStorageLocationUri());

Review Comment:
   yes 



-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org

For queries about this service, please contact Infrastructure at:
users@infra.apache.org


---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For additional commands, e-mail: issues-help@iceberg.apache.org


Re: [PR] Catalog: Add BigQuery Metastore Catalog Support [iceberg]

Posted by "nastra (via GitHub)" <gi...@apache.org>.
nastra commented on code in PR #12808:
URL: https://github.com/apache/iceberg/pull/12808#discussion_r2055635017


##########
bigquery/src/main/java/org/apache/iceberg/gcp/bigquery/BigQueryMetastoreCatalog.java:
##########
@@ -0,0 +1,400 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.iceberg.gcp.bigquery;
+
+import com.google.api.services.bigquery.model.Dataset;
+import com.google.api.services.bigquery.model.DatasetList.Datasets;
+import com.google.api.services.bigquery.model.DatasetReference;
+import com.google.api.services.bigquery.model.ExternalCatalogDatasetOptions;
+import com.google.api.services.bigquery.model.TableReference;
+import com.google.cloud.ServiceOptions;
+import com.google.cloud.bigquery.BigQueryOptions;
+import java.io.IOException;
+import java.security.GeneralSecurityException;
+import java.util.List;
+import java.util.Locale;
+import java.util.Map;
+import java.util.Objects;
+import java.util.Set;
+import org.apache.hadoop.conf.Configurable;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.iceberg.BaseMetastoreCatalog;
+import org.apache.iceberg.CatalogProperties;
+import org.apache.iceberg.CatalogUtil;
+import org.apache.iceberg.TableMetadata;
+import org.apache.iceberg.TableOperations;
+import org.apache.iceberg.catalog.Namespace;
+import org.apache.iceberg.catalog.SupportsNamespaces;
+import org.apache.iceberg.catalog.TableIdentifier;
+import org.apache.iceberg.exceptions.AlreadyExistsException;
+import org.apache.iceberg.exceptions.NoSuchNamespaceException;
+import org.apache.iceberg.exceptions.NoSuchTableException;
+import org.apache.iceberg.exceptions.ServiceFailureException;
+import org.apache.iceberg.exceptions.ValidationException;
+import org.apache.iceberg.io.FileIO;
+import org.apache.iceberg.relocated.com.google.common.annotations.VisibleForTesting;
+import org.apache.iceberg.relocated.com.google.common.base.Preconditions;
+import org.apache.iceberg.relocated.com.google.common.base.Strings;
+import org.apache.iceberg.relocated.com.google.common.collect.ImmutableList;
+import org.apache.iceberg.relocated.com.google.common.collect.ImmutableMap;
+import org.apache.iceberg.relocated.com.google.common.collect.Maps;
+import org.apache.iceberg.util.LocationUtil;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/** Iceberg Bigquery Metastore Catalog implementation. */
+public final class BigQueryMetastoreCatalog extends BaseMetastoreCatalog
+    implements SupportsNamespaces, Configurable {
+
+  // User provided properties.
+  public static final String PROJECT_ID = "gcp-project";
+  public static final String GCP_LOCATION = "gcp-location";
+  public static final String FILTER_UNSUPPORTED_TABLES = "filter-unsupported-tables";
+  public static final String TESTING_ENABLED = "testing-enabled";
+
+  public static final String HIVE_METASTORE_WAREHOUSE_DIR = "hive.metastore.warehouse.dir";
+
+  private static final Logger LOG = LoggerFactory.getLogger(BigQueryMetastoreCatalog.class);
+
+  private static final String DEFAULT_GCP_LOCATION = "us";
+
+  private String catalogName;
+  private Map<String, String> catalogProperties;
+  private FileIO fileIO;
+  private Configuration conf;
+  private String projectId;
+  private String location;
+  private BigQueryMetastoreClient client;
+  private boolean filterUnsupportedTables;
+
+  // Must have a no-arg constructor to be dynamically loaded
+  // initialize(String name, Map<String, String> properties) will be called to complete
+  // initialization
+  public BigQueryMetastoreCatalog() {}
+
+  @Override
+  public void initialize(String inputName, Map<String, String> properties) {
+    Preconditions.checkArgument(
+        properties.containsKey(PROJECT_ID), "GCP project must be specified");
+
+    projectId = properties.get(PROJECT_ID);
+    location = properties.getOrDefault(GCP_LOCATION, DEFAULT_GCP_LOCATION);
+    boolean testingEnabled =
+        Boolean.parseBoolean(properties.getOrDefault(TESTING_ENABLED, "false"));
+
+    BigQueryOptions options =
+        BigQueryOptions.newBuilder()
+            .setProjectId(projectId)
+            .setLocation(location)
+            .setRetrySettings(ServiceOptions.getDefaultRetrySettings())
+            .build();
+
+    try {
+      if (testingEnabled) {
+        client = new FakeBigQueryMetastoreClient(options);
+
+      } else {
+        client = new BigQueryMetastoreClientImpl(options);
+      }
+    } catch (IOException e) {
+      throw new ServiceFailureException(e, "Creating BigQuery client failed");
+    } catch (GeneralSecurityException e) {
+      throw new ValidationException(e, "Creating BigQuery client failed due to a security issue");
+    }
+    initialize(inputName, properties, projectId, location, client);
+  }
+
+  @VisibleForTesting
+  void initialize(
+      String inputName,
+      Map<String, String> properties,
+      String initialProjectId,
+      String initialLocation,
+      BigQueryMetastoreClient bigQueryMetaStoreClient) {
+    Preconditions.checkArgument(bigQueryMetaStoreClient != null, "BigQuery client can not be null");
+    this.catalogName = inputName;
+    this.catalogProperties = ImmutableMap.copyOf(properties);
+    this.projectId = initialProjectId;
+    this.location = initialLocation;
+    this.client = bigQueryMetaStoreClient;
+
+    if (this.conf == null) {
+      LOG.warn("No configuration was set, using the default environment Configuration");
+      this.conf = new Configuration();
+    }
+
+    LOG.info("Using BigQuery Metastore Iceberg Catalog: {}", inputName);
+
+    if (properties.containsKey(CatalogProperties.WAREHOUSE_LOCATION)) {
+      // Iceberg always removes trailing slash to avoid paths like "<folder>//data" in file systems
+      // like s3.
+      this.conf.set(
+          HIVE_METASTORE_WAREHOUSE_DIR,
+          LocationUtil.stripTrailingSlash(properties.get(CatalogProperties.WAREHOUSE_LOCATION)));
+    }
+
+    String fileIoImpl =
+        properties.getOrDefault(
+            CatalogProperties.FILE_IO_IMPL, "org.apache.iceberg.hadoop.HadoopFileIO");
+    this.fileIO = CatalogUtil.loadFileIO(fileIoImpl, properties, conf);
+
+    this.filterUnsupportedTables =
+        Boolean.parseBoolean(properties.getOrDefault(FILTER_UNSUPPORTED_TABLES, "false"));
+  }
+
+  @Override
+  protected TableOperations newTableOps(TableIdentifier identifier) {
+    return new BigQueryTableOperations(
+        client,
+        fileIO,
+        projectId,
+        // Sometimes extensions have the namespace contain the table name too, so we are forced to
+        // allow invalid namespace and just take the first part here like other catalog
+        // implementations do.
+        identifier.namespace().level(0),
+        identifier.name(),
+        conf);
+  }
+
+  @Override
+  protected String defaultWarehouseLocation(TableIdentifier identifier) {
+    String locationUri = null;
+    DatasetReference datasetReference = toDatasetReference(identifier.namespace());
+    Dataset dataset = client.getDataset(datasetReference);
+    if (dataset != null && dataset.getExternalCatalogDatasetOptions() != null) {
+      locationUri = dataset.getExternalCatalogDatasetOptions().getDefaultStorageLocationUri();
+    }
+    return String.format(
+        "%s/%s",
+        Strings.isNullOrEmpty(locationUri)
+            ? datasetReference.getDatasetId()
+            : LocationUtil.stripTrailingSlash(locationUri),
+        identifier.name());
+  }
+
+  @Override
+  public List<TableIdentifier> listTables(Namespace namespace) {
+    validateNamespace(namespace);
+
+    return client.listTables(toDatasetReference(namespace), filterUnsupportedTables).stream()
+        .map(
+            table -> TableIdentifier.of(namespace.level(0), table.getTableReference().getTableId()))
+        .collect(ImmutableList.toImmutableList());
+  }
+
+  @Override
+  public boolean dropTable(TableIdentifier identifier, boolean purge) {
+    try {
+      TableOperations ops = newTableOps(identifier);
+      TableMetadata lastMetadata = ops.current();
+
+      client.deleteTable(toTableReference(identifier));
+
+      if (purge && lastMetadata != null) {
+        CatalogUtil.dropTableData(ops.io(), lastMetadata);
+      }
+    } catch (NoSuchTableException e) { // Not catching a NoSuchIcebergTableException on purpose
+      return false; // The documentation says just return false in this case
+    }
+    return true;
+  }
+
+  @Override
+  public void renameTable(TableIdentifier from, TableIdentifier to) {
+    validateNamespace(from.namespace());
+    validateNamespace(to.namespace());
+
+    try {
+      client.getDataset(toDatasetReference(to.namespace()));
+      client.getDataset(toDatasetReference(from.namespace()));
+    } catch (NoSuchNamespaceException e) {
+      throw e;
+    }
+
+    TableReference fromTableRef = toTableReference(from);
+    TableReference toTableRef = toTableReference(to);
+
+    // Check if the source table exists
+    try {
+      client.getTable(fromTableRef);
+    } catch (NoSuchTableException e) {
+      throw new NoSuchTableException("Table does not exist: %s", from.name());
+    }
+
+    // Check if the destination table already exists
+    try {
+      client.getTable(toTableRef);
+      // If getTable succeeds, the destination table exists
+      throw new AlreadyExistsException("Table already exists: %s", to.name());
+    } catch (NoSuchTableException e) {
+      // Destination table does not exist, proceed with throwing exception
+    }
+
+    if (!from.namespace().equals(to.namespace())) {
+      throw new ValidationException("New table name must be in the same namespace");
+    }
+
+    // TODO(b/354981675): Enable once supported by the API.
+    throw new UnsupportedOperationException("Table rename operation is unsupported.");
+  }
+
+  @Override
+  public void createNamespace(Namespace namespace, Map<String, String> metadata) {
+    Dataset builder = new Dataset();
+    DatasetReference datasetReference = toDatasetReference(namespace);
+    builder.setLocation(this.location);
+    builder.setDatasetReference(datasetReference);
+    builder.setExternalCatalogDatasetOptions(
+        BigQueryMetastoreUtils.createExternalCatalogDatasetOptions(
+            datasetReference.getDatasetId(), metadata));
+
+    client.createDataset(builder);
+  }
+
+  /**
+   * Since this catalog only supports one-level namespaces, it always returns an empty list unless
+   * passed an empty namespace to list all namespaces within the catalog.
+   */
+  @Override
+  public List<Namespace> listNamespaces(Namespace namespace) {
+    if (namespace.levels().length != 0) {
+      // BQMS does not support namespaces under database or tables, returns empty.
+      // It is called when dropping a namespace to make sure it's empty (listTables is called as
+      // well), returns empty to unblock deletion.
+      return ImmutableList.of();
+    }
+    return client.listDatasets(projectId).stream()
+        .map(BigQueryMetastoreCatalog::getNamespace)
+        .collect(ImmutableList.toImmutableList());
+  }
+
+  @Override
+  public boolean dropNamespace(Namespace namespace) {
+    try {
+      client.deleteDataset(toDatasetReference(namespace));
+      // We don't delete the data folder for safety, which aligns with Hive Metastore's default
+      // behavior.
+      // We can support database or catalog level config controlling file deletion in the future.
+      return true;
+    } catch (NoSuchNamespaceException e) {
+      return false;
+    }
+  }
+
+  @Override
+  public boolean setProperties(Namespace namespace, Map<String, String> properties) {
+    Dataset dataset = client.getDataset(toDatasetReference(namespace));
+
+    ExternalCatalogDatasetOptions existingOptions = dataset.getExternalCatalogDatasetOptions();
+    Map<String, String> existingParameters =
+        existingOptions != null ? existingOptions.getParameters() : null;
+
+    Map<String, String> newParameters = Maps.newHashMap();
+    if (existingParameters != null) {
+      newParameters.putAll(existingParameters);
+    }
+    newParameters.putAll(properties);
+
+    if (Objects.equals(existingParameters, newParameters)) {
+      // No change in parameters detected
+      return false;
+    }
+
+    client.setDatasetParameters(toDatasetReference(namespace), properties);
+    return true;
+  }
+
+  @Override
+  public boolean removeProperties(Namespace namespace, Set<String> properties) {
+    client.removeDatasetParameters(toDatasetReference(namespace), properties);
+    return true;
+  }
+
+  @SuppressWarnings("FormatStringAnnotation")
+  @Override
+  public Map<String, String> loadNamespaceMetadata(Namespace namespace) {
+    try {
+      return getMetadata(client.getDataset(toDatasetReference(namespace)));
+    } catch (IllegalArgumentException e) {
+      throw new NoSuchNamespaceException(e.getMessage());
+    }
+  }
+
+  @Override
+  public String name() {
+    return catalogName;
+  }
+
+  @Override
+  protected Map<String, String> properties() {
+    return catalogProperties == null ? ImmutableMap.of() : catalogProperties;
+  }
+
+  @Override
+  public void setConf(Configuration conf) {
+    this.conf = new Configuration(conf);
+  }
+
+  @Override
+  public Configuration getConf() {
+    return this.conf;
+  }
+
+  private static Namespace getNamespace(Datasets dataset) {

Review Comment:
   ```suggestion
     privateNamespace toNamespace(Datasets dataset) {
   ```



-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org

For queries about this service, please contact Infrastructure at:
users@infra.apache.org


---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For additional commands, e-mail: issues-help@iceberg.apache.org


Re: [PR] Catalog: Add BigQuery Metastore Catalog Support [iceberg]

Posted by "nastra (via GitHub)" <gi...@apache.org>.
nastra commented on code in PR #12808:
URL: https://github.com/apache/iceberg/pull/12808#discussion_r2054374124


##########
bigquery/src/main/java/org/apache/iceberg/gcp/bigquery/BigQueryMetastoreCatalog.java:
##########
@@ -0,0 +1,402 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.iceberg.gcp.bigquery;
+
+import com.google.api.services.bigquery.model.Dataset;
+import com.google.api.services.bigquery.model.DatasetList.Datasets;
+import com.google.api.services.bigquery.model.DatasetReference;
+import com.google.api.services.bigquery.model.ExternalCatalogDatasetOptions;
+import com.google.api.services.bigquery.model.TableReference;
+import com.google.cloud.ServiceOptions;
+import com.google.cloud.bigquery.BigQueryOptions;
+import java.io.IOException;
+import java.security.GeneralSecurityException;
+import java.util.List;
+import java.util.Locale;
+import java.util.Map;
+import java.util.Objects;
+import java.util.Set;
+import org.apache.hadoop.conf.Configurable;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.iceberg.BaseMetastoreCatalog;
+import org.apache.iceberg.CatalogProperties;
+import org.apache.iceberg.CatalogUtil;
+import org.apache.iceberg.TableMetadata;
+import org.apache.iceberg.TableOperations;
+import org.apache.iceberg.catalog.Namespace;
+import org.apache.iceberg.catalog.SupportsNamespaces;
+import org.apache.iceberg.catalog.TableIdentifier;
+import org.apache.iceberg.exceptions.AlreadyExistsException;
+import org.apache.iceberg.exceptions.NoSuchNamespaceException;
+import org.apache.iceberg.exceptions.NoSuchTableException;
+import org.apache.iceberg.exceptions.ServiceFailureException;
+import org.apache.iceberg.exceptions.ValidationException;
+import org.apache.iceberg.io.FileIO;
+import org.apache.iceberg.relocated.com.google.common.annotations.VisibleForTesting;
+import org.apache.iceberg.relocated.com.google.common.base.Preconditions;
+import org.apache.iceberg.relocated.com.google.common.base.Strings;
+import org.apache.iceberg.relocated.com.google.common.collect.ImmutableList;
+import org.apache.iceberg.relocated.com.google.common.collect.ImmutableMap;
+import org.apache.iceberg.relocated.com.google.common.collect.Maps;
+import org.apache.iceberg.util.LocationUtil;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/** Iceberg Bigquery Metastore Catalog implementation. */
+public final class BigQueryMetastoreCatalog extends BaseMetastoreCatalog
+    implements SupportsNamespaces, Configurable {
+
+  // User provided properties.
+  public static final String PROPERTIES_KEY_GCP_PROJECT = "gcp-project";
+  public static final String PROPERTIES_KEY_GCP_LOCATION = "gcp-location";
+  public static final String PROPERTIES_KEY_FILTER_UNSUPPORTED_TABLES = "filter-unsupported-tables";
+  public static final String PROPERTIES_KEY_TESTING_ENABLED = "testing-enabled";
+
+  public static final String HIVE_METASTORE_WAREHOUSE_DIR = "hive.metastore.warehouse.dir";
+
+  private static final Logger LOG = LoggerFactory.getLogger(BigQueryMetastoreCatalog.class);
+
+  private static final String DEFAULT_GCP_LOCATION = "us";
+
+  private String catalogPluginName;
+  private Map<String, String> catalogProperties;
+  private FileIO fileIO;
+  private Configuration conf;
+  private String projectId;
+  private String location;
+  private BigQueryMetastoreClient client;
+  private boolean filterUnsupportedTables;
+
+  // Must have a no-arg constructor to be dynamically loaded
+  // initialize(String name, Map<String, String> properties) will be called to complete
+  // initialization
+  public BigQueryMetastoreCatalog() {}
+
+  @Override
+  public void initialize(String inputName, Map<String, String> properties) {
+    if (!properties.containsKey(PROPERTIES_KEY_GCP_PROJECT)) {
+      throw new ValidationException("GCP project must be specified");
+    }
+    projectId = properties.get(PROPERTIES_KEY_GCP_PROJECT);
+    location = properties.getOrDefault(PROPERTIES_KEY_GCP_LOCATION, DEFAULT_GCP_LOCATION);
+    boolean testingEnabled =
+        Boolean.parseBoolean(properties.getOrDefault(PROPERTIES_KEY_TESTING_ENABLED, "false"));
+
+    BigQueryOptions options =
+        BigQueryOptions.newBuilder()
+            .setProjectId(projectId)
+            .setLocation(location)
+            .setRetrySettings(ServiceOptions.getDefaultRetrySettings())
+            .build();
+
+    try {
+      if (testingEnabled) {
+        client = new FakeBigQueryMetastoreClient(options);
+
+      } else {
+        client = new BigQueryMetastoreClientImpl(options);
+      }
+    } catch (IOException e) {
+      throw new ServiceFailureException(e, "Creating BigQuery client failed");
+    } catch (GeneralSecurityException e) {
+      throw new ValidationException(e, "Creating BigQuery client failed due to a security issue");
+    }
+    initialize(inputName, properties, projectId, location, client);
+  }
+
+  @VisibleForTesting
+  void initialize(
+      String inputName,
+      Map<String, String> properties,
+      String initialProjectId,
+      String initialLocation,
+      BigQueryMetastoreClient bigQueryMetaStoreClient) {
+    this.catalogPluginName = inputName;
+    this.catalogProperties = ImmutableMap.copyOf(properties);
+    this.projectId = initialProjectId;
+    this.location = initialLocation;
+    this.client =
+        Preconditions.checkNotNull(bigQueryMetaStoreClient, "BigQuery client can not be null");

Review Comment:
   IMO it's better to use `Preconditions.checkArgument(null != ..., "Invalid BigQuery client: null")` because it indicates that a wrong argument was passed in a clearer way than a NPE



-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org

For queries about this service, please contact Infrastructure at:
users@infra.apache.org


---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For additional commands, e-mail: issues-help@iceberg.apache.org


Re: [PR] Catalog: Add BigQuery Metastore Catalog Support [iceberg]

Posted by "nastra (via GitHub)" <gi...@apache.org>.
nastra commented on code in PR #12808:
URL: https://github.com/apache/iceberg/pull/12808#discussion_r2054371218


##########
bigquery/src/main/java/org/apache/iceberg/gcp/bigquery/BigQueryMetastoreCatalog.java:
##########
@@ -0,0 +1,402 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.iceberg.gcp.bigquery;
+
+import com.google.api.services.bigquery.model.Dataset;
+import com.google.api.services.bigquery.model.DatasetList.Datasets;
+import com.google.api.services.bigquery.model.DatasetReference;
+import com.google.api.services.bigquery.model.ExternalCatalogDatasetOptions;
+import com.google.api.services.bigquery.model.TableReference;
+import com.google.cloud.ServiceOptions;
+import com.google.cloud.bigquery.BigQueryOptions;
+import java.io.IOException;
+import java.security.GeneralSecurityException;
+import java.util.List;
+import java.util.Locale;
+import java.util.Map;
+import java.util.Objects;
+import java.util.Set;
+import org.apache.hadoop.conf.Configurable;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.iceberg.BaseMetastoreCatalog;
+import org.apache.iceberg.CatalogProperties;
+import org.apache.iceberg.CatalogUtil;
+import org.apache.iceberg.TableMetadata;
+import org.apache.iceberg.TableOperations;
+import org.apache.iceberg.catalog.Namespace;
+import org.apache.iceberg.catalog.SupportsNamespaces;
+import org.apache.iceberg.catalog.TableIdentifier;
+import org.apache.iceberg.exceptions.AlreadyExistsException;
+import org.apache.iceberg.exceptions.NoSuchNamespaceException;
+import org.apache.iceberg.exceptions.NoSuchTableException;
+import org.apache.iceberg.exceptions.ServiceFailureException;
+import org.apache.iceberg.exceptions.ValidationException;
+import org.apache.iceberg.io.FileIO;
+import org.apache.iceberg.relocated.com.google.common.annotations.VisibleForTesting;
+import org.apache.iceberg.relocated.com.google.common.base.Preconditions;
+import org.apache.iceberg.relocated.com.google.common.base.Strings;
+import org.apache.iceberg.relocated.com.google.common.collect.ImmutableList;
+import org.apache.iceberg.relocated.com.google.common.collect.ImmutableMap;
+import org.apache.iceberg.relocated.com.google.common.collect.Maps;
+import org.apache.iceberg.util.LocationUtil;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/** Iceberg Bigquery Metastore Catalog implementation. */
+public final class BigQueryMetastoreCatalog extends BaseMetastoreCatalog
+    implements SupportsNamespaces, Configurable {
+
+  // User provided properties.
+  public static final String PROPERTIES_KEY_GCP_PROJECT = "gcp-project";
+  public static final String PROPERTIES_KEY_GCP_LOCATION = "gcp-location";
+  public static final String PROPERTIES_KEY_FILTER_UNSUPPORTED_TABLES = "filter-unsupported-tables";
+  public static final String PROPERTIES_KEY_TESTING_ENABLED = "testing-enabled";

Review Comment:
   does it make sense to add `testing-enabled` and `filter-unsupported-tables` to the Iceberg codebase? I would suggest that we find a way on how we can prevent introducing those properties



-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org

For queries about this service, please contact Infrastructure at:
users@infra.apache.org


---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For additional commands, e-mail: issues-help@iceberg.apache.org


Re: [PR] Catalog: Add BigQuery Metastore Catalog Support [iceberg]

Posted by "talatuyarer (via GitHub)" <gi...@apache.org>.
talatuyarer commented on code in PR #12808:
URL: https://github.com/apache/iceberg/pull/12808#discussion_r2060849882


##########
bigquery/src/test/java/org/apache/iceberg/gcp/bigquery/BigQueryMetastoreTestUtils.java:
##########
@@ -0,0 +1,87 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.iceberg.gcp.bigquery;
+
+import static org.apache.iceberg.types.Types.NestedField.required;
+import static org.assertj.core.api.Assertions.assertThat;
+
+import com.google.api.services.bigquery.model.ExternalCatalogTableOptions;
+import com.google.api.services.bigquery.model.StorageDescriptor;
+import com.google.api.services.bigquery.model.Table;
+import com.google.api.services.bigquery.model.TableReference;
+import java.io.File;
+import java.io.IOException;
+import java.util.Collections;
+import java.util.Optional;
+import org.apache.commons.io.FileUtils;
+import org.apache.commons.io.filefilter.TrueFileFilter;
+import org.apache.iceberg.Schema;
+import org.apache.iceberg.catalog.TableIdentifier;
+import org.apache.iceberg.types.Types;
+
+/** Test utility methods for BigQuery Metastore Iceberg catalog. */
+public final class BigQueryMetastoreTestUtils {
+  private BigQueryMetastoreTestUtils() {}
+
+  public static final String METADATA_LOCATION_PROP = "metadata_location";
+
+  public static Schema getTestSchema() {
+    return new Schema(
+        required(1, "id", Types.IntegerType.get(), "unique ID"),
+        required(2, "data", Types.StringType.get()));
+  }
+
+  public static Table createTestTable(
+      File tempFolder,
+      BigQueryMetastoreCatalog bigQueryMetastoreCatalog,
+      TableReference tableReference)
+      throws IOException {
+    Schema schema = getTestSchema();
+    TableIdentifier tableIdentifier =
+        TableIdentifier.of(tableReference.getDatasetId(), tableReference.getTableId());
+    String tableDir = tempFolder.toPath().resolve(tableReference.getTableId()).toString();
+
+    bigQueryMetastoreCatalog
+        .buildTable(tableIdentifier, schema)
+        .withLocation(tableDir)
+        .createTransaction()
+        .commitTransaction();
+
+    Optional<String> metadataLocation = getIcebergMetadataFilePath(tableDir);
+    assertThat(metadataLocation.isPresent());

Review Comment:
   Thank you Done.



-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org

For queries about this service, please contact Infrastructure at:
users@infra.apache.org


---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For additional commands, e-mail: issues-help@iceberg.apache.org


Re: [PR] Catalog: Add BigQuery Metastore Catalog Support [iceberg]

Posted by "danielcweeks (via GitHub)" <gi...@apache.org>.
danielcweeks commented on code in PR #12808:
URL: https://github.com/apache/iceberg/pull/12808#discussion_r2067560229


##########
bigquery/src/main/java/org/apache/iceberg/gcp/bigquery/BigQueryMetastoreCatalog.java:
##########
@@ -0,0 +1,405 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.iceberg.gcp.bigquery;
+
+import com.google.api.services.bigquery.model.Dataset;
+import com.google.api.services.bigquery.model.DatasetList.Datasets;
+import com.google.api.services.bigquery.model.DatasetReference;
+import com.google.api.services.bigquery.model.ExternalCatalogDatasetOptions;
+import com.google.api.services.bigquery.model.TableReference;
+import com.google.cloud.ServiceOptions;
+import com.google.cloud.bigquery.BigQueryOptions;
+import java.io.IOException;
+import java.security.GeneralSecurityException;
+import java.util.List;
+import java.util.Locale;
+import java.util.Map;
+import java.util.Objects;
+import java.util.Set;
+import org.apache.hadoop.conf.Configurable;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.iceberg.BaseMetastoreCatalog;
+import org.apache.iceberg.CatalogProperties;
+import org.apache.iceberg.CatalogUtil;
+import org.apache.iceberg.TableMetadata;
+import org.apache.iceberg.TableOperations;
+import org.apache.iceberg.catalog.Namespace;
+import org.apache.iceberg.catalog.SupportsNamespaces;
+import org.apache.iceberg.catalog.TableIdentifier;
+import org.apache.iceberg.exceptions.AlreadyExistsException;
+import org.apache.iceberg.exceptions.NoSuchNamespaceException;
+import org.apache.iceberg.exceptions.NoSuchTableException;
+import org.apache.iceberg.exceptions.ServiceFailureException;
+import org.apache.iceberg.exceptions.ValidationException;
+import org.apache.iceberg.io.FileIO;
+import org.apache.iceberg.relocated.com.google.common.annotations.VisibleForTesting;
+import org.apache.iceberg.relocated.com.google.common.base.Preconditions;
+import org.apache.iceberg.relocated.com.google.common.base.Strings;
+import org.apache.iceberg.relocated.com.google.common.collect.ImmutableList;
+import org.apache.iceberg.relocated.com.google.common.collect.ImmutableMap;
+import org.apache.iceberg.relocated.com.google.common.collect.Maps;
+import org.apache.iceberg.util.LocationUtil;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/** Iceberg Bigquery Metastore Catalog implementation. */
+public class BigQueryMetastoreCatalog extends BaseMetastoreCatalog
+    implements SupportsNamespaces, Configurable {
+
+  // User provided properties.
+  public static final String PROJECT_ID = "gcp-project";
+  public static final String GCP_LOCATION = "gcp-location";
+  public static final String LIST_ALL_TABLES = "list-all-tables";
+
+  public static final String HIVE_METASTORE_WAREHOUSE_DIR = "hive.metastore.warehouse.dir";
+
+  private static final Logger LOG = LoggerFactory.getLogger(BigQueryMetastoreCatalog.class);
+
+  private static final String DEFAULT_GCP_LOCATION = "us";
+
+  private String catalogName;
+  private Map<String, String> catalogProperties;
+  private FileIO fileIO;
+  private Configuration conf;
+  private String projectId;
+  private String location;
+  private BigQueryMetastoreClient client;
+  private boolean listAllTables;
+
+  // Must have a no-arg constructor to be dynamically loaded
+  // initialize(String name, Map<String, String> properties) will be called to complete
+  // initialization
+  public BigQueryMetastoreCatalog() {}
+
+  @Override
+  public void initialize(String inputName, Map<String, String> properties) {
+    Preconditions.checkArgument(
+        properties.containsKey(PROJECT_ID),
+        "Invalid GCP project: %s must be specified",
+        PROJECT_ID);
+
+    projectId = properties.get(PROJECT_ID);
+    location = properties.getOrDefault(GCP_LOCATION, DEFAULT_GCP_LOCATION);
+
+    BigQueryOptions options =
+        BigQueryOptions.newBuilder()
+            .setProjectId(projectId)
+            .setLocation(location)
+            .setRetrySettings(ServiceOptions.getDefaultRetrySettings())
+            .build();
+
+    try {
+      client = new BigQueryMetastoreClientImpl(options);
+    } catch (IOException e) {
+      throw new ServiceFailureException(e, "Creating BigQuery client failed");
+    } catch (GeneralSecurityException e) {
+      throw new ValidationException(e, "Creating BigQuery client failed due to a security issue");
+    }
+
+    initialize(inputName, properties, projectId, location, client);
+  }
+
+  @VisibleForTesting
+  void initialize(
+      String inputName,
+      Map<String, String> properties,
+      String initialProjectId,
+      String initialLocation,
+      BigQueryMetastoreClient bigQueryMetaStoreClient) {
+    Preconditions.checkArgument(bigQueryMetaStoreClient != null, "Invalid BigQuery client: null");
+    this.catalogName = inputName;
+    this.catalogProperties = ImmutableMap.copyOf(properties);
+    this.projectId = initialProjectId;
+    this.location = initialLocation;
+    this.client = bigQueryMetaStoreClient;
+
+    if (this.conf == null) {
+      LOG.warn("No configuration was set, using the default environment Configuration");
+      this.conf = new Configuration();
+    }
+
+    LOG.info("Using BigQuery Metastore Iceberg Catalog: {}", inputName);
+
+    if (properties.containsKey(CatalogProperties.WAREHOUSE_LOCATION)) {
+      this.conf.set(
+          HIVE_METASTORE_WAREHOUSE_DIR,
+          LocationUtil.stripTrailingSlash(properties.get(CatalogProperties.WAREHOUSE_LOCATION)));
+    }
+
+    String fileIoImpl =
+        properties.getOrDefault(
+            CatalogProperties.FILE_IO_IMPL, "org.apache.iceberg.hadoop.HadoopFileIO");
+    this.fileIO = CatalogUtil.loadFileIO(fileIoImpl, properties, conf);
+
+    this.listAllTables = Boolean.parseBoolean(properties.getOrDefault(LIST_ALL_TABLES, "true"));
+  }
+
+  @Override
+  protected TableOperations newTableOps(TableIdentifier identifier) {
+    return new BigQueryTableOperations(client, fileIO, toTableReference(identifier), conf);
+  }
+
+  @Override
+  protected String defaultWarehouseLocation(TableIdentifier identifier) {
+    String locationUri = null;
+    DatasetReference datasetReference = toDatasetReference(identifier.namespace());
+    Dataset dataset = client.load(datasetReference);
+    if (dataset != null && dataset.getExternalCatalogDatasetOptions() != null) {
+      locationUri = dataset.getExternalCatalogDatasetOptions().getDefaultStorageLocationUri();
+    }
+
+    return String.format(
+        "%s/%s",
+        Strings.isNullOrEmpty(locationUri)
+            ? createDefaultStorageLocationUri(datasetReference.getDatasetId())
+            : LocationUtil.stripTrailingSlash(locationUri),
+        identifier.name());
+  }
+
+  @Override
+  public List<TableIdentifier> listTables(Namespace namespace) {
+    validateNamespace(namespace);
+
+    return client.list(toDatasetReference(namespace), listAllTables).stream()
+        .map(
+            table -> TableIdentifier.of(namespace.level(0), table.getTableReference().getTableId()))
+        .collect(ImmutableList.toImmutableList());
+  }
+
+  @Override
+  public boolean dropTable(TableIdentifier identifier, boolean purge) {
+    try {
+      TableOperations ops = newTableOps(identifier);
+      TableMetadata lastMetadata = ops.current();
+
+      client.delete(toTableReference(identifier));
+
+      if (purge && lastMetadata != null) {
+        CatalogUtil.dropTableData(ops.io(), lastMetadata);
+      }
+    } catch (NoSuchTableException e) {
+      return false;
+    }
+
+    return true;
+  }
+
+  @Override
+  public void renameTable(TableIdentifier from, TableIdentifier to) {
+    validateNamespace(from.namespace());
+    validateNamespace(to.namespace());
+
+    if (!namespaceExists(to.namespace())) {
+      throw new NoSuchNamespaceException(
+          "Cannot rename %s to %s. Namespace does not exist: %s",
+          from.name(), to.name(), to.namespace());
+    }
+
+    if (!namespaceExists(from.namespace())) {
+      throw new NoSuchNamespaceException(
+          "Cannot rename %s to %s. Namespace does not exist: %s",
+          from.name(), to.name(), from.namespace());
+    }
+
+    if (!tableExists(from)) {
+      throw new NoSuchTableException(
+          "Cannot rename %s to %s. Table does not exist: %s", from.name(), to.name(), from.name());
+    }
+
+    if (!from.namespace().equals(to.namespace())) {
+      throw new ValidationException(
+          "Cannot rename %s to %s. New table name must be in the same namespace",
+          from.name(), to.name());
+    }
+
+    if (tableExists(to)) {
+      throw new AlreadyExistsException(
+          "Cannot rename %s to %s. Table already exists: %s", from.name(), to.name(), to.name());
+    }
+
+    // TODO: Enable once supported by BigQuery API.
+    throw new UnsupportedOperationException("Table rename operation is unsupported.");
+  }
+
+  @Override
+  public void createNamespace(Namespace namespace, Map<String, String> metadata) {
+    Dataset builder = new Dataset();
+    DatasetReference datasetReference = toDatasetReference(namespace);
+    builder.setLocation(this.location);
+    builder.setDatasetReference(datasetReference);
+    builder.setExternalCatalogDatasetOptions(
+        BigQueryMetastoreUtils.createExternalCatalogDatasetOptions(
+            createDefaultStorageLocationUri(datasetReference.getDatasetId()), metadata));
+
+    client.create(builder);
+  }
+
+  @Override
+  public List<Namespace> listNamespaces() {
+    try {
+      return listNamespaces(Namespace.empty());
+    } catch (NoSuchNamespaceException e) {
+      return ImmutableList.of();
+    }
+  }
+
+  /**
+   * Since this catalog only supports one-level namespaces, it always returns an empty list unless
+   * passed an empty namespace to list all namespaces within the catalog.
+   */
+  @Override
+  public List<Namespace> listNamespaces(Namespace namespace) {
+    List<Datasets> allDatasets = client.list(projectId);
+
+    ImmutableList<Namespace> namespaces =
+        allDatasets.stream()
+            .map(BigQueryMetastoreCatalog::toNamespace)
+            .collect(ImmutableList.toImmutableList());
+    if (namespaces.isEmpty()) {
+      throw new NoSuchNamespaceException("Namespace does not exist: %s", namespace);
+    }
+
+    return namespaces;
+  }
+
+  @Override
+  public boolean dropNamespace(Namespace namespace) {
+    try {
+      client.delete(toDatasetReference(namespace));
+      // We don't delete the data folder for safety, which aligns with Hive Metastore's default
+      // behavior.
+      // We can support database or catalog level config controlling file deletion in the future.
+      return true;
+    } catch (NoSuchNamespaceException e) {
+      return false;
+    }
+  }
+
+  @Override
+  public boolean setProperties(Namespace namespace, Map<String, String> properties) {
+    Dataset dataset = client.load(toDatasetReference(namespace));
+
+    ExternalCatalogDatasetOptions existingOptions = dataset.getExternalCatalogDatasetOptions();
+    Map<String, String> existingParameters =
+        existingOptions != null ? existingOptions.getParameters() : null;
+
+    Map<String, String> newParameters = Maps.newHashMap();
+    if (existingParameters != null) {
+      newParameters.putAll(existingParameters);
+    }
+
+    newParameters.putAll(properties);
+
+    if (Objects.equals(existingParameters, newParameters)) {
+      // No change in parameters detected
+      return false;
+    }
+
+    client.setParameters(toDatasetReference(namespace), properties);
+    return true;
+  }
+
+  @Override
+  public boolean removeProperties(Namespace namespace, Set<String> properties) {
+    client.removeParameters(toDatasetReference(namespace), properties);
+    return true;
+  }
+
+  @SuppressWarnings("FormatStringAnnotation")
+  @Override
+  public Map<String, String> loadNamespaceMetadata(Namespace namespace) {
+    try {
+      return toMetadata(client.load(toDatasetReference(namespace)));
+    } catch (IllegalArgumentException e) {
+      throw new NoSuchNamespaceException(e.getMessage());
+    }
+  }
+
+  @Override
+  public String name() {
+    return catalogName;
+  }
+
+  @Override
+  protected Map<String, String> properties() {
+    return catalogProperties == null ? ImmutableMap.of() : catalogProperties;
+  }
+
+  @Override
+  public void setConf(Configuration conf) {
+    this.conf = new Configuration(conf);
+  }
+
+  @Override
+  public Configuration getConf() {
+    return this.conf;
+  }
+
+  private String createDefaultStorageLocationUri(String dbId) {

Review Comment:
   are there any limitations for the dbId that we need to validate (e.g. disallowed characters/etc)?



-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org

For queries about this service, please contact Infrastructure at:
users@infra.apache.org


---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For additional commands, e-mail: issues-help@iceberg.apache.org


Re: [PR] Catalog: Add BigQuery Metastore Catalog Support [iceberg]

Posted by "nastra (via GitHub)" <gi...@apache.org>.
nastra commented on code in PR #12808:
URL: https://github.com/apache/iceberg/pull/12808#discussion_r2065996484


##########
bigquery/src/main/java/org/apache/iceberg/gcp/bigquery/BigQueryMetastoreCatalog.java:
##########
@@ -0,0 +1,405 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.iceberg.gcp.bigquery;
+
+import com.google.api.services.bigquery.model.Dataset;
+import com.google.api.services.bigquery.model.DatasetList.Datasets;
+import com.google.api.services.bigquery.model.DatasetReference;
+import com.google.api.services.bigquery.model.ExternalCatalogDatasetOptions;
+import com.google.api.services.bigquery.model.TableReference;
+import com.google.cloud.ServiceOptions;
+import com.google.cloud.bigquery.BigQueryOptions;
+import java.io.IOException;
+import java.security.GeneralSecurityException;
+import java.util.List;
+import java.util.Locale;
+import java.util.Map;
+import java.util.Objects;
+import java.util.Set;
+import org.apache.hadoop.conf.Configurable;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.iceberg.BaseMetastoreCatalog;
+import org.apache.iceberg.CatalogProperties;
+import org.apache.iceberg.CatalogUtil;
+import org.apache.iceberg.TableMetadata;
+import org.apache.iceberg.TableOperations;
+import org.apache.iceberg.catalog.Namespace;
+import org.apache.iceberg.catalog.SupportsNamespaces;
+import org.apache.iceberg.catalog.TableIdentifier;
+import org.apache.iceberg.exceptions.AlreadyExistsException;
+import org.apache.iceberg.exceptions.NoSuchNamespaceException;
+import org.apache.iceberg.exceptions.NoSuchTableException;
+import org.apache.iceberg.exceptions.ServiceFailureException;
+import org.apache.iceberg.exceptions.ValidationException;
+import org.apache.iceberg.io.FileIO;
+import org.apache.iceberg.relocated.com.google.common.annotations.VisibleForTesting;
+import org.apache.iceberg.relocated.com.google.common.base.Preconditions;
+import org.apache.iceberg.relocated.com.google.common.base.Strings;
+import org.apache.iceberg.relocated.com.google.common.collect.ImmutableList;
+import org.apache.iceberg.relocated.com.google.common.collect.ImmutableMap;
+import org.apache.iceberg.relocated.com.google.common.collect.Maps;
+import org.apache.iceberg.util.LocationUtil;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/** Iceberg Bigquery Metastore Catalog implementation. */
+public class BigQueryMetastoreCatalog extends BaseMetastoreCatalog
+    implements SupportsNamespaces, Configurable {
+
+  // User provided properties.
+  public static final String PROJECT_ID = "gcp-project";
+  public static final String GCP_LOCATION = "gcp-location";
+  public static final String LIST_ALL_TABLES = "list-all-tables";
+
+  public static final String HIVE_METASTORE_WAREHOUSE_DIR = "hive.metastore.warehouse.dir";

Review Comment:
   What do we generally expect users to set. Is it `CatalogProperties.WAREHOUSE_LOCATION` or `HIVE_METASTORE_WAREHOUSE_DIR`? I'm aware that this is being done here similarly to how it's done in `HiveCatalog`but if the warehouse location isn't set, would we tell (via an error msg) to set it via `CatalogProperties.WAREHOUSE_LOCATION` or via `HIVE_METASTORE_WAREHOUSE_DIR`? /cc @danielcweeks 



-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org

For queries about this service, please contact Infrastructure at:
users@infra.apache.org


---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For additional commands, e-mail: issues-help@iceberg.apache.org


Re: [PR] Catalog: Add BigQuery Metastore Catalog Support [iceberg]

Posted by "talatuyarer (via GitHub)" <gi...@apache.org>.
talatuyarer commented on code in PR #12808:
URL: https://github.com/apache/iceberg/pull/12808#discussion_r2081049542


##########
bigquery/src/main/java/org/apache/iceberg/gcp/bigquery/BigQueryMetastoreUtils.java:
##########
@@ -0,0 +1,95 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.iceberg.gcp.bigquery;
+
+import com.google.api.services.bigquery.model.ExternalCatalogDatasetOptions;
+import com.google.api.services.bigquery.model.ExternalCatalogTableOptions;
+import com.google.api.services.bigquery.model.SerDeInfo;
+import com.google.api.services.bigquery.model.StorageDescriptor;
+import java.util.Map;
+
+/** Shared utilities for BigQuery Metastore specific functions and constants. */
+public final class BigQueryMetastoreUtils {
+
+  private BigQueryMetastoreUtils() {}
+
+  private static final String HIVE_SERIALIZATION_LIBRARY =
+      "org.apache.iceberg.mr.hive.HiveIcebergSerDe";
+  private static final String HIVE_FILE_INPUT_FORMAT =
+      "org.apache.iceberg.mr.hive.HiveIcebergInputFormat";
+  private static final String HIVE_FILE_OUTPUT_FORMAT =
+      "org.apache.iceberg.mr.hive.HiveIcebergOutputFormat";
+
+  private static final String SERIALIZATION_LIBRARY =
+      "org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe";
+  private static final String FILE_INPUT_FORMAT = "org.apache.hadoop.mapred.FileInputFormat";
+  private static final String FILE_OUTPUT_FORMAT = "org.apache.hadoop.mapred.FileOutputFormat";
+
+  /**
+   * Creates a new ExternalCatalogTableOptions object populated with the supported library constants
+   * and parameters given.
+   *
+   * @param locationUri storage location uri
+   * @param parameters table metadata parameters
+   */
+  public static ExternalCatalogTableOptions createExternalCatalogTableOptions(
+      String locationUri, Map<String, String> parameters, boolean hiveEngineEnabled) {
+    return new ExternalCatalogTableOptions()
+        .setStorageDescriptor(createStorageDescriptor(locationUri, hiveEngineEnabled))
+        .setParameters(parameters);
+  }
+
+  /**
+   * Creates a new ExternalCatalogDatasetOptions object populated with the supported library
+   * constants and parameters given.
+   *
+   * @param defaultStorageLocationUri dataset's default location uri
+   * @param metadataParameters metadata parameters for the dataset
+   */
+  public static ExternalCatalogDatasetOptions createExternalCatalogDatasetOptions(
+      String defaultStorageLocationUri, Map<String, String> metadataParameters) {
+    return new ExternalCatalogDatasetOptions()
+        .setDefaultStorageLocationUri(defaultStorageLocationUri)
+        .setParameters(metadataParameters);
+  }
+
+  private static StorageDescriptor createStorageDescriptor(
+      String locationUri, boolean hiveEngineEnabled) {
+    String inputFormat;
+    String outputFormat;
+    String serializationLibrary;
+    if (hiveEngineEnabled) {
+      serializationLibrary = HIVE_SERIALIZATION_LIBRARY;
+      inputFormat = HIVE_FILE_INPUT_FORMAT;
+      outputFormat = HIVE_FILE_OUTPUT_FORMAT;
+    } else {
+      serializationLibrary = SERIALIZATION_LIBRARY;

Review Comment:
   No We dont. I removed the code block



-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org

For queries about this service, please contact Infrastructure at:
users@infra.apache.org


---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For additional commands, e-mail: issues-help@iceberg.apache.org


Re: [PR] Catalog: Add BigQuery Metastore Catalog Support [iceberg]

Posted by "danielcweeks (via GitHub)" <gi...@apache.org>.
danielcweeks commented on code in PR #12808:
URL: https://github.com/apache/iceberg/pull/12808#discussion_r2072196526


##########
bigquery/src/main/java/org/apache/iceberg/gcp/bigquery/BigQueryTableOperations.java:
##########
@@ -0,0 +1,296 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.iceberg.gcp.bigquery;
+
+import com.google.api.services.bigquery.model.ExternalCatalogTableOptions;
+import com.google.api.services.bigquery.model.Table;
+import com.google.api.services.bigquery.model.TableReference;
+import java.util.Locale;
+import java.util.Map;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hive.common.StatsSetupConst;
+import org.apache.iceberg.BaseMetastoreOperations;
+import org.apache.iceberg.BaseMetastoreTableOperations;
+import org.apache.iceberg.SnapshotSummary;
+import org.apache.iceberg.TableMetadata;
+import org.apache.iceberg.TableProperties;
+import org.apache.iceberg.exceptions.AlreadyExistsException;
+import org.apache.iceberg.exceptions.CommitFailedException;
+import org.apache.iceberg.exceptions.CommitStateUnknownException;
+import org.apache.iceberg.exceptions.NoSuchTableException;
+import org.apache.iceberg.exceptions.ValidationException;
+import org.apache.iceberg.hadoop.ConfigProperties;
+import org.apache.iceberg.io.FileIO;
+import org.apache.iceberg.relocated.com.google.common.collect.Maps;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/** Handles BigQuery metastore table operations. */
+public final class BigQueryTableOperations extends BaseMetastoreTableOperations {
+
+  private static final Logger LOG = LoggerFactory.getLogger(BigQueryTableOperations.class);
+
+  private static final String TABLE_PROPERTIES_BQ_CONNECTION = "bq_connection";

Review Comment:
   It's unclear to me what this is and how it's used.  



-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org

For queries about this service, please contact Infrastructure at:
users@infra.apache.org


---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For additional commands, e-mail: issues-help@iceberg.apache.org


Re: [PR] Catalog: Add BigQuery Metastore Catalog Support [iceberg]

Posted by "danielcweeks (via GitHub)" <gi...@apache.org>.
danielcweeks commented on code in PR #12808:
URL: https://github.com/apache/iceberg/pull/12808#discussion_r2072158736


##########
bigquery/src/main/java/org/apache/iceberg/gcp/bigquery/BigQueryMetastoreCatalog.java:
##########
@@ -0,0 +1,366 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.iceberg.gcp.bigquery;
+
+import com.google.api.services.bigquery.model.Dataset;
+import com.google.api.services.bigquery.model.DatasetList.Datasets;
+import com.google.api.services.bigquery.model.DatasetReference;
+import com.google.api.services.bigquery.model.ExternalCatalogDatasetOptions;
+import com.google.api.services.bigquery.model.TableReference;
+import com.google.cloud.ServiceOptions;
+import com.google.cloud.bigquery.BigQueryOptions;
+import java.io.IOException;
+import java.io.UncheckedIOException;
+import java.security.GeneralSecurityException;
+import java.util.List;
+import java.util.Locale;
+import java.util.Map;
+import java.util.Objects;
+import java.util.Set;
+import org.apache.hadoop.conf.Configurable;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.iceberg.BaseMetastoreCatalog;
+import org.apache.iceberg.CatalogProperties;
+import org.apache.iceberg.CatalogUtil;
+import org.apache.iceberg.TableMetadata;
+import org.apache.iceberg.TableOperations;
+import org.apache.iceberg.catalog.Namespace;
+import org.apache.iceberg.catalog.SupportsNamespaces;
+import org.apache.iceberg.catalog.TableIdentifier;
+import org.apache.iceberg.exceptions.NoSuchNamespaceException;
+import org.apache.iceberg.exceptions.NoSuchTableException;
+import org.apache.iceberg.io.FileIO;
+import org.apache.iceberg.relocated.com.google.common.annotations.VisibleForTesting;
+import org.apache.iceberg.relocated.com.google.common.base.Preconditions;
+import org.apache.iceberg.relocated.com.google.common.base.Strings;
+import org.apache.iceberg.relocated.com.google.common.collect.ImmutableList;
+import org.apache.iceberg.relocated.com.google.common.collect.ImmutableMap;
+import org.apache.iceberg.relocated.com.google.common.collect.Maps;
+import org.apache.iceberg.util.LocationUtil;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/** Iceberg Bigquery Metastore Catalog implementation. */
+public class BigQueryMetastoreCatalog extends BaseMetastoreCatalog
+    implements SupportsNamespaces, Configurable {
+
+  // User provided properties.
+  public static final String PROJECT_ID = "gcp.bigquery.project-id";
+  public static final String GCP_LOCATION = "gcp.bigquery.location";
+  public static final String LIST_ALL_TABLES = "gcp.bigquery.list-all-tables";
+
+  private static final Logger LOG = LoggerFactory.getLogger(BigQueryMetastoreCatalog.class);
+
+  private static final String DEFAULT_GCP_LOCATION = "us";
+
+  private String catalogName;
+  private Map<String, String> catalogProperties;
+  private FileIO fileIO;
+  private Configuration conf;
+  private String projectId;
+  private String projectLocation;
+  private BigQueryMetastoreClient client;
+  private boolean listAllTables;
+  private String warehouseLocation;
+
+  public BigQueryMetastoreCatalog() {}
+
+  @Override
+  public void initialize(String name, Map<String, String> properties) {
+    Preconditions.checkArgument(
+        properties.containsKey(PROJECT_ID),
+        "Invalid GCP project: %s must be specified",
+        PROJECT_ID);
+
+    this.projectId = properties.get(PROJECT_ID);
+    this.projectLocation = properties.getOrDefault(GCP_LOCATION, DEFAULT_GCP_LOCATION);
+
+    BigQueryOptions options =
+        BigQueryOptions.newBuilder()
+            .setProjectId(projectId)
+            .setLocation(projectLocation)
+            .setRetrySettings(ServiceOptions.getDefaultRetrySettings())
+            .build();
+
+    try {
+      client = new BigQueryMetastoreClientImpl(options);
+    } catch (IOException e) {
+      throw new UncheckedIOException("Creating BigQuery client failed", e);
+    } catch (GeneralSecurityException e) {
+      throw new RuntimeException("Creating BigQuery client failed due to a security issue", e);
+    }
+
+    initialize(name, properties, projectId, projectLocation, client);
+  }
+
+  @VisibleForTesting
+  void initialize(
+      String name,
+      Map<String, String> properties,
+      String initialProjectId,
+      String initialLocation,
+      BigQueryMetastoreClient bigQueryMetaStoreClient) {
+    Preconditions.checkArgument(bigQueryMetaStoreClient != null, "Invalid BigQuery client: null");
+    this.catalogName = name;
+    this.catalogProperties = ImmutableMap.copyOf(properties);
+    this.projectId = initialProjectId;
+    this.projectLocation = initialLocation;
+    this.client = bigQueryMetaStoreClient;
+
+    if (this.conf == null) {
+      LOG.warn("No configuration was set, using the default environment Configuration");
+      this.conf = new Configuration();
+    }
+
+    LOG.info("Using BigQuery Metastore Iceberg Catalog: {}", name);
+
+    if (properties.containsKey(CatalogProperties.WAREHOUSE_LOCATION)) {
+      this.warehouseLocation =
+          LocationUtil.stripTrailingSlash(properties.get(CatalogProperties.WAREHOUSE_LOCATION));
+    }
+
+    this.fileIO =
+        CatalogUtil.loadFileIO(
+            properties.getOrDefault(
+                CatalogProperties.FILE_IO_IMPL, "org.apache.iceberg.io.ResolvingFileIO"),
+            properties,
+            conf);
+
+    this.listAllTables = Boolean.parseBoolean(properties.getOrDefault(LIST_ALL_TABLES, "true"));
+  }
+
+  @Override
+  protected TableOperations newTableOps(TableIdentifier identifier) {
+    return new BigQueryTableOperations(client, fileIO, toTableReference(identifier), conf);
+  }
+
+  @Override
+  protected String defaultWarehouseLocation(TableIdentifier identifier) {
+    String locationUri = null;
+    DatasetReference datasetReference = toDatasetReference(identifier.namespace());
+    Dataset dataset = client.load(datasetReference);
+    if (dataset != null && dataset.getExternalCatalogDatasetOptions() != null) {
+      locationUri = dataset.getExternalCatalogDatasetOptions().getDefaultStorageLocationUri();
+    }
+
+    return String.format(
+        "%s/%s",
+        Strings.isNullOrEmpty(locationUri)
+            ? createDefaultStorageLocationUri(datasetReference.getDatasetId())
+            : LocationUtil.stripTrailingSlash(locationUri),
+        identifier.name());
+  }
+
+  @Override
+  public List<TableIdentifier> listTables(Namespace namespace) {
+    validateNamespace(namespace);
+
+    return client.list(toDatasetReference(namespace), listAllTables).stream()
+        .map(
+            table -> TableIdentifier.of(namespace.level(0), table.getTableReference().getTableId()))
+        .collect(ImmutableList.toImmutableList());
+  }
+
+  @Override
+  public boolean dropTable(TableIdentifier identifier, boolean purge) {
+    try {
+      TableOperations ops = newTableOps(identifier);
+      TableMetadata lastMetadata = ops.current();
+
+      client.delete(toTableReference(identifier));
+
+      if (purge && lastMetadata != null) {
+        CatalogUtil.dropTableData(ops.io(), lastMetadata);
+      }
+    } catch (NoSuchTableException e) {
+      return false;
+    }
+
+    return true;
+  }
+
+  @Override
+  public void renameTable(TableIdentifier from, TableIdentifier to) {
+    // TODO: Enable once supported by BigQuery API.
+    throw new UnsupportedOperationException("Table rename operation is unsupported.");
+  }
+
+  @Override
+  public void createNamespace(Namespace namespace, Map<String, String> metadata) {
+    Dataset builder = new Dataset();
+    DatasetReference datasetReference = toDatasetReference(namespace);
+    builder.setLocation(this.projectLocation);
+    builder.setDatasetReference(datasetReference);
+    builder.setExternalCatalogDatasetOptions(
+        BigQueryMetastoreUtils.createExternalCatalogDatasetOptions(
+            createDefaultStorageLocationUri(datasetReference.getDatasetId()), metadata));
+
+    client.create(builder);
+  }
+
+  @Override
+  public List<Namespace> listNamespaces() {
+    try {
+      return listNamespaces(Namespace.empty());
+    } catch (NoSuchNamespaceException e) {
+      return ImmutableList.of();
+    }
+  }
+
+  /**
+   * Since this catalog only supports one-level namespaces, it always returns an empty list unless
+   * passed an empty namespace to list all namespaces within the catalog.
+   */
+  @Override
+  public List<Namespace> listNamespaces(Namespace namespace) {
+    List<Datasets> allDatasets = client.list(projectId);
+
+    ImmutableList<Namespace> namespaces =
+        allDatasets.stream().map(this::toNamespace).collect(ImmutableList.toImmutableList());
+    if (namespaces.isEmpty()) {
+      throw new NoSuchNamespaceException("Namespace does not exist: %s", namespace);
+    }
+
+    return namespaces;
+  }
+
+  @Override
+  public boolean dropNamespace(Namespace namespace) {
+    try {
+      client.delete(toDatasetReference(namespace));

Review Comment:
   It's not clear to me that we're throwing properly if the namespace is not empty (per the Catalog API via SupportsNamespaces)



-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org

For queries about this service, please contact Infrastructure at:
users@infra.apache.org


---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For additional commands, e-mail: issues-help@iceberg.apache.org


Re: [PR] Catalog: Add BigQuery Metastore Catalog Support [iceberg]

Posted by "ebyhr (via GitHub)" <gi...@apache.org>.
ebyhr commented on code in PR #12808:
URL: https://github.com/apache/iceberg/pull/12808#discussion_r2059704342


##########
bigquery/src/test/java/org/apache/iceberg/gcp/bigquery/BigQueryMetastoreTestUtils.java:
##########
@@ -0,0 +1,86 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.iceberg.gcp.bigquery;
+
+import static org.apache.iceberg.types.Types.NestedField.required;
+import static org.assertj.core.api.Assertions.assertThat;
+
+import com.google.api.services.bigquery.model.ExternalCatalogTableOptions;
+import com.google.api.services.bigquery.model.StorageDescriptor;
+import com.google.api.services.bigquery.model.Table;
+import com.google.api.services.bigquery.model.TableReference;
+import java.io.File;
+import java.io.IOException;
+import java.util.Collections;
+import java.util.Optional;
+import org.apache.commons.io.FileUtils;
+import org.apache.commons.io.filefilter.TrueFileFilter;
+import org.apache.iceberg.Schema;
+import org.apache.iceberg.catalog.TableIdentifier;
+import org.apache.iceberg.types.Types;
+
+/** Test utility methods for BigQuery Metastore Iceberg catalog. */
+public final class BigQueryMetastoreTestUtils {
+  private BigQueryMetastoreTestUtils() {}
+
+  public static final String METADATA_LOCATION_PROP = "metadata_location";
+
+  public static Schema getTestSchema() {
+    return new Schema(
+        required(1, "id", Types.IntegerType.get(), "unique ID"),
+        required(2, "data", Types.StringType.get()));
+  }
+
+  public static Table createTestTable(
+      File tempFolder,
+      BigQueryMetastoreCatalog bigQueryMetastoreCatalog,
+      TableReference tableReference)
+      throws IOException {
+    Schema schema = getTestSchema();
+    TableIdentifier tableIdentifier =
+        TableIdentifier.of(tableReference.getDatasetId(), tableReference.getTableId());
+    String tableDir = tempFolder.toPath().resolve(tableReference.getTableId()).toString();
+
+    bigQueryMetastoreCatalog
+        .buildTable(tableIdentifier, schema)
+        .withLocation(tableDir)
+        .createTransaction()
+        .commitTransaction();
+
+    Optional<String> metadataLocation = getIcebergMetadataFilePath(tableDir);
+    assertThat(metadataLocation.isPresent());

Review Comment:
   That is not true since `assertThat(Optional.empty())` always passes. 



-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org

For queries about this service, please contact Infrastructure at:
users@infra.apache.org


---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For additional commands, e-mail: issues-help@iceberg.apache.org


Re: [PR] Catalog: Add BigQuery Metastore Catalog Support [iceberg]

Posted by "talatuyarer (via GitHub)" <gi...@apache.org>.
talatuyarer commented on code in PR #12808:
URL: https://github.com/apache/iceberg/pull/12808#discussion_r2059722252


##########
bigquery/src/main/java/org/apache/iceberg/gcp/bigquery/BigQueryMetastoreCatalog.java:
##########
@@ -0,0 +1,402 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.iceberg.gcp.bigquery;
+
+import com.google.api.services.bigquery.model.Dataset;
+import com.google.api.services.bigquery.model.DatasetList.Datasets;
+import com.google.api.services.bigquery.model.DatasetReference;
+import com.google.api.services.bigquery.model.ExternalCatalogDatasetOptions;
+import com.google.api.services.bigquery.model.TableReference;
+import com.google.cloud.ServiceOptions;
+import com.google.cloud.bigquery.BigQueryOptions;
+import java.io.IOException;
+import java.security.GeneralSecurityException;
+import java.util.List;
+import java.util.Locale;
+import java.util.Map;
+import java.util.Objects;
+import java.util.Set;
+import org.apache.hadoop.conf.Configurable;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.iceberg.BaseMetastoreCatalog;
+import org.apache.iceberg.CatalogProperties;
+import org.apache.iceberg.CatalogUtil;
+import org.apache.iceberg.TableMetadata;
+import org.apache.iceberg.TableOperations;
+import org.apache.iceberg.catalog.Namespace;
+import org.apache.iceberg.catalog.SupportsNamespaces;
+import org.apache.iceberg.catalog.TableIdentifier;
+import org.apache.iceberg.exceptions.AlreadyExistsException;
+import org.apache.iceberg.exceptions.NoSuchNamespaceException;
+import org.apache.iceberg.exceptions.NoSuchTableException;
+import org.apache.iceberg.exceptions.ServiceFailureException;
+import org.apache.iceberg.exceptions.ValidationException;
+import org.apache.iceberg.io.FileIO;
+import org.apache.iceberg.relocated.com.google.common.annotations.VisibleForTesting;
+import org.apache.iceberg.relocated.com.google.common.base.Preconditions;
+import org.apache.iceberg.relocated.com.google.common.base.Strings;
+import org.apache.iceberg.relocated.com.google.common.collect.ImmutableList;
+import org.apache.iceberg.relocated.com.google.common.collect.ImmutableMap;
+import org.apache.iceberg.relocated.com.google.common.collect.Maps;
+import org.apache.iceberg.util.LocationUtil;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/** Iceberg Bigquery Metastore Catalog implementation. */
+public final class BigQueryMetastoreCatalog extends BaseMetastoreCatalog
+    implements SupportsNamespaces, Configurable {
+
+  // User provided properties.
+  public static final String PROPERTIES_KEY_GCP_PROJECT = "gcp-project";
+  public static final String PROPERTIES_KEY_GCP_LOCATION = "gcp-location";
+  public static final String PROPERTIES_KEY_FILTER_UNSUPPORTED_TABLES = "filter-unsupported-tables";

Review Comment:
   @nastra suggested to remove from GCP Properties. I moved them here. If you guys think it has to be under GCP Properties I am fine to move them



-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org

For queries about this service, please contact Infrastructure at:
users@infra.apache.org


---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For additional commands, e-mail: issues-help@iceberg.apache.org


Re: [PR] Catalog: Add BigQuery Metastore Catalog Support [iceberg]

Posted by "nastra (via GitHub)" <gi...@apache.org>.
nastra commented on code in PR #12808:
URL: https://github.com/apache/iceberg/pull/12808#discussion_r2059731731


##########
bigquery/src/main/java/org/apache/iceberg/gcp/bigquery/BigQueryMetastoreCatalog.java:
##########
@@ -0,0 +1,400 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.iceberg.gcp.bigquery;
+
+import com.google.api.services.bigquery.model.Dataset;
+import com.google.api.services.bigquery.model.DatasetList.Datasets;
+import com.google.api.services.bigquery.model.DatasetReference;
+import com.google.api.services.bigquery.model.ExternalCatalogDatasetOptions;
+import com.google.api.services.bigquery.model.TableReference;
+import com.google.cloud.ServiceOptions;
+import com.google.cloud.bigquery.BigQueryOptions;
+import java.io.IOException;
+import java.security.GeneralSecurityException;
+import java.util.List;
+import java.util.Locale;
+import java.util.Map;
+import java.util.Objects;
+import java.util.Set;
+import org.apache.hadoop.conf.Configurable;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.iceberg.BaseMetastoreCatalog;
+import org.apache.iceberg.CatalogProperties;
+import org.apache.iceberg.CatalogUtil;
+import org.apache.iceberg.TableMetadata;
+import org.apache.iceberg.TableOperations;
+import org.apache.iceberg.catalog.Namespace;
+import org.apache.iceberg.catalog.SupportsNamespaces;
+import org.apache.iceberg.catalog.TableIdentifier;
+import org.apache.iceberg.exceptions.AlreadyExistsException;
+import org.apache.iceberg.exceptions.NoSuchNamespaceException;
+import org.apache.iceberg.exceptions.NoSuchTableException;
+import org.apache.iceberg.exceptions.ServiceFailureException;
+import org.apache.iceberg.exceptions.ValidationException;
+import org.apache.iceberg.io.FileIO;
+import org.apache.iceberg.relocated.com.google.common.annotations.VisibleForTesting;
+import org.apache.iceberg.relocated.com.google.common.base.Preconditions;
+import org.apache.iceberg.relocated.com.google.common.base.Strings;
+import org.apache.iceberg.relocated.com.google.common.collect.ImmutableList;
+import org.apache.iceberg.relocated.com.google.common.collect.ImmutableMap;
+import org.apache.iceberg.relocated.com.google.common.collect.Maps;
+import org.apache.iceberg.util.LocationUtil;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/** Iceberg Bigquery Metastore Catalog implementation. */
+public final class BigQueryMetastoreCatalog extends BaseMetastoreCatalog
+    implements SupportsNamespaces, Configurable {
+
+  // User provided properties.
+  public static final String PROJECT_ID = "gcp-project";
+  public static final String GCP_LOCATION = "gcp-location";
+  public static final String FILTER_UNSUPPORTED_TABLES = "filter-unsupported-tables";
+  public static final String TESTING_ENABLED = "testing-enabled";
+
+  public static final String HIVE_METASTORE_WAREHOUSE_DIR = "hive.metastore.warehouse.dir";
+
+  private static final Logger LOG = LoggerFactory.getLogger(BigQueryMetastoreCatalog.class);
+
+  private static final String DEFAULT_GCP_LOCATION = "us";
+
+  private String catalogName;
+  private Map<String, String> catalogProperties;
+  private FileIO fileIO;
+  private Configuration conf;
+  private String projectId;
+  private String location;
+  private BigQueryMetastoreClient client;
+  private boolean filterUnsupportedTables;
+
+  // Must have a no-arg constructor to be dynamically loaded
+  // initialize(String name, Map<String, String> properties) will be called to complete
+  // initialization
+  public BigQueryMetastoreCatalog() {}
+
+  @Override
+  public void initialize(String inputName, Map<String, String> properties) {
+    Preconditions.checkArgument(
+        properties.containsKey(PROJECT_ID), "GCP project must be specified");
+
+    projectId = properties.get(PROJECT_ID);
+    location = properties.getOrDefault(GCP_LOCATION, DEFAULT_GCP_LOCATION);
+    boolean testingEnabled =

Review Comment:
   I don't think we should keep this flag as we don't do something similar in other catalog implementations in the Iceberg codebase and that would set a precedent that using something like this is ok whilein fact there are better ways to tackle this (having a separate `initialize` method that takes a custom client impl needed for testing)



-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org

For queries about this service, please contact Infrastructure at:
users@infra.apache.org


---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For additional commands, e-mail: issues-help@iceberg.apache.org


Re: [PR] Catalog: Add BigQuery Metastore Catalog Support [iceberg]

Posted by "ebyhr (via GitHub)" <gi...@apache.org>.
ebyhr commented on code in PR #12808:
URL: https://github.com/apache/iceberg/pull/12808#discussion_r2085738367


##########
bigquery/src/main/java/org/apache/iceberg/gcp/bigquery/BigQueryMetastoreClientImpl.java:
##########
@@ -0,0 +1,579 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.iceberg.gcp.bigquery;
+
+import com.google.api.client.googleapis.javanet.GoogleNetHttpTransport;
+import com.google.api.client.http.HttpHeaders;
+import com.google.api.client.http.HttpResponse;
+import com.google.api.client.http.HttpResponseException;
+import com.google.api.client.http.HttpStatusCodes;
+import com.google.api.client.json.gson.GsonFactory;
+import com.google.api.client.util.Data;
+import com.google.api.services.bigquery.Bigquery;
+import com.google.api.services.bigquery.BigqueryScopes;
+import com.google.api.services.bigquery.model.Dataset;
+import com.google.api.services.bigquery.model.DatasetList;
+import com.google.api.services.bigquery.model.DatasetList.Datasets;
+import com.google.api.services.bigquery.model.DatasetReference;
+import com.google.api.services.bigquery.model.ExternalCatalogDatasetOptions;
+import com.google.api.services.bigquery.model.ExternalCatalogTableOptions;
+import com.google.api.services.bigquery.model.Table;
+import com.google.api.services.bigquery.model.TableList;
+import com.google.api.services.bigquery.model.TableList.Tables;
+import com.google.api.services.bigquery.model.TableReference;
+import com.google.api.services.bigquery.model.TableSchema;
+import com.google.auth.http.HttpCredentialsAdapter;
+import com.google.auth.oauth2.GoogleCredentials;
+import com.google.cloud.BaseServiceException;
+import com.google.cloud.ExceptionHandler;
+import com.google.cloud.bigquery.BigQueryErrorMessages;
+import com.google.cloud.bigquery.BigQueryOptions;
+import com.google.cloud.bigquery.BigQueryRetryConfig;
+import com.google.cloud.bigquery.BigQueryRetryHelper;
+import java.io.IOException;
+import java.nio.charset.StandardCharsets;
+import java.security.GeneralSecurityException;
+import java.util.List;
+import java.util.Locale;
+import java.util.Map;
+import java.util.Set;
+import java.util.stream.Collectors;
+import java.util.stream.Stream;
+import org.apache.iceberg.BaseMetastoreTableOperations;
+import org.apache.iceberg.exceptions.AlreadyExistsException;
+import org.apache.iceberg.exceptions.BadRequestException;
+import org.apache.iceberg.exceptions.ForbiddenException;
+import org.apache.iceberg.exceptions.NoSuchIcebergTableException;
+import org.apache.iceberg.exceptions.NoSuchNamespaceException;
+import org.apache.iceberg.exceptions.NoSuchTableException;
+import org.apache.iceberg.exceptions.NotAuthorizedException;
+import org.apache.iceberg.exceptions.RuntimeIOException;
+import org.apache.iceberg.exceptions.ServiceFailureException;
+import org.apache.iceberg.exceptions.ServiceUnavailableException;
+import org.apache.iceberg.exceptions.ValidationException;
+import org.apache.iceberg.relocated.com.google.common.base.Preconditions;
+import org.apache.iceberg.relocated.com.google.common.collect.Lists;
+import org.apache.iceberg.relocated.com.google.common.collect.Maps;
+
+/** A client of Google Bigquery Metastore functions over the BigQuery service. */
+public final class BigQueryMetastoreClientImpl implements BigQueryMetastoreClient {
+
+  private final Bigquery client;
+  private final BigQueryOptions bigqueryOptions;
+
+  public static final ExceptionHandler.Interceptor EXCEPTION_HANDLER_INTERCEPTOR =
+      new ExceptionHandler.Interceptor() {
+
+        @Override
+        public RetryResult afterEval(Exception exception, RetryResult retryResult) {
+          return ExceptionHandler.Interceptor.RetryResult.CONTINUE_EVALUATION;
+        }
+
+        @Override
+        public RetryResult beforeEval(Exception exception) {
+          if (exception instanceof BaseServiceException) {
+            boolean retriable = ((BaseServiceException) exception).isRetryable();
+            return retriable
+                ? ExceptionHandler.Interceptor.RetryResult.RETRY
+                : ExceptionHandler.Interceptor.RetryResult.CONTINUE_EVALUATION;
+          }
+
+          return ExceptionHandler.Interceptor.RetryResult.CONTINUE_EVALUATION;
+        }
+      };
+
+  // Retry config with error messages and regex for rate limit exceeded errors.
+  private static final BigQueryRetryConfig DEFAULT_RETRY_CONFIG =
+      BigQueryRetryConfig.newBuilder()
+          .retryOnMessage(BigQueryErrorMessages.RATE_LIMIT_EXCEEDED_MSG)
+          .retryOnMessage(BigQueryErrorMessages.JOB_RATE_LIMIT_EXCEEDED_MSG)
+          .retryOnRegEx(BigQueryErrorMessages.RetryRegExPatterns.RATE_LIMIT_EXCEEDED_REGEX)
+          .build();
+
+  public static final ExceptionHandler BIGQUERY_EXCEPTION_HANDLER =
+      ExceptionHandler.newBuilder()
+          .abortOn(RuntimeException.class)
+          // Retry on connection failures due to transient network issues.
+          .retryOn(java.net.ConnectException.class)
+          // Retry to recover from temporary DNS resolution failures.
+          .retryOn(java.net.UnknownHostException.class)
+          .addInterceptors(EXCEPTION_HANDLER_INTERCEPTOR)
+          .build();
+
+  /** Constructs a client of the Google BigQuery service. */
+  public BigQueryMetastoreClientImpl(BigQueryOptions options)
+      throws IOException, GeneralSecurityException {
+    // Initialize client that will be used to send requests. This client only needs to be created
+    // once, and can be reused for multiple requests
+    HttpCredentialsAdapter httpCredentialsAdapter =
+        new HttpCredentialsAdapter(
+            GoogleCredentials.getApplicationDefault().createScoped(BigqueryScopes.all()));

Review Comment:
   https://github.com/apache/iceberg/issues/13036



-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org

For queries about this service, please contact Infrastructure at:
users@infra.apache.org


---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For additional commands, e-mail: issues-help@iceberg.apache.org


Re: [PR] Catalog: Add BigQuery Metastore Catalog Support [iceberg]

Posted by "talatuyarer (via GitHub)" <gi...@apache.org>.
talatuyarer commented on code in PR #12808:
URL: https://github.com/apache/iceberg/pull/12808#discussion_r2047635969


##########
bigquery/src/main/java/org/apache/iceberg/gcp/bigquery/BigQueryMetaStoreClientImpl.java:
##########
@@ -0,0 +1,620 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.iceberg.gcp.bigquery;
+
+import com.google.api.client.googleapis.javanet.GoogleNetHttpTransport;
+import com.google.api.client.http.HttpHeaders;
+import com.google.api.client.http.HttpResponse;
+import com.google.api.client.http.HttpResponseException;
+import com.google.api.client.http.HttpStatusCodes;
+import com.google.api.client.json.gson.GsonFactory;
+import com.google.api.client.util.Data;
+import com.google.api.services.bigquery.Bigquery;
+import com.google.api.services.bigquery.BigqueryScopes;
+import com.google.api.services.bigquery.model.Dataset;
+import com.google.api.services.bigquery.model.DatasetList;
+import com.google.api.services.bigquery.model.DatasetList.Datasets;
+import com.google.api.services.bigquery.model.DatasetReference;
+import com.google.api.services.bigquery.model.ExternalCatalogDatasetOptions;
+import com.google.api.services.bigquery.model.ExternalCatalogTableOptions;
+import com.google.api.services.bigquery.model.Table;
+import com.google.api.services.bigquery.model.TableList;
+import com.google.api.services.bigquery.model.TableList.Tables;
+import com.google.api.services.bigquery.model.TableReference;
+import com.google.api.services.bigquery.model.TableSchema;
+import com.google.auth.http.HttpCredentialsAdapter;
+import com.google.auth.oauth2.GoogleCredentials;
+import com.google.cloud.BaseServiceException;
+import com.google.cloud.ExceptionHandler;
+import com.google.cloud.bigquery.BigQueryErrorMessages;
+import com.google.cloud.bigquery.BigQueryOptions;
+import com.google.cloud.bigquery.BigQueryRetryConfig;
+import com.google.cloud.bigquery.BigQueryRetryHelper;
+import java.io.IOException;
+import java.nio.charset.StandardCharsets;
+import java.security.GeneralSecurityException;
+import java.util.List;
+import java.util.Locale;
+import java.util.Map;
+import java.util.Set;
+import java.util.concurrent.Callable;
+import java.util.stream.Collectors;
+import java.util.stream.Stream;
+import org.apache.iceberg.BaseMetastoreTableOperations;
+import org.apache.iceberg.exceptions.AlreadyExistsException;
+import org.apache.iceberg.exceptions.BadRequestException;
+import org.apache.iceberg.exceptions.ForbiddenException;
+import org.apache.iceberg.exceptions.NoSuchIcebergTableException;
+import org.apache.iceberg.exceptions.NoSuchNamespaceException;
+import org.apache.iceberg.exceptions.NoSuchTableException;
+import org.apache.iceberg.exceptions.NotAuthorizedException;
+import org.apache.iceberg.exceptions.RuntimeIOException;
+import org.apache.iceberg.exceptions.ServiceFailureException;
+import org.apache.iceberg.exceptions.ServiceUnavailableException;
+import org.apache.iceberg.exceptions.ValidationException;
+import org.apache.iceberg.relocated.com.google.common.base.Preconditions;
+import org.apache.iceberg.relocated.com.google.common.collect.Lists;
+import org.apache.iceberg.relocated.com.google.common.collect.Maps;
+
+/** A client of Google Bigquery Metastore functions over the BigQuery service. */
+public final class BigQueryMetaStoreClientImpl implements BigQueryMetaStoreClient {
+
+  public static final ExceptionHandler.Interceptor EXCEPTION_HANDLER_INTERCEPTOR =
+      new ExceptionHandler.Interceptor() {
+
+        private static final long serialVersionUID = -8429573486870467828L;
+
+        @Override
+        public RetryResult afterEval(Exception exception, RetryResult retryResult) {
+          return ExceptionHandler.Interceptor.RetryResult.CONTINUE_EVALUATION;
+        }
+
+        @Override
+        public RetryResult beforeEval(Exception exception) {
+          if (exception instanceof BaseServiceException) {
+            boolean retriable = ((BaseServiceException) exception).isRetryable();
+            return retriable
+                ? ExceptionHandler.Interceptor.RetryResult.RETRY
+                : ExceptionHandler.Interceptor.RetryResult.CONTINUE_EVALUATION;
+          }
+          return ExceptionHandler.Interceptor.RetryResult.CONTINUE_EVALUATION;
+        }
+      };
+
+  private final Bigquery client;
+  private final BigQueryOptions bigqueryOptions;
+
+  // Retry config with error messages and regex for rate limit exceeded errors.
+  private static final BigQueryRetryConfig DEFAULT_RETRY_CONFIG =
+      BigQueryRetryConfig.newBuilder()
+          .retryOnMessage(BigQueryErrorMessages.RATE_LIMIT_EXCEEDED_MSG)
+          .retryOnMessage(BigQueryErrorMessages.JOB_RATE_LIMIT_EXCEEDED_MSG)
+          .retryOnRegEx(BigQueryErrorMessages.RetryRegExPatterns.RATE_LIMIT_EXCEEDED_REGEX)
+          .build();
+
+  public static final ExceptionHandler BIGQUERY_EXCEPTION_HANDLER =
+      ExceptionHandler.newBuilder()
+          .abortOn(RuntimeException.class)
+          .retryOn(java.net.ConnectException.class) // retry on Connection Exception
+          .retryOn(java.net.UnknownHostException.class) // retry on UnknownHostException
+          .addInterceptors(EXCEPTION_HANDLER_INTERCEPTOR)
+          .build();
+
+  /** Constructs a client of the Google BigQuery service. */
+  public BigQueryMetaStoreClientImpl(BigQueryOptions options)
+      throws IOException, GeneralSecurityException {
+    // Initialize client that will be used to send requests. This client only needs to be created
+    // once, and can be reused for multiple requests
+    HttpCredentialsAdapter httpCredentialsAdapter =
+        new HttpCredentialsAdapter(
+            GoogleCredentials.getApplicationDefault().createScoped(BigqueryScopes.all()));
+    this.client =
+        new Bigquery.Builder(
+                GoogleNetHttpTransport.newTrustedTransport(),
+                GsonFactory.getDefaultInstance(),
+                httpRequest -> {
+                  httpCredentialsAdapter.initialize(httpRequest);
+                  // Instead of throwing exceptions, analyze the HttpResponse object and inspect its
+                  // status code. This will allow BigQuery API errors to be converted into Iceberg
+                  // exceptions.
+                  httpRequest.setThrowExceptionOnExecuteError(false);
+                })
+            .setApplicationName("BigQuery Iceberg Catalog Plugin")
+            .build();
+    this.bigqueryOptions = options;
+  }
+
+  @Override
+  public Dataset createDataset(Dataset dataset) {
+    // TODO (b/399885863): Ensure Dataset creation is idempotent when handling retries.
+    Dataset response = null;
+    try {
+      response =
+          BigQueryRetryHelper.runWithRetries(
+              new Callable<Dataset>() {
+                @Override
+                public Dataset call() {
+                  return create(dataset);
+                }
+              },
+              bigqueryOptions.getRetrySettings(),
+              BIGQUERY_EXCEPTION_HANDLER,
+              bigqueryOptions.getClock(),
+              DEFAULT_RETRY_CONFIG);
+    } catch (BigQueryRetryHelper.BigQueryRetryHelperException e) {
+      handleBigQueryRetryException(e);
+    }
+    return response;
+  }
+
+  @SuppressWarnings("FormatStringAnnotation")
+  private Dataset create(Dataset dataset) {
+    try {
+      HttpResponse response =
+          client
+              .datasets()
+              .insert(dataset.getDatasetReference().getProjectId(), dataset)
+              .executeUnparsed();
+      return convertExceptionIfUnsuccessful(response).parseAs(Dataset.class);
+    } catch (IOException e) {
+      throw new RuntimeIOException(e);
+    } catch (AlreadyExistsException e) {
+      throw new AlreadyExistsException("Namespace already exists: " + dataset.getId());
+    }
+  }
+
+  @Override
+  @SuppressWarnings("FormatStringAnnotation")
+  public Dataset getDataset(DatasetReference datasetReference) {
+    try {
+      HttpResponse response =
+          client
+              .datasets()
+              .get(datasetReference.getProjectId(), datasetReference.getDatasetId())
+              .executeUnparsed();
+      if (response.getStatusCode() == HttpStatusCodes.STATUS_CODE_NOT_FOUND) {
+        throw new NoSuchNamespaceException(
+            "Namespace does not exist: " + datasetReference.getDatasetId());
+      }
+      return convertExceptionIfUnsuccessful(response).parseAs(Dataset.class);
+    } catch (IOException e) {
+      throw new RuntimeIOException(e);
+    }
+  }
+
+  @Override
+  @SuppressWarnings("FormatStringAnnotation")
+  public void deleteDataset(DatasetReference datasetReference) {
+    try {
+      HttpResponse response =
+          client
+              .datasets()
+              .delete(datasetReference.getProjectId(), datasetReference.getDatasetId())
+              .executeUnparsed();
+      if (response.getStatusCode() == HttpStatusCodes.STATUS_CODE_NOT_FOUND) {
+        throw new NoSuchNamespaceException(
+            "Namespace does not exist: " + datasetReference.getDatasetId());
+      }
+      convertExceptionIfUnsuccessful(response);
+    } catch (IOException e) {
+      throw new RuntimeIOException(e);
+    }
+  }
+
+  @Override
+  public Dataset setDatasetParameters(
+      DatasetReference datasetReference, Map<String, String> parameters) {
+    Dataset dataset = getDataset(datasetReference);
+    ExternalCatalogDatasetOptions externalCatalogDatasetOptions =
+        dataset.getExternalCatalogDatasetOptions() == null
+            ? new ExternalCatalogDatasetOptions()
+            : dataset.getExternalCatalogDatasetOptions();
+    Map<String, String> finalParameters =
+        externalCatalogDatasetOptions.getParameters() == null
+            ? Maps.newHashMap()
+            : externalCatalogDatasetOptions.getParameters();
+    finalParameters.putAll(parameters);
+
+    dataset.setExternalCatalogDatasetOptions(
+        externalCatalogDatasetOptions.setParameters(finalParameters));
+
+    return updateDataset(dataset);
+  }
+
+  @Override
+  public Dataset removeDatasetParameters(
+      DatasetReference datasetReference, Set<String> parameters) {
+    Dataset dataset = getDataset(datasetReference);
+    ExternalCatalogDatasetOptions externalCatalogDatasetOptions =
+        dataset.getExternalCatalogDatasetOptions() == null
+            ? new ExternalCatalogDatasetOptions()
+            : dataset.getExternalCatalogDatasetOptions();
+    Map<String, String> finalParameters =
+        externalCatalogDatasetOptions.getParameters() == null
+            ? Maps.newHashMap()
+            : externalCatalogDatasetOptions.getParameters();
+    parameters.forEach(finalParameters::remove);
+
+    dataset.setExternalCatalogDatasetOptions(
+        externalCatalogDatasetOptions.setParameters(finalParameters));
+
+    return updateDataset(dataset);
+  }
+
+  @Override
+  public List<Datasets> listDatasets(String projectId) {
+    try {
+      String nextPageToken = null;
+      List<Datasets> datasets = Lists.newArrayList();
+      do {
+        HttpResponse pageResponse =
+            client.datasets().list(projectId).setPageToken(nextPageToken).executeUnparsed();
+        DatasetList result =
+            convertExceptionIfUnsuccessful(pageResponse).parseAs(DatasetList.class);
+        nextPageToken = result.getNextPageToken();
+        if (result.getDatasets() != null) {
+          datasets.addAll(result.getDatasets());
+        }
+      } while (nextPageToken != null && !nextPageToken.isEmpty());
+      return datasets;
+    } catch (IOException e) {
+      throw new RuntimeIOException(e);
+    }
+  }
+
+  @Override
+  public Table createTable(Table table) {
+    // Ensure it is an Iceberg table supported by the BQ metastore catalog.
+    validateTable(table);
+    // TODO (b/399885863): Ensure table creation is idempotent when handling retries.

Review Comment:
   I want to keep them until we solve user limitation. This comment mostly is read by Google engineers or customers 



-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org

For queries about this service, please contact Infrastructure at:
users@infra.apache.org


---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For additional commands, e-mail: issues-help@iceberg.apache.org


Re: [PR] Catalog: Add BigQuery Metastore Catalog Support [iceberg]

Posted by "nastra (via GitHub)" <gi...@apache.org>.
nastra commented on code in PR #12808:
URL: https://github.com/apache/iceberg/pull/12808#discussion_r2055588983


##########
bigquery/src/main/java/org/apache/iceberg/gcp/bigquery/BigQueryMetastoreCatalog.java:
##########
@@ -0,0 +1,400 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.iceberg.gcp.bigquery;
+
+import com.google.api.services.bigquery.model.Dataset;
+import com.google.api.services.bigquery.model.DatasetList.Datasets;
+import com.google.api.services.bigquery.model.DatasetReference;
+import com.google.api.services.bigquery.model.ExternalCatalogDatasetOptions;
+import com.google.api.services.bigquery.model.TableReference;
+import com.google.cloud.ServiceOptions;
+import com.google.cloud.bigquery.BigQueryOptions;
+import java.io.IOException;
+import java.security.GeneralSecurityException;
+import java.util.List;
+import java.util.Locale;
+import java.util.Map;
+import java.util.Objects;
+import java.util.Set;
+import org.apache.hadoop.conf.Configurable;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.iceberg.BaseMetastoreCatalog;
+import org.apache.iceberg.CatalogProperties;
+import org.apache.iceberg.CatalogUtil;
+import org.apache.iceberg.TableMetadata;
+import org.apache.iceberg.TableOperations;
+import org.apache.iceberg.catalog.Namespace;
+import org.apache.iceberg.catalog.SupportsNamespaces;
+import org.apache.iceberg.catalog.TableIdentifier;
+import org.apache.iceberg.exceptions.AlreadyExistsException;
+import org.apache.iceberg.exceptions.NoSuchNamespaceException;
+import org.apache.iceberg.exceptions.NoSuchTableException;
+import org.apache.iceberg.exceptions.ServiceFailureException;
+import org.apache.iceberg.exceptions.ValidationException;
+import org.apache.iceberg.io.FileIO;
+import org.apache.iceberg.relocated.com.google.common.annotations.VisibleForTesting;
+import org.apache.iceberg.relocated.com.google.common.base.Preconditions;
+import org.apache.iceberg.relocated.com.google.common.base.Strings;
+import org.apache.iceberg.relocated.com.google.common.collect.ImmutableList;
+import org.apache.iceberg.relocated.com.google.common.collect.ImmutableMap;
+import org.apache.iceberg.relocated.com.google.common.collect.Maps;
+import org.apache.iceberg.util.LocationUtil;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/** Iceberg Bigquery Metastore Catalog implementation. */
+public final class BigQueryMetastoreCatalog extends BaseMetastoreCatalog
+    implements SupportsNamespaces, Configurable {
+
+  // User provided properties.
+  public static final String PROJECT_ID = "gcp-project";
+  public static final String GCP_LOCATION = "gcp-location";
+  public static final String FILTER_UNSUPPORTED_TABLES = "filter-unsupported-tables";
+  public static final String TESTING_ENABLED = "testing-enabled";
+
+  public static final String HIVE_METASTORE_WAREHOUSE_DIR = "hive.metastore.warehouse.dir";
+
+  private static final Logger LOG = LoggerFactory.getLogger(BigQueryMetastoreCatalog.class);
+
+  private static final String DEFAULT_GCP_LOCATION = "us";
+
+  private String catalogName;
+  private Map<String, String> catalogProperties;
+  private FileIO fileIO;
+  private Configuration conf;
+  private String projectId;
+  private String location;
+  private BigQueryMetastoreClient client;
+  private boolean filterUnsupportedTables;
+
+  // Must have a no-arg constructor to be dynamically loaded
+  // initialize(String name, Map<String, String> properties) will be called to complete
+  // initialization
+  public BigQueryMetastoreCatalog() {}
+
+  @Override
+  public void initialize(String inputName, Map<String, String> properties) {
+    Preconditions.checkArgument(
+        properties.containsKey(PROJECT_ID), "GCP project must be specified");
+
+    projectId = properties.get(PROJECT_ID);
+    location = properties.getOrDefault(GCP_LOCATION, DEFAULT_GCP_LOCATION);
+    boolean testingEnabled =
+        Boolean.parseBoolean(properties.getOrDefault(TESTING_ENABLED, "false"));
+
+    BigQueryOptions options =
+        BigQueryOptions.newBuilder()
+            .setProjectId(projectId)
+            .setLocation(location)
+            .setRetrySettings(ServiceOptions.getDefaultRetrySettings())
+            .build();
+
+    try {
+      if (testingEnabled) {
+        client = new FakeBigQueryMetastoreClient(options);
+
+      } else {
+        client = new BigQueryMetastoreClientImpl(options);
+      }
+    } catch (IOException e) {
+      throw new ServiceFailureException(e, "Creating BigQuery client failed");
+    } catch (GeneralSecurityException e) {
+      throw new ValidationException(e, "Creating BigQuery client failed due to a security issue");
+    }
+    initialize(inputName, properties, projectId, location, client);
+  }
+
+  @VisibleForTesting
+  void initialize(
+      String inputName,
+      Map<String, String> properties,
+      String initialProjectId,
+      String initialLocation,
+      BigQueryMetastoreClient bigQueryMetaStoreClient) {
+    Preconditions.checkArgument(bigQueryMetaStoreClient != null, "BigQuery client can not be null");
+    this.catalogName = inputName;
+    this.catalogProperties = ImmutableMap.copyOf(properties);
+    this.projectId = initialProjectId;
+    this.location = initialLocation;
+    this.client = bigQueryMetaStoreClient;
+
+    if (this.conf == null) {
+      LOG.warn("No configuration was set, using the default environment Configuration");
+      this.conf = new Configuration();
+    }
+
+    LOG.info("Using BigQuery Metastore Iceberg Catalog: {}", inputName);
+
+    if (properties.containsKey(CatalogProperties.WAREHOUSE_LOCATION)) {
+      // Iceberg always removes trailing slash to avoid paths like "<folder>//data" in file systems
+      // like s3.
+      this.conf.set(
+          HIVE_METASTORE_WAREHOUSE_DIR,
+          LocationUtil.stripTrailingSlash(properties.get(CatalogProperties.WAREHOUSE_LOCATION)));
+    }
+
+    String fileIoImpl =
+        properties.getOrDefault(
+            CatalogProperties.FILE_IO_IMPL, "org.apache.iceberg.hadoop.HadoopFileIO");
+    this.fileIO = CatalogUtil.loadFileIO(fileIoImpl, properties, conf);
+
+    this.filterUnsupportedTables =
+        Boolean.parseBoolean(properties.getOrDefault(FILTER_UNSUPPORTED_TABLES, "false"));
+  }
+
+  @Override
+  protected TableOperations newTableOps(TableIdentifier identifier) {
+    return new BigQueryTableOperations(
+        client,
+        fileIO,
+        projectId,
+        // Sometimes extensions have the namespace contain the table name too, so we are forced to
+        // allow invalid namespace and just take the first part here like other catalog
+        // implementations do.
+        identifier.namespace().level(0),
+        identifier.name(),
+        conf);
+  }
+
+  @Override
+  protected String defaultWarehouseLocation(TableIdentifier identifier) {
+    String locationUri = null;
+    DatasetReference datasetReference = toDatasetReference(identifier.namespace());
+    Dataset dataset = client.getDataset(datasetReference);
+    if (dataset != null && dataset.getExternalCatalogDatasetOptions() != null) {
+      locationUri = dataset.getExternalCatalogDatasetOptions().getDefaultStorageLocationUri();
+    }
+    return String.format(

Review Comment:
   nit: please add a newline after a closing }. See also https://iceberg.apache.org/contribute/#block-spacing



-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org

For queries about this service, please contact Infrastructure at:
users@infra.apache.org


---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For additional commands, e-mail: issues-help@iceberg.apache.org


Re: [PR] Catalog: Add BigQuery Metastore Catalog Support [iceberg]

Posted by "nastra (via GitHub)" <gi...@apache.org>.
nastra commented on code in PR #12808:
URL: https://github.com/apache/iceberg/pull/12808#discussion_r2054352834


##########
bigquery/src/main/java/org/apache/iceberg/gcp/bigquery/BigQueryMetastoreCatalog.java:
##########
@@ -0,0 +1,402 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.iceberg.gcp.bigquery;
+
+import com.google.api.services.bigquery.model.Dataset;
+import com.google.api.services.bigquery.model.DatasetList.Datasets;
+import com.google.api.services.bigquery.model.DatasetReference;
+import com.google.api.services.bigquery.model.ExternalCatalogDatasetOptions;
+import com.google.api.services.bigquery.model.TableReference;
+import com.google.cloud.ServiceOptions;
+import com.google.cloud.bigquery.BigQueryOptions;
+import java.io.IOException;
+import java.security.GeneralSecurityException;
+import java.util.List;
+import java.util.Locale;
+import java.util.Map;
+import java.util.Objects;
+import java.util.Set;
+import org.apache.hadoop.conf.Configurable;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.iceberg.BaseMetastoreCatalog;
+import org.apache.iceberg.CatalogProperties;
+import org.apache.iceberg.CatalogUtil;
+import org.apache.iceberg.TableMetadata;
+import org.apache.iceberg.TableOperations;
+import org.apache.iceberg.catalog.Namespace;
+import org.apache.iceberg.catalog.SupportsNamespaces;
+import org.apache.iceberg.catalog.TableIdentifier;
+import org.apache.iceberg.exceptions.AlreadyExistsException;
+import org.apache.iceberg.exceptions.NoSuchNamespaceException;
+import org.apache.iceberg.exceptions.NoSuchTableException;
+import org.apache.iceberg.exceptions.ServiceFailureException;
+import org.apache.iceberg.exceptions.ValidationException;
+import org.apache.iceberg.io.FileIO;
+import org.apache.iceberg.relocated.com.google.common.annotations.VisibleForTesting;
+import org.apache.iceberg.relocated.com.google.common.base.Preconditions;
+import org.apache.iceberg.relocated.com.google.common.base.Strings;
+import org.apache.iceberg.relocated.com.google.common.collect.ImmutableList;
+import org.apache.iceberg.relocated.com.google.common.collect.ImmutableMap;
+import org.apache.iceberg.relocated.com.google.common.collect.Maps;
+import org.apache.iceberg.util.LocationUtil;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/** Iceberg Bigquery Metastore Catalog implementation. */
+public final class BigQueryMetastoreCatalog extends BaseMetastoreCatalog
+    implements SupportsNamespaces, Configurable {
+
+  // User provided properties.
+  public static final String PROPERTIES_KEY_GCP_PROJECT = "gcp-project";
+  public static final String PROPERTIES_KEY_GCP_LOCATION = "gcp-location";
+  public static final String PROPERTIES_KEY_FILTER_UNSUPPORTED_TABLES = "filter-unsupported-tables";
+  public static final String PROPERTIES_KEY_TESTING_ENABLED = "testing-enabled";
+
+  public static final String HIVE_METASTORE_WAREHOUSE_DIR = "hive.metastore.warehouse.dir";
+
+  private static final Logger LOG = LoggerFactory.getLogger(BigQueryMetastoreCatalog.class);
+
+  private static final String DEFAULT_GCP_LOCATION = "us";
+
+  private String catalogPluginName;
+  private Map<String, String> catalogProperties;
+  private FileIO fileIO;
+  private Configuration conf;
+  private String projectId;
+  private String location;
+  private BigQueryMetastoreClient client;
+  private boolean filterUnsupportedTables;
+
+  // Must have a no-arg constructor to be dynamically loaded
+  // initialize(String name, Map<String, String> properties) will be called to complete
+  // initialization
+  public BigQueryMetastoreCatalog() {}
+
+  @Override
+  public void initialize(String inputName, Map<String, String> properties) {
+    if (!properties.containsKey(PROPERTIES_KEY_GCP_PROJECT)) {
+      throw new ValidationException("GCP project must be specified");
+    }
+    projectId = properties.get(PROPERTIES_KEY_GCP_PROJECT);
+    location = properties.getOrDefault(PROPERTIES_KEY_GCP_LOCATION, DEFAULT_GCP_LOCATION);
+    boolean testingEnabled =
+        Boolean.parseBoolean(properties.getOrDefault(PROPERTIES_KEY_TESTING_ENABLED, "false"));
+
+    BigQueryOptions options =
+        BigQueryOptions.newBuilder()
+            .setProjectId(projectId)
+            .setLocation(location)
+            .setRetrySettings(ServiceOptions.getDefaultRetrySettings())
+            .build();
+
+    try {
+      if (testingEnabled) {
+        client = new FakeBigQueryMetastoreClient(options);
+
+      } else {
+        client = new BigQueryMetastoreClientImpl(options);
+      }
+    } catch (IOException e) {
+      throw new ServiceFailureException(e, "Creating BigQuery client failed");
+    } catch (GeneralSecurityException e) {
+      throw new ValidationException(e, "Creating BigQuery client failed due to a security issue");
+    }
+    initialize(inputName, properties, projectId, location, client);
+  }
+
+  @VisibleForTesting
+  void initialize(
+      String inputName,
+      Map<String, String> properties,
+      String initialProjectId,
+      String initialLocation,
+      BigQueryMetastoreClient bigQueryMetaStoreClient) {
+    this.catalogPluginName = inputName;
+    this.catalogProperties = ImmutableMap.copyOf(properties);
+    this.projectId = initialProjectId;
+    this.location = initialLocation;
+    this.client =
+        Preconditions.checkNotNull(bigQueryMetaStoreClient, "BigQuery client can not be null");
+
+    if (this.conf == null) {
+      LOG.warn("No configuration was set, using the default environment Configuration");
+      this.conf = new Configuration();
+    }
+
+    LOG.info("Using BigQuery Metastore Iceberg Catalog: {}", inputName);
+
+    if (properties.containsKey(CatalogProperties.WAREHOUSE_LOCATION)) {
+      // Iceberg always removes trailing slash to avoid paths like "<folder>//data" in file systems
+      // like s3.
+      this.conf.set(
+          HIVE_METASTORE_WAREHOUSE_DIR,
+          LocationUtil.stripTrailingSlash(properties.get(CatalogProperties.WAREHOUSE_LOCATION)));
+    }
+
+    String fileIoImpl =
+        properties.getOrDefault(
+            CatalogProperties.FILE_IO_IMPL, "org.apache.iceberg.hadoop.HadoopFileIO");
+    this.fileIO = CatalogUtil.loadFileIO(fileIoImpl, properties, conf);
+
+    this.filterUnsupportedTables =
+        Boolean.parseBoolean(
+            properties.getOrDefault(PROPERTIES_KEY_FILTER_UNSUPPORTED_TABLES, "false"));
+  }
+
+  @Override
+  protected TableOperations newTableOps(TableIdentifier identifier) {
+    return new BigQueryTableOperations(
+        client,
+        fileIO,
+        projectId,
+        // Sometimes extensions have the namespace contain the table name too, so we are forced to
+        // allow invalid namespace and just take the first part here like other catalog
+        // implementations do.
+        identifier.namespace().level(0),
+        identifier.name(),
+        conf);
+  }
+
+  @Override
+  protected String defaultWarehouseLocation(TableIdentifier identifier) {
+    String locationUri = null;
+    DatasetReference datasetReference = toDatasetReference(identifier.namespace());
+    Dataset dataset = client.getDataset(datasetReference);
+    if (dataset != null && dataset.getExternalCatalogDatasetOptions() != null) {
+      locationUri = dataset.getExternalCatalogDatasetOptions().getDefaultStorageLocationUri();
+    }
+    return String.format(
+        "%s/%s",
+        Strings.isNullOrEmpty(locationUri)
+            ? datasetReference.getDatasetId()
+            : LocationUtil.stripTrailingSlash(locationUri),
+        identifier.name());
+  }
+
+  @Override
+  public List<TableIdentifier> listTables(Namespace namespace) {
+    validateNamespace(namespace);
+
+    return client.listTables(toDatasetReference(namespace), filterUnsupportedTables).stream()
+        .map(
+            table -> TableIdentifier.of(namespace.level(0), table.getTableReference().getTableId()))
+        .collect(ImmutableList.toImmutableList());
+  }
+
+  @Override
+  public boolean dropTable(TableIdentifier identifier, boolean purge) {
+    try {
+      TableOperations ops = newTableOps(identifier);
+      TableMetadata lastMetadata = ops.current();
+
+      client.deleteTable(toTableReference(identifier));
+
+      if (purge && lastMetadata != null) {
+        CatalogUtil.dropTableData(ops.io(), lastMetadata);
+      }
+    } catch (NoSuchTableException e) { // Not catching a NoSuchIcebergTableException on purpose
+      return false; // The documentation says just return false in this case
+    }
+    return true;
+  }
+
+  @Override
+  public void renameTable(TableIdentifier from, TableIdentifier to) {
+    validateNamespace(from.namespace());
+    validateNamespace(to.namespace());
+
+    try {
+      client.getDataset(toDatasetReference(to.namespace()));
+      client.getDataset(toDatasetReference(from.namespace()));
+    } catch (NoSuchNamespaceException e) {
+      throw e;
+    }
+
+    TableReference fromTableRef = toTableReference(from);
+    TableReference toTableRef = toTableReference(to);
+
+    // Check if the source table exists
+    try {
+      client.getTable(fromTableRef);
+    } catch (NoSuchTableException e) {
+      throw new NoSuchTableException("Table does not exist: %s", from.name());
+    }
+
+    // Check if the destination table already exists
+    try {
+      client.getTable(toTableRef);
+      // If getTable succeeds, the destination table exists
+      throw new AlreadyExistsException("Table already exists: %s", to.name());
+    } catch (NoSuchTableException e) {
+      // Destination table does not exist, proceed with throwing exception
+    }
+
+    if (!from.namespace().equals(to.namespace())) {
+      throw new ValidationException("New table name must be in the same namespace");
+    }
+
+    // TODO(b/354981675): Enable once supported by the API.
+    throw new UnsupportedOperationException("Table rename operation is unsupported.");
+  }
+
+  @Override
+  public void createNamespace(Namespace namespace, Map<String, String> metadata) {
+    Dataset builder = new Dataset();
+    DatasetReference datasetReference = toDatasetReference(namespace);
+    builder.setLocation(this.location);
+    builder.setDatasetReference(datasetReference);
+    builder.setExternalCatalogDatasetOptions(
+        BigQueryMetastoreUtils.createExternalCatalogDatasetOptions(
+            datasetReference.getDatasetId(), metadata));
+
+    client.createDataset(builder);
+  }
+
+  /**
+   * Since this catalog only supports one-level namespaces, it always returns an empty list unless
+   * passed an empty namespace to list all namespaces within the catalog.
+   */
+  @Override
+  public List<Namespace> listNamespaces(Namespace namespace) {
+    if (namespace.levels().length != 0) {
+      // BQMS does not support namespaces under database or tables, returns empty.
+      // It is called when dropping a namespace to make sure it's empty (listTables is called as
+      // well), returns empty to unblock deletion.
+      return ImmutableList.of();
+    }
+    return client.listDatasets(projectId).stream()
+        .map(BigQueryMetastoreCatalog::getNamespace)
+        .collect(ImmutableList.toImmutableList());
+  }
+
+  @Override
+  public boolean dropNamespace(Namespace namespace) {
+    try {
+      client.deleteDataset(toDatasetReference(namespace));
+      /* We don't delete the data folder for safety, which aligns with Hive Metastore's default

Review Comment:
   please use // for inline comments like these



-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org

For queries about this service, please contact Infrastructure at:
users@infra.apache.org


---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For additional commands, e-mail: issues-help@iceberg.apache.org


Re: [PR] Catalog: Add BigQuery Metastore Catalog Support [iceberg]

Posted by "nastra (via GitHub)" <gi...@apache.org>.
nastra commented on code in PR #12808:
URL: https://github.com/apache/iceberg/pull/12808#discussion_r2055581654


##########
bigquery/src/main/java/org/apache/iceberg/gcp/bigquery/BigQueryMetastoreCatalog.java:
##########
@@ -0,0 +1,400 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.iceberg.gcp.bigquery;
+
+import com.google.api.services.bigquery.model.Dataset;
+import com.google.api.services.bigquery.model.DatasetList.Datasets;
+import com.google.api.services.bigquery.model.DatasetReference;
+import com.google.api.services.bigquery.model.ExternalCatalogDatasetOptions;
+import com.google.api.services.bigquery.model.TableReference;
+import com.google.cloud.ServiceOptions;
+import com.google.cloud.bigquery.BigQueryOptions;
+import java.io.IOException;
+import java.security.GeneralSecurityException;
+import java.util.List;
+import java.util.Locale;
+import java.util.Map;
+import java.util.Objects;
+import java.util.Set;
+import org.apache.hadoop.conf.Configurable;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.iceberg.BaseMetastoreCatalog;
+import org.apache.iceberg.CatalogProperties;
+import org.apache.iceberg.CatalogUtil;
+import org.apache.iceberg.TableMetadata;
+import org.apache.iceberg.TableOperations;
+import org.apache.iceberg.catalog.Namespace;
+import org.apache.iceberg.catalog.SupportsNamespaces;
+import org.apache.iceberg.catalog.TableIdentifier;
+import org.apache.iceberg.exceptions.AlreadyExistsException;
+import org.apache.iceberg.exceptions.NoSuchNamespaceException;
+import org.apache.iceberg.exceptions.NoSuchTableException;
+import org.apache.iceberg.exceptions.ServiceFailureException;
+import org.apache.iceberg.exceptions.ValidationException;
+import org.apache.iceberg.io.FileIO;
+import org.apache.iceberg.relocated.com.google.common.annotations.VisibleForTesting;
+import org.apache.iceberg.relocated.com.google.common.base.Preconditions;
+import org.apache.iceberg.relocated.com.google.common.base.Strings;
+import org.apache.iceberg.relocated.com.google.common.collect.ImmutableList;
+import org.apache.iceberg.relocated.com.google.common.collect.ImmutableMap;
+import org.apache.iceberg.relocated.com.google.common.collect.Maps;
+import org.apache.iceberg.util.LocationUtil;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/** Iceberg Bigquery Metastore Catalog implementation. */
+public final class BigQueryMetastoreCatalog extends BaseMetastoreCatalog
+    implements SupportsNamespaces, Configurable {
+
+  // User provided properties.
+  public static final String PROJECT_ID = "gcp-project";
+  public static final String GCP_LOCATION = "gcp-location";
+  public static final String FILTER_UNSUPPORTED_TABLES = "filter-unsupported-tables";
+  public static final String TESTING_ENABLED = "testing-enabled";
+
+  public static final String HIVE_METASTORE_WAREHOUSE_DIR = "hive.metastore.warehouse.dir";
+
+  private static final Logger LOG = LoggerFactory.getLogger(BigQueryMetastoreCatalog.class);
+
+  private static final String DEFAULT_GCP_LOCATION = "us";
+
+  private String catalogName;
+  private Map<String, String> catalogProperties;
+  private FileIO fileIO;
+  private Configuration conf;
+  private String projectId;
+  private String location;
+  private BigQueryMetastoreClient client;
+  private boolean filterUnsupportedTables;
+
+  // Must have a no-arg constructor to be dynamically loaded
+  // initialize(String name, Map<String, String> properties) will be called to complete
+  // initialization
+  public BigQueryMetastoreCatalog() {}
+
+  @Override
+  public void initialize(String inputName, Map<String, String> properties) {
+    Preconditions.checkArgument(
+        properties.containsKey(PROJECT_ID), "GCP project must be specified");
+
+    projectId = properties.get(PROJECT_ID);
+    location = properties.getOrDefault(GCP_LOCATION, DEFAULT_GCP_LOCATION);
+    boolean testingEnabled =
+        Boolean.parseBoolean(properties.getOrDefault(TESTING_ENABLED, "false"));
+
+    BigQueryOptions options =
+        BigQueryOptions.newBuilder()
+            .setProjectId(projectId)
+            .setLocation(location)
+            .setRetrySettings(ServiceOptions.getDefaultRetrySettings())
+            .build();
+
+    try {
+      if (testingEnabled) {
+        client = new FakeBigQueryMetastoreClient(options);
+
+      } else {
+        client = new BigQueryMetastoreClientImpl(options);
+      }
+    } catch (IOException e) {
+      throw new ServiceFailureException(e, "Creating BigQuery client failed");
+    } catch (GeneralSecurityException e) {
+      throw new ValidationException(e, "Creating BigQuery client failed due to a security issue");
+    }
+    initialize(inputName, properties, projectId, location, client);
+  }
+
+  @VisibleForTesting
+  void initialize(
+      String inputName,
+      Map<String, String> properties,
+      String initialProjectId,
+      String initialLocation,
+      BigQueryMetastoreClient bigQueryMetaStoreClient) {
+    Preconditions.checkArgument(bigQueryMetaStoreClient != null, "BigQuery client can not be null");
+    this.catalogName = inputName;
+    this.catalogProperties = ImmutableMap.copyOf(properties);
+    this.projectId = initialProjectId;
+    this.location = initialLocation;
+    this.client = bigQueryMetaStoreClient;
+
+    if (this.conf == null) {
+      LOG.warn("No configuration was set, using the default environment Configuration");
+      this.conf = new Configuration();
+    }
+
+    LOG.info("Using BigQuery Metastore Iceberg Catalog: {}", inputName);
+
+    if (properties.containsKey(CatalogProperties.WAREHOUSE_LOCATION)) {
+      // Iceberg always removes trailing slash to avoid paths like "<folder>//data" in file systems

Review Comment:
   I would probably just remove this comment



-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org

For queries about this service, please contact Infrastructure at:
users@infra.apache.org


---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For additional commands, e-mail: issues-help@iceberg.apache.org


Re: [PR] Catalog: Add BigQuery Metastore Catalog Support [iceberg]

Posted by "nastra (via GitHub)" <gi...@apache.org>.
nastra commented on code in PR #12808:
URL: https://github.com/apache/iceberg/pull/12808#discussion_r2055579553


##########
bigquery/src/main/java/org/apache/iceberg/gcp/bigquery/BigQueryMetastoreCatalog.java:
##########
@@ -0,0 +1,400 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.iceberg.gcp.bigquery;
+
+import com.google.api.services.bigquery.model.Dataset;
+import com.google.api.services.bigquery.model.DatasetList.Datasets;
+import com.google.api.services.bigquery.model.DatasetReference;
+import com.google.api.services.bigquery.model.ExternalCatalogDatasetOptions;
+import com.google.api.services.bigquery.model.TableReference;
+import com.google.cloud.ServiceOptions;
+import com.google.cloud.bigquery.BigQueryOptions;
+import java.io.IOException;
+import java.security.GeneralSecurityException;
+import java.util.List;
+import java.util.Locale;
+import java.util.Map;
+import java.util.Objects;
+import java.util.Set;
+import org.apache.hadoop.conf.Configurable;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.iceberg.BaseMetastoreCatalog;
+import org.apache.iceberg.CatalogProperties;
+import org.apache.iceberg.CatalogUtil;
+import org.apache.iceberg.TableMetadata;
+import org.apache.iceberg.TableOperations;
+import org.apache.iceberg.catalog.Namespace;
+import org.apache.iceberg.catalog.SupportsNamespaces;
+import org.apache.iceberg.catalog.TableIdentifier;
+import org.apache.iceberg.exceptions.AlreadyExistsException;
+import org.apache.iceberg.exceptions.NoSuchNamespaceException;
+import org.apache.iceberg.exceptions.NoSuchTableException;
+import org.apache.iceberg.exceptions.ServiceFailureException;
+import org.apache.iceberg.exceptions.ValidationException;
+import org.apache.iceberg.io.FileIO;
+import org.apache.iceberg.relocated.com.google.common.annotations.VisibleForTesting;
+import org.apache.iceberg.relocated.com.google.common.base.Preconditions;
+import org.apache.iceberg.relocated.com.google.common.base.Strings;
+import org.apache.iceberg.relocated.com.google.common.collect.ImmutableList;
+import org.apache.iceberg.relocated.com.google.common.collect.ImmutableMap;
+import org.apache.iceberg.relocated.com.google.common.collect.Maps;
+import org.apache.iceberg.util.LocationUtil;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/** Iceberg Bigquery Metastore Catalog implementation. */
+public final class BigQueryMetastoreCatalog extends BaseMetastoreCatalog
+    implements SupportsNamespaces, Configurable {
+
+  // User provided properties.
+  public static final String PROJECT_ID = "gcp-project";
+  public static final String GCP_LOCATION = "gcp-location";
+  public static final String FILTER_UNSUPPORTED_TABLES = "filter-unsupported-tables";

Review Comment:
   the hive catalog uses `list-all-tables`, would it make sense to name it the same way here?



-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org

For queries about this service, please contact Infrastructure at:
users@infra.apache.org


---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For additional commands, e-mail: issues-help@iceberg.apache.org


Re: [PR] Catalog: Add BigQuery Metastore Catalog Support [iceberg]

Posted by "amogh-jahagirdar (via GitHub)" <gi...@apache.org>.
amogh-jahagirdar commented on code in PR #12808:
URL: https://github.com/apache/iceberg/pull/12808#discussion_r2052650701


##########
bigquery/src/main/java/org/apache/iceberg/gcp/bigquery/BigQueryMetastoreCatalog.java:
##########
@@ -0,0 +1,402 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.iceberg.gcp.bigquery;
+
+import com.google.api.services.bigquery.model.Dataset;
+import com.google.api.services.bigquery.model.DatasetList.Datasets;
+import com.google.api.services.bigquery.model.DatasetReference;
+import com.google.api.services.bigquery.model.ExternalCatalogDatasetOptions;
+import com.google.api.services.bigquery.model.TableReference;
+import com.google.cloud.ServiceOptions;
+import com.google.cloud.bigquery.BigQueryOptions;
+import java.io.IOException;
+import java.security.GeneralSecurityException;
+import java.util.List;
+import java.util.Locale;
+import java.util.Map;
+import java.util.Objects;
+import java.util.Set;
+import org.apache.hadoop.conf.Configurable;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.iceberg.BaseMetastoreCatalog;
+import org.apache.iceberg.CatalogProperties;
+import org.apache.iceberg.CatalogUtil;
+import org.apache.iceberg.TableMetadata;
+import org.apache.iceberg.TableOperations;
+import org.apache.iceberg.catalog.Namespace;
+import org.apache.iceberg.catalog.SupportsNamespaces;
+import org.apache.iceberg.catalog.TableIdentifier;
+import org.apache.iceberg.exceptions.AlreadyExistsException;
+import org.apache.iceberg.exceptions.NoSuchNamespaceException;
+import org.apache.iceberg.exceptions.NoSuchTableException;
+import org.apache.iceberg.exceptions.ServiceFailureException;
+import org.apache.iceberg.exceptions.ValidationException;
+import org.apache.iceberg.io.FileIO;
+import org.apache.iceberg.relocated.com.google.common.annotations.VisibleForTesting;
+import org.apache.iceberg.relocated.com.google.common.base.Preconditions;
+import org.apache.iceberg.relocated.com.google.common.base.Strings;
+import org.apache.iceberg.relocated.com.google.common.collect.ImmutableList;
+import org.apache.iceberg.relocated.com.google.common.collect.ImmutableMap;
+import org.apache.iceberg.relocated.com.google.common.collect.Maps;
+import org.apache.iceberg.util.LocationUtil;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/** Iceberg Bigquery Metastore Catalog implementation. */
+public final class BigQueryMetastoreCatalog extends BaseMetastoreCatalog
+    implements SupportsNamespaces, Configurable {
+
+  // User provided properties.
+  public static final String PROPERTIES_KEY_GCP_PROJECT = "gcp-project";
+  public static final String PROPERTIES_KEY_GCP_LOCATION = "gcp-location";
+  public static final String PROPERTIES_KEY_FILTER_UNSUPPORTED_TABLES = "filter-unsupported-tables";

Review Comment:
   Should these be in GCPProperties? 



##########
bigquery/src/main/java/org/apache/iceberg/gcp/bigquery/BigQueryMetastoreCatalog.java:
##########
@@ -0,0 +1,402 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.iceberg.gcp.bigquery;
+
+import com.google.api.services.bigquery.model.Dataset;
+import com.google.api.services.bigquery.model.DatasetList.Datasets;
+import com.google.api.services.bigquery.model.DatasetReference;
+import com.google.api.services.bigquery.model.ExternalCatalogDatasetOptions;
+import com.google.api.services.bigquery.model.TableReference;
+import com.google.cloud.ServiceOptions;
+import com.google.cloud.bigquery.BigQueryOptions;
+import java.io.IOException;
+import java.security.GeneralSecurityException;
+import java.util.List;
+import java.util.Locale;
+import java.util.Map;
+import java.util.Objects;
+import java.util.Set;
+import org.apache.hadoop.conf.Configurable;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.iceberg.BaseMetastoreCatalog;
+import org.apache.iceberg.CatalogProperties;
+import org.apache.iceberg.CatalogUtil;
+import org.apache.iceberg.TableMetadata;
+import org.apache.iceberg.TableOperations;
+import org.apache.iceberg.catalog.Namespace;
+import org.apache.iceberg.catalog.SupportsNamespaces;
+import org.apache.iceberg.catalog.TableIdentifier;
+import org.apache.iceberg.exceptions.AlreadyExistsException;
+import org.apache.iceberg.exceptions.NoSuchNamespaceException;
+import org.apache.iceberg.exceptions.NoSuchTableException;
+import org.apache.iceberg.exceptions.ServiceFailureException;
+import org.apache.iceberg.exceptions.ValidationException;
+import org.apache.iceberg.io.FileIO;
+import org.apache.iceberg.relocated.com.google.common.annotations.VisibleForTesting;
+import org.apache.iceberg.relocated.com.google.common.base.Preconditions;
+import org.apache.iceberg.relocated.com.google.common.base.Strings;
+import org.apache.iceberg.relocated.com.google.common.collect.ImmutableList;
+import org.apache.iceberg.relocated.com.google.common.collect.ImmutableMap;
+import org.apache.iceberg.relocated.com.google.common.collect.Maps;
+import org.apache.iceberg.util.LocationUtil;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/** Iceberg Bigquery Metastore Catalog implementation. */
+public final class BigQueryMetastoreCatalog extends BaseMetastoreCatalog
+    implements SupportsNamespaces, Configurable {
+
+  // User provided properties.
+  public static final String PROPERTIES_KEY_GCP_PROJECT = "gcp-project";
+  public static final String PROPERTIES_KEY_GCP_LOCATION = "gcp-location";
+  public static final String PROPERTIES_KEY_FILTER_UNSUPPORTED_TABLES = "filter-unsupported-tables";
+  public static final String PROPERTIES_KEY_TESTING_ENABLED = "testing-enabled";
+
+  public static final String HIVE_METASTORE_WAREHOUSE_DIR = "hive.metastore.warehouse.dir";
+
+  private static final Logger LOG = LoggerFactory.getLogger(BigQueryMetastoreCatalog.class);
+
+  private static final String DEFAULT_GCP_LOCATION = "us";
+
+  private String catalogPluginName;
+  private Map<String, String> catalogProperties;
+  private FileIO fileIO;
+  private Configuration conf;
+  private String projectId;
+  private String location;
+  private BigQueryMetastoreClient client;
+  private boolean filterUnsupportedTables;
+
+  // Must have a no-arg constructor to be dynamically loaded
+  // initialize(String name, Map<String, String> properties) will be called to complete
+  // initialization
+  public BigQueryMetastoreCatalog() {}
+
+  @Override
+  public void initialize(String inputName, Map<String, String> properties) {
+    if (!properties.containsKey(PROPERTIES_KEY_GCP_PROJECT)) {
+      throw new ValidationException("GCP project must be specified");
+    }
+    projectId = properties.get(PROPERTIES_KEY_GCP_PROJECT);
+    location = properties.getOrDefault(PROPERTIES_KEY_GCP_LOCATION, DEFAULT_GCP_LOCATION);
+    boolean testingEnabled =
+        Boolean.parseBoolean(properties.getOrDefault(PROPERTIES_KEY_TESTING_ENABLED, "false"));
+
+    BigQueryOptions options =
+        BigQueryOptions.newBuilder()
+            .setProjectId(projectId)
+            .setLocation(location)
+            .setRetrySettings(ServiceOptions.getDefaultRetrySettings())
+            .build();
+
+    try {
+      if (testingEnabled) {
+        client = new FakeBigQueryMetastoreClient(options);

Review Comment:
   Seems like we do that below, do we need the testingEnabled flag then? Doesn't seem like we should need it



##########
bigquery/src/main/java/org/apache/iceberg/gcp/bigquery/BigQueryMetastoreCatalog.java:
##########
@@ -0,0 +1,402 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.iceberg.gcp.bigquery;
+
+import com.google.api.services.bigquery.model.Dataset;
+import com.google.api.services.bigquery.model.DatasetList.Datasets;
+import com.google.api.services.bigquery.model.DatasetReference;
+import com.google.api.services.bigquery.model.ExternalCatalogDatasetOptions;
+import com.google.api.services.bigquery.model.TableReference;
+import com.google.cloud.ServiceOptions;
+import com.google.cloud.bigquery.BigQueryOptions;
+import java.io.IOException;
+import java.security.GeneralSecurityException;
+import java.util.List;
+import java.util.Locale;
+import java.util.Map;
+import java.util.Objects;
+import java.util.Set;
+import org.apache.hadoop.conf.Configurable;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.iceberg.BaseMetastoreCatalog;
+import org.apache.iceberg.CatalogProperties;
+import org.apache.iceberg.CatalogUtil;
+import org.apache.iceberg.TableMetadata;
+import org.apache.iceberg.TableOperations;
+import org.apache.iceberg.catalog.Namespace;
+import org.apache.iceberg.catalog.SupportsNamespaces;
+import org.apache.iceberg.catalog.TableIdentifier;
+import org.apache.iceberg.exceptions.AlreadyExistsException;
+import org.apache.iceberg.exceptions.NoSuchNamespaceException;
+import org.apache.iceberg.exceptions.NoSuchTableException;
+import org.apache.iceberg.exceptions.ServiceFailureException;
+import org.apache.iceberg.exceptions.ValidationException;
+import org.apache.iceberg.io.FileIO;
+import org.apache.iceberg.relocated.com.google.common.annotations.VisibleForTesting;
+import org.apache.iceberg.relocated.com.google.common.base.Preconditions;
+import org.apache.iceberg.relocated.com.google.common.base.Strings;
+import org.apache.iceberg.relocated.com.google.common.collect.ImmutableList;
+import org.apache.iceberg.relocated.com.google.common.collect.ImmutableMap;
+import org.apache.iceberg.relocated.com.google.common.collect.Maps;
+import org.apache.iceberg.util.LocationUtil;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/** Iceberg Bigquery Metastore Catalog implementation. */
+public final class BigQueryMetastoreCatalog extends BaseMetastoreCatalog
+    implements SupportsNamespaces, Configurable {
+
+  // User provided properties.
+  public static final String PROPERTIES_KEY_GCP_PROJECT = "gcp-project";
+  public static final String PROPERTIES_KEY_GCP_LOCATION = "gcp-location";
+  public static final String PROPERTIES_KEY_FILTER_UNSUPPORTED_TABLES = "filter-unsupported-tables";
+  public static final String PROPERTIES_KEY_TESTING_ENABLED = "testing-enabled";
+
+  public static final String HIVE_METASTORE_WAREHOUSE_DIR = "hive.metastore.warehouse.dir";
+
+  private static final Logger LOG = LoggerFactory.getLogger(BigQueryMetastoreCatalog.class);
+
+  private static final String DEFAULT_GCP_LOCATION = "us";
+
+  private String catalogPluginName;
+  private Map<String, String> catalogProperties;
+  private FileIO fileIO;
+  private Configuration conf;
+  private String projectId;
+  private String location;
+  private BigQueryMetastoreClient client;
+  private boolean filterUnsupportedTables;
+
+  // Must have a no-arg constructor to be dynamically loaded
+  // initialize(String name, Map<String, String> properties) will be called to complete
+  // initialization
+  public BigQueryMetastoreCatalog() {}
+
+  @Override
+  public void initialize(String inputName, Map<String, String> properties) {
+    if (!properties.containsKey(PROPERTIES_KEY_GCP_PROJECT)) {
+      throw new ValidationException("GCP project must be specified");
+    }
+    projectId = properties.get(PROPERTIES_KEY_GCP_PROJECT);
+    location = properties.getOrDefault(PROPERTIES_KEY_GCP_LOCATION, DEFAULT_GCP_LOCATION);
+    boolean testingEnabled =
+        Boolean.parseBoolean(properties.getOrDefault(PROPERTIES_KEY_TESTING_ENABLED, "false"));
+
+    BigQueryOptions options =
+        BigQueryOptions.newBuilder()
+            .setProjectId(projectId)
+            .setLocation(location)
+            .setRetrySettings(ServiceOptions.getDefaultRetrySettings())
+            .build();
+
+    try {
+      if (testingEnabled) {
+        client = new FakeBigQueryMetastoreClient(options);

Review Comment:
   Can we follow a similar pattern to what the `GlueCatalog` implementation does? We can have an initialize whihc accepts the client, and for testing purposes we pass in a mocked client. Instead of having to handle some "testing-property" and branch off of that



-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org

For queries about this service, please contact Infrastructure at:
users@infra.apache.org


---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For additional commands, e-mail: issues-help@iceberg.apache.org


Re: [PR] Catalog: Add BigQuery Metastore Catalog Support [iceberg]

Posted by "talatuyarer (via GitHub)" <gi...@apache.org>.
talatuyarer commented on code in PR #12808:
URL: https://github.com/apache/iceberg/pull/12808#discussion_r2059693384


##########
bigquery/src/main/java/org/apache/iceberg/gcp/bigquery/BigQueryMetastoreClient.java:
##########
@@ -0,0 +1,121 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.iceberg.gcp.bigquery;
+
+import com.google.api.services.bigquery.model.Dataset;
+import com.google.api.services.bigquery.model.DatasetList.Datasets;
+import com.google.api.services.bigquery.model.DatasetReference;
+import com.google.api.services.bigquery.model.Table;
+import com.google.api.services.bigquery.model.TableList.Tables;
+import com.google.api.services.bigquery.model.TableReference;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+
+/**
+ * A client of Google BigQuery Metastore functions over the BigQuery service. Uses the Google
+ * BigQuery API.
+ */
+public interface BigQueryMetastoreClient {
+
+  /**
+   * Creates and returns a new dataset.
+   *
+   * @param dataset the dataset to create
+   */
+  Dataset createDataset(Dataset dataset);

Review Comment:
   Changed according your comment



##########
bigquery/src/main/java/org/apache/iceberg/gcp/bigquery/BigQueryMetastoreClient.java:
##########
@@ -0,0 +1,121 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.iceberg.gcp.bigquery;
+
+import com.google.api.services.bigquery.model.Dataset;
+import com.google.api.services.bigquery.model.DatasetList.Datasets;
+import com.google.api.services.bigquery.model.DatasetReference;
+import com.google.api.services.bigquery.model.Table;
+import com.google.api.services.bigquery.model.TableList.Tables;
+import com.google.api.services.bigquery.model.TableReference;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+
+/**
+ * A client of Google BigQuery Metastore functions over the BigQuery service. Uses the Google
+ * BigQuery API.
+ */
+public interface BigQueryMetastoreClient {
+
+  /**
+   * Creates and returns a new dataset.
+   *
+   * @param dataset the dataset to create
+   */
+  Dataset createDataset(Dataset dataset);
+
+  /**
+   * Returns a dataset.
+   *
+   * @param datasetReference full dataset reference
+   */
+  Dataset getDataset(DatasetReference datasetReference);
+
+  /**
+   * Deletes a dataset.
+   *
+   * @param datasetReference full dataset reference
+   */
+  void deleteDataset(DatasetReference datasetReference);

Review Comment:
   Done



-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org

For queries about this service, please contact Infrastructure at:
users@infra.apache.org


---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For additional commands, e-mail: issues-help@iceberg.apache.org


Re: [PR] Catalog: Add BigQuery Metastore Catalog Support [iceberg]

Posted by "talatuyarer (via GitHub)" <gi...@apache.org>.
talatuyarer commented on code in PR #12808:
URL: https://github.com/apache/iceberg/pull/12808#discussion_r2059830145


##########
bigquery/src/main/java/org/apache/iceberg/gcp/bigquery/BigQueryMetastoreCatalog.java:
##########
@@ -0,0 +1,400 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.iceberg.gcp.bigquery;
+
+import com.google.api.services.bigquery.model.Dataset;
+import com.google.api.services.bigquery.model.DatasetList.Datasets;
+import com.google.api.services.bigquery.model.DatasetReference;
+import com.google.api.services.bigquery.model.ExternalCatalogDatasetOptions;
+import com.google.api.services.bigquery.model.TableReference;
+import com.google.cloud.ServiceOptions;
+import com.google.cloud.bigquery.BigQueryOptions;
+import java.io.IOException;
+import java.security.GeneralSecurityException;
+import java.util.List;
+import java.util.Locale;
+import java.util.Map;
+import java.util.Objects;
+import java.util.Set;
+import org.apache.hadoop.conf.Configurable;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.iceberg.BaseMetastoreCatalog;
+import org.apache.iceberg.CatalogProperties;
+import org.apache.iceberg.CatalogUtil;
+import org.apache.iceberg.TableMetadata;
+import org.apache.iceberg.TableOperations;
+import org.apache.iceberg.catalog.Namespace;
+import org.apache.iceberg.catalog.SupportsNamespaces;
+import org.apache.iceberg.catalog.TableIdentifier;
+import org.apache.iceberg.exceptions.AlreadyExistsException;
+import org.apache.iceberg.exceptions.NoSuchNamespaceException;
+import org.apache.iceberg.exceptions.NoSuchTableException;
+import org.apache.iceberg.exceptions.ServiceFailureException;
+import org.apache.iceberg.exceptions.ValidationException;
+import org.apache.iceberg.io.FileIO;
+import org.apache.iceberg.relocated.com.google.common.annotations.VisibleForTesting;
+import org.apache.iceberg.relocated.com.google.common.base.Preconditions;
+import org.apache.iceberg.relocated.com.google.common.base.Strings;
+import org.apache.iceberg.relocated.com.google.common.collect.ImmutableList;
+import org.apache.iceberg.relocated.com.google.common.collect.ImmutableMap;
+import org.apache.iceberg.relocated.com.google.common.collect.Maps;
+import org.apache.iceberg.util.LocationUtil;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/** Iceberg Bigquery Metastore Catalog implementation. */
+public final class BigQueryMetastoreCatalog extends BaseMetastoreCatalog
+    implements SupportsNamespaces, Configurable {
+
+  // User provided properties.
+  public static final String PROJECT_ID = "gcp-project";
+  public static final String GCP_LOCATION = "gcp-location";
+  public static final String FILTER_UNSUPPORTED_TABLES = "filter-unsupported-tables";

Review Comment:
   Done



-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org

For queries about this service, please contact Infrastructure at:
users@infra.apache.org


---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For additional commands, e-mail: issues-help@iceberg.apache.org


Re: [PR] Catalog: Add BigQuery Metastore Catalog Support [iceberg]

Posted by "nastra (via GitHub)" <gi...@apache.org>.
nastra commented on code in PR #12808:
URL: https://github.com/apache/iceberg/pull/12808#discussion_r2059734346


##########
build.gradle:
##########
@@ -657,6 +657,47 @@ project(':iceberg-delta-lake') {
   }
 }
 
+project(':iceberg-bigquery') {
+  test {
+    useJUnitPlatform()
+  }
+
+  dependencies {
+    implementation project(path: ':iceberg-bundled-guava', configuration: 'shadow')
+    implementation project(':iceberg-gcp')
+
+    implementation platform(libs.google.libraries.bom)
+    compileOnly "com.google.cloud:google-cloud-storage"
+    implementation "com.google.cloud:google-cloud-bigquery"
+    implementation "com.google.cloud:google-cloud-core"
+
+    api project(':iceberg-api')
+    implementation project(':iceberg-common')
+    implementation project(':iceberg-core')
+
+    compileOnly libs.hadoop3.common
+    compileOnly (libs.hive4.metastore) {
+      exclude group: 'org.apache.avro', module: 'avro'
+      exclude group: 'org.slf4j', module: 'slf4j-log4j12'
+      exclude group: 'org.pentaho'
+      exclude group: 'org.apache.hbase'
+      exclude group: 'org.apache.logging.log4j'
+      exclude group: 'co.cask.tephra'
+      exclude group: 'com.google.code.findbugs', module: 'jsr305'
+      exclude group: 'org.eclipse.jetty.aggregate', module: 'jetty-all'
+      exclude group: 'org.eclipse.jetty.orbit', module: 'javax.servlet'
+      exclude group: 'org.apache.parquet', module: 'parquet-hadoop-bundle'
+      exclude group: 'com.tdunning', module: 'json'
+      exclude group: 'javax.transaction', module: 'transaction-api'
+      exclude group: 'com.zaxxer', module: 'HikariCP'
+      exclude group: 'com.google.code.gson', module: 'gson'
+  }

Review Comment:
   looks like indentation is off here



-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org

For queries about this service, please contact Infrastructure at:
users@infra.apache.org


---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For additional commands, e-mail: issues-help@iceberg.apache.org


Re: [PR] Catalog: Add BigQuery Metastore Catalog Support [iceberg]

Posted by "nastra (via GitHub)" <gi...@apache.org>.
nastra commented on code in PR #12808:
URL: https://github.com/apache/iceberg/pull/12808#discussion_r2063947252


##########
build.gradle:
##########
@@ -657,6 +657,47 @@ project(':iceberg-delta-lake') {
   }
 }
 
+project(':iceberg-bigquery') {
+  test {
+    useJUnitPlatform()
+  }
+
+  dependencies {
+    implementation project(path: ':iceberg-bundled-guava', configuration: 'shadow')
+    implementation project(':iceberg-gcp')
+
+    implementation platform(libs.google.libraries.bom)
+    compileOnly "com.google.cloud:google-cloud-storage"
+    implementation "com.google.cloud:google-cloud-bigquery"
+    implementation "com.google.cloud:google-cloud-core"
+
+    api project(':iceberg-api')

Review Comment:
   nit: you might want to move this and other project dependencies a few lines further above so that project dependencies are grouped together



-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org

For queries about this service, please contact Infrastructure at:
users@infra.apache.org


---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For additional commands, e-mail: issues-help@iceberg.apache.org


Re: [PR] Catalog: Add BigQuery Metastore Catalog Support [iceberg]

Posted by "nastra (via GitHub)" <gi...@apache.org>.
nastra commented on code in PR #12808:
URL: https://github.com/apache/iceberg/pull/12808#discussion_r2063965579


##########
bigquery/src/main/java/org/apache/iceberg/gcp/bigquery/BigQueryMetastoreCatalog.java:
##########
@@ -0,0 +1,394 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.iceberg.gcp.bigquery;
+
+import com.google.api.services.bigquery.model.Dataset;
+import com.google.api.services.bigquery.model.DatasetList.Datasets;
+import com.google.api.services.bigquery.model.DatasetReference;
+import com.google.api.services.bigquery.model.ExternalCatalogDatasetOptions;
+import com.google.api.services.bigquery.model.TableReference;
+import com.google.cloud.ServiceOptions;
+import com.google.cloud.bigquery.BigQueryOptions;
+import java.io.IOException;
+import java.security.GeneralSecurityException;
+import java.util.List;
+import java.util.Locale;
+import java.util.Map;
+import java.util.Objects;
+import java.util.Set;
+import org.apache.hadoop.conf.Configurable;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.iceberg.BaseMetastoreCatalog;
+import org.apache.iceberg.CatalogProperties;
+import org.apache.iceberg.CatalogUtil;
+import org.apache.iceberg.TableMetadata;
+import org.apache.iceberg.TableOperations;
+import org.apache.iceberg.catalog.Namespace;
+import org.apache.iceberg.catalog.SupportsNamespaces;
+import org.apache.iceberg.catalog.TableIdentifier;
+import org.apache.iceberg.exceptions.AlreadyExistsException;
+import org.apache.iceberg.exceptions.NoSuchNamespaceException;
+import org.apache.iceberg.exceptions.NoSuchTableException;
+import org.apache.iceberg.exceptions.ServiceFailureException;
+import org.apache.iceberg.exceptions.ValidationException;
+import org.apache.iceberg.io.FileIO;
+import org.apache.iceberg.relocated.com.google.common.annotations.VisibleForTesting;
+import org.apache.iceberg.relocated.com.google.common.base.Preconditions;
+import org.apache.iceberg.relocated.com.google.common.base.Strings;
+import org.apache.iceberg.relocated.com.google.common.collect.ImmutableList;
+import org.apache.iceberg.relocated.com.google.common.collect.ImmutableMap;
+import org.apache.iceberg.relocated.com.google.common.collect.Maps;
+import org.apache.iceberg.util.LocationUtil;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/** Iceberg Bigquery Metastore Catalog implementation. */
+public class BigQueryMetastoreCatalog extends BaseMetastoreCatalog
+    implements SupportsNamespaces, Configurable {
+
+  // User provided properties.
+  public static final String PROJECT_ID = "gcp-project";
+  public static final String GCP_LOCATION = "gcp-location";
+  public static final String LIST_ALL_TABLES = "list-all-tables";
+
+  public static final String HIVE_METASTORE_WAREHOUSE_DIR = "hive.metastore.warehouse.dir";
+
+  private static final Logger LOG = LoggerFactory.getLogger(BigQueryMetastoreCatalog.class);
+
+  private static final String DEFAULT_GCP_LOCATION = "us";
+
+  private String catalogName;
+  private Map<String, String> catalogProperties;
+  private FileIO fileIO;
+  private Configuration conf;
+  private String projectId;
+  private String location;
+  private BigQueryMetastoreClient client;
+  private boolean listAllTables;
+
+  // Must have a no-arg constructor to be dynamically loaded
+  // initialize(String name, Map<String, String> properties) will be called to complete
+  // initialization
+  public BigQueryMetastoreCatalog() {}
+
+  @Override
+  public void initialize(String inputName, Map<String, String> properties) {
+    Preconditions.checkArgument(
+        properties.containsKey(PROJECT_ID), "GCP project must be specified");
+
+    projectId = properties.get(PROJECT_ID);
+    location = properties.getOrDefault(GCP_LOCATION, DEFAULT_GCP_LOCATION);
+
+    BigQueryOptions options =
+        BigQueryOptions.newBuilder()
+            .setProjectId(projectId)
+            .setLocation(location)
+            .setRetrySettings(ServiceOptions.getDefaultRetrySettings())
+            .build();
+
+    try {
+      client = new BigQueryMetastoreClientImpl(options);
+    } catch (IOException e) {
+      throw new ServiceFailureException(e, "Creating BigQuery client failed");
+    } catch (GeneralSecurityException e) {
+      throw new ValidationException(e, "Creating BigQuery client failed due to a security issue");
+    }
+
+    initialize(inputName, properties, projectId, location, client);
+  }
+
+  @VisibleForTesting
+  void initialize(
+      String inputName,
+      Map<String, String> properties,
+      String initialProjectId,
+      String initialLocation,
+      BigQueryMetastoreClient bigQueryMetaStoreClient) {
+    Preconditions.checkArgument(bigQueryMetaStoreClient != null, "BigQuery client can not be null");

Review Comment:
   ```suggestion
       Preconditions.checkArgument(bigQueryMetaStoreClient != null, "Invalid BigQuery client: null");
   ```
   this follows the error messaging pattern in the existing codebase



-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org

For queries about this service, please contact Infrastructure at:
users@infra.apache.org


---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For additional commands, e-mail: issues-help@iceberg.apache.org


Re: [PR] Catalog: Add BigQuery Metastore Catalog Support [iceberg]

Posted by "nastra (via GitHub)" <gi...@apache.org>.
nastra commented on code in PR #12808:
URL: https://github.com/apache/iceberg/pull/12808#discussion_r2066006593


##########
bigquery/src/main/java/org/apache/iceberg/gcp/bigquery/BigQueryMetastoreClientImpl.java:
##########
@@ -0,0 +1,594 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.iceberg.gcp.bigquery;
+
+import com.google.api.client.googleapis.javanet.GoogleNetHttpTransport;
+import com.google.api.client.http.HttpHeaders;
+import com.google.api.client.http.HttpResponse;
+import com.google.api.client.http.HttpResponseException;
+import com.google.api.client.http.HttpStatusCodes;
+import com.google.api.client.json.gson.GsonFactory;
+import com.google.api.client.util.Data;
+import com.google.api.services.bigquery.Bigquery;
+import com.google.api.services.bigquery.BigqueryScopes;
+import com.google.api.services.bigquery.model.Dataset;
+import com.google.api.services.bigquery.model.DatasetList;
+import com.google.api.services.bigquery.model.DatasetList.Datasets;
+import com.google.api.services.bigquery.model.DatasetReference;
+import com.google.api.services.bigquery.model.ExternalCatalogDatasetOptions;
+import com.google.api.services.bigquery.model.ExternalCatalogTableOptions;
+import com.google.api.services.bigquery.model.Table;
+import com.google.api.services.bigquery.model.TableList;
+import com.google.api.services.bigquery.model.TableList.Tables;
+import com.google.api.services.bigquery.model.TableReference;
+import com.google.api.services.bigquery.model.TableSchema;
+import com.google.auth.http.HttpCredentialsAdapter;
+import com.google.auth.oauth2.GoogleCredentials;
+import com.google.cloud.BaseServiceException;
+import com.google.cloud.ExceptionHandler;
+import com.google.cloud.bigquery.BigQueryErrorMessages;
+import com.google.cloud.bigquery.BigQueryOptions;
+import com.google.cloud.bigquery.BigQueryRetryConfig;
+import com.google.cloud.bigquery.BigQueryRetryHelper;
+import java.io.IOException;
+import java.nio.charset.StandardCharsets;
+import java.security.GeneralSecurityException;
+import java.util.List;
+import java.util.Locale;
+import java.util.Map;
+import java.util.Set;
+import java.util.stream.Collectors;
+import java.util.stream.Stream;
+import org.apache.iceberg.BaseMetastoreTableOperations;
+import org.apache.iceberg.exceptions.AlreadyExistsException;
+import org.apache.iceberg.exceptions.BadRequestException;
+import org.apache.iceberg.exceptions.ForbiddenException;
+import org.apache.iceberg.exceptions.NoSuchIcebergTableException;
+import org.apache.iceberg.exceptions.NoSuchNamespaceException;
+import org.apache.iceberg.exceptions.NoSuchTableException;
+import org.apache.iceberg.exceptions.NotAuthorizedException;
+import org.apache.iceberg.exceptions.RuntimeIOException;
+import org.apache.iceberg.exceptions.ServiceFailureException;
+import org.apache.iceberg.exceptions.ServiceUnavailableException;
+import org.apache.iceberg.exceptions.ValidationException;
+import org.apache.iceberg.relocated.com.google.common.base.Preconditions;
+import org.apache.iceberg.relocated.com.google.common.collect.Lists;
+import org.apache.iceberg.relocated.com.google.common.collect.Maps;
+
+/** A client of Google Bigquery Metastore functions over the BigQuery service. */
+public final class BigQueryMetastoreClientImpl implements BigQueryMetastoreClient {
+
+  public static final ExceptionHandler.Interceptor EXCEPTION_HANDLER_INTERCEPTOR =
+      new ExceptionHandler.Interceptor() {
+
+        private static final long serialVersionUID = -8429573486870467828L;
+
+        @Override
+        public RetryResult afterEval(Exception exception, RetryResult retryResult) {
+          return ExceptionHandler.Interceptor.RetryResult.CONTINUE_EVALUATION;
+        }
+
+        @Override
+        public RetryResult beforeEval(Exception exception) {
+          if (exception instanceof BaseServiceException) {
+            boolean retriable = ((BaseServiceException) exception).isRetryable();
+            return retriable
+                ? ExceptionHandler.Interceptor.RetryResult.RETRY
+                : ExceptionHandler.Interceptor.RetryResult.CONTINUE_EVALUATION;
+          }
+
+          return ExceptionHandler.Interceptor.RetryResult.CONTINUE_EVALUATION;
+        }
+      };
+
+  private final Bigquery client;
+  private final BigQueryOptions bigqueryOptions;
+
+  // Retry config with error messages and regex for rate limit exceeded errors.
+  private static final BigQueryRetryConfig DEFAULT_RETRY_CONFIG =
+      BigQueryRetryConfig.newBuilder()
+          .retryOnMessage(BigQueryErrorMessages.RATE_LIMIT_EXCEEDED_MSG)
+          .retryOnMessage(BigQueryErrorMessages.JOB_RATE_LIMIT_EXCEEDED_MSG)
+          .retryOnRegEx(BigQueryErrorMessages.RetryRegExPatterns.RATE_LIMIT_EXCEEDED_REGEX)
+          .build();
+
+  public static final ExceptionHandler BIGQUERY_EXCEPTION_HANDLER =
+      ExceptionHandler.newBuilder()
+          .abortOn(RuntimeException.class)
+          .retryOn(
+              java.net.ConnectException
+                  .class) // Retry on connection failures due to transient network issues.

Review Comment:
   the comment causes the code to be weirdly formatted, maybe just move it right before the `.retryOn`. Same for the comment a few lines below



-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org

For queries about this service, please contact Infrastructure at:
users@infra.apache.org


---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For additional commands, e-mail: issues-help@iceberg.apache.org


Re: [PR] Catalog: Add BigQuery Metastore Catalog Support [iceberg]

Posted by "nastra (via GitHub)" <gi...@apache.org>.
nastra commented on code in PR #12808:
URL: https://github.com/apache/iceberg/pull/12808#discussion_r2066066709


##########
bigquery/src/main/java/org/apache/iceberg/gcp/bigquery/BigQueryTableOperations.java:
##########
@@ -0,0 +1,297 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.iceberg.gcp.bigquery;
+
+import com.google.api.services.bigquery.model.ExternalCatalogTableOptions;
+import com.google.api.services.bigquery.model.Table;
+import com.google.api.services.bigquery.model.TableReference;
+import java.util.Locale;
+import java.util.Map;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hive.common.StatsSetupConst;
+import org.apache.iceberg.BaseMetastoreOperations;
+import org.apache.iceberg.BaseMetastoreTableOperations;
+import org.apache.iceberg.SnapshotSummary;
+import org.apache.iceberg.TableMetadata;
+import org.apache.iceberg.TableProperties;
+import org.apache.iceberg.exceptions.AlreadyExistsException;
+import org.apache.iceberg.exceptions.CommitFailedException;
+import org.apache.iceberg.exceptions.CommitStateUnknownException;
+import org.apache.iceberg.exceptions.NoSuchTableException;
+import org.apache.iceberg.exceptions.ValidationException;
+import org.apache.iceberg.hadoop.ConfigProperties;
+import org.apache.iceberg.io.FileIO;
+import org.apache.iceberg.relocated.com.google.common.collect.Maps;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/** Handles BigQuery metastore table operations. */
+public final class BigQueryTableOperations extends BaseMetastoreTableOperations {
+
+  private static final Logger LOG = LoggerFactory.getLogger(BigQueryTableOperations.class);
+
+  private static final String TABLE_PROPERTIES_BQ_CONNECTION = "bq_connection";
+
+  private final BigQueryMetastoreClient client;
+  private final FileIO fileIO;
+  private final TableReference tableReference;
+  private final Configuration conf;
+
+  BigQueryTableOperations(
+      BigQueryMetastoreClient client,
+      FileIO fileIO,
+      TableReference tableReference,
+      Configuration conf) {
+    this.client = client;
+    this.fileIO = fileIO;
+    this.tableReference = tableReference;
+    this.conf = conf;
+  }
+
+  // The doRefresh method should provide implementation on how to get the metadata location.
+  @Override
+  public void doRefresh() {
+    // Must default to null.
+    String metadataLocation = null;
+    try {
+      metadataLocation =
+          getMetadataLocationOrThrow(
+              client.load(this.tableReference).getExternalCatalogTableOptions());
+    } catch (NoSuchTableException e) {
+      if (currentMetadataLocation() != null) {
+        // Re-throws the exception because the table must exist in this case.
+        throw e;
+      }
+    }
+    refreshFromMetadataLocation(metadataLocation);
+  }
+
+  // The doCommit method should provide implementation on how to update with metadata location
+  // atomically
+  @Override
+  public void doCommit(TableMetadata base, TableMetadata metadata) {
+    String newMetadataLocation =
+        base == null && metadata.metadataFileLocation() != null
+            ? metadata.metadataFileLocation()
+            : writeNewMetadata(metadata, currentVersion() + 1);
+    BaseMetastoreOperations.CommitStatus commitStatus =
+        BaseMetastoreOperations.CommitStatus.FAILURE;
+    try {
+      if (base == null) {
+        createTable(newMetadataLocation, metadata);
+      } else {
+        updateTable(base.metadataFileLocation(), newMetadataLocation, metadata);
+      }
+      commitStatus = BaseMetastoreOperations.CommitStatus.SUCCESS;
+    } catch (CommitFailedException | CommitStateUnknownException e) {
+      throw e;
+    } catch (Throwable e) {
+      LOG.error("Exception thrown on commit: ", e);
+      if (e instanceof AlreadyExistsException) {
+        throw e;
+      }
+      commitStatus =
+          BaseMetastoreOperations.CommitStatus.valueOf(
+              checkCommitStatus(newMetadataLocation, metadata).name());
+      if (commitStatus == BaseMetastoreOperations.CommitStatus.FAILURE) {
+        throw new CommitFailedException(e, "Failed to commit");
+      }
+      if (commitStatus == BaseMetastoreOperations.CommitStatus.UNKNOWN) {
+        throw new CommitStateUnknownException(e);
+      }
+    } finally {
+      try {
+        if (commitStatus == BaseMetastoreOperations.CommitStatus.FAILURE) {
+          LOG.warn("Failed to commit updates to table {}", tableName());
+          io().deleteFile(newMetadataLocation);
+        }
+      } catch (RuntimeException e) {
+        LOG.error(
+            "Failed to cleanup metadata file at {} for table {}",
+            newMetadataLocation,
+            tableName(),
+            e);
+      }
+    }
+  }
+
+  @Override
+  public String tableName() {
+    return String.format("%s.%s", tableReference.getDatasetId(), tableReference.getTableId());
+  }
+
+  @Override
+  public FileIO io() {
+    return fileIO;
+  }
+
+  private void createTable(String newMetadataLocation, TableMetadata metadata) {
+    LOG.debug("Creating a new Iceberg table: {}", tableName());
+    Table tableBuilder = makeNewTable(metadata, newMetadataLocation);
+    tableBuilder.setTableReference(this.tableReference);
+    addConnectionIfProvided(tableBuilder, metadata.properties());
+
+    client.create(tableBuilder);
+  }
+
+  private void addConnectionIfProvided(Table tableBuilder, Map<String, String> metadataProperties) {
+    if (metadataProperties.containsKey(TABLE_PROPERTIES_BQ_CONNECTION)) {
+      tableBuilder
+          .getExternalCatalogTableOptions()
+          .setConnectionId(metadataProperties.get(TABLE_PROPERTIES_BQ_CONNECTION));
+    }
+  }
+
+  /** Update table properties with concurrent update detection using etag. */
+  private void updateTable(
+      String oldMetadataLocation, String newMetadataLocation, TableMetadata metadata) {
+    Table table = client.load(this.tableReference);
+    if (table.getEtag().isEmpty()) {
+      throw new ValidationException(
+          "Etag of legacy table %s is empty, manually update the table via the BigQuery API or"
+              + " recreate and retry",
+          tableName());
+    }
+    ExternalCatalogTableOptions options = table.getExternalCatalogTableOptions();
+    addConnectionIfProvided(table, metadata.properties());
+
+    // If `metadataLocationFromMetastore` is different from metadata location of base, it means
+    // someone has updated metadata location in metastore, which is a conflict update.
+    String metadataLocationFromMetastore =
+        options.getParameters().getOrDefault(METADATA_LOCATION_PROP, "");
+    if (!metadataLocationFromMetastore.isEmpty()
+        && !metadataLocationFromMetastore.equals(oldMetadataLocation)) {
+      throw new CommitFailedException(
+          "Cannot commit base metadata location '%s' is not same as the current table metadata location '%s' for"
+              + " %s.%s",
+          oldMetadataLocation,
+          metadataLocationFromMetastore,
+          tableReference.getDatasetId(),
+          tableReference.getTableId());
+    }
+
+    options.setParameters(buildTableParameters(newMetadataLocation, metadata));
+    try {
+      client.update(tableReference, table);
+    } catch (ValidationException e) {
+      if (e.getMessage().toLowerCase(Locale.ENGLISH).contains("etag mismatch")) {
+        throw new CommitFailedException(
+            "Updating table failed due to conflict updates (etag mismatch). Retry the update");
+      }
+
+      throw e;
+    }
+  }
+
+  // To make the table queryable from Hive, the user would likely be setting the HIVE_ENGINE_ENABLED
+  // parameter.
+  //
+  // TODO: We need to make a decision on how to make the table queryable from Hive.
+  // (could be a server side change or a client side change - that's TBD).
+  private Table makeNewTable(TableMetadata metadata, String metadataFileLocation) {
+    boolean hiveEngineEnabled = hiveEngineEnabled(metadata);

Review Comment:
   nit: can be inlined



-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org

For queries about this service, please contact Infrastructure at:
users@infra.apache.org


---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For additional commands, e-mail: issues-help@iceberg.apache.org


Re: [PR] Catalog: Add BigQuery Metastore Catalog Support [iceberg]

Posted by "nastra (via GitHub)" <gi...@apache.org>.
nastra commented on code in PR #12808:
URL: https://github.com/apache/iceberg/pull/12808#discussion_r2065999585


##########
bigquery/src/main/java/org/apache/iceberg/gcp/bigquery/BigQueryMetastoreCatalog.java:
##########
@@ -0,0 +1,405 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.iceberg.gcp.bigquery;
+
+import com.google.api.services.bigquery.model.Dataset;
+import com.google.api.services.bigquery.model.DatasetList.Datasets;
+import com.google.api.services.bigquery.model.DatasetReference;
+import com.google.api.services.bigquery.model.ExternalCatalogDatasetOptions;
+import com.google.api.services.bigquery.model.TableReference;
+import com.google.cloud.ServiceOptions;
+import com.google.cloud.bigquery.BigQueryOptions;
+import java.io.IOException;
+import java.security.GeneralSecurityException;
+import java.util.List;
+import java.util.Locale;
+import java.util.Map;
+import java.util.Objects;
+import java.util.Set;
+import org.apache.hadoop.conf.Configurable;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.iceberg.BaseMetastoreCatalog;
+import org.apache.iceberg.CatalogProperties;
+import org.apache.iceberg.CatalogUtil;
+import org.apache.iceberg.TableMetadata;
+import org.apache.iceberg.TableOperations;
+import org.apache.iceberg.catalog.Namespace;
+import org.apache.iceberg.catalog.SupportsNamespaces;
+import org.apache.iceberg.catalog.TableIdentifier;
+import org.apache.iceberg.exceptions.AlreadyExistsException;
+import org.apache.iceberg.exceptions.NoSuchNamespaceException;
+import org.apache.iceberg.exceptions.NoSuchTableException;
+import org.apache.iceberg.exceptions.ServiceFailureException;
+import org.apache.iceberg.exceptions.ValidationException;
+import org.apache.iceberg.io.FileIO;
+import org.apache.iceberg.relocated.com.google.common.annotations.VisibleForTesting;
+import org.apache.iceberg.relocated.com.google.common.base.Preconditions;
+import org.apache.iceberg.relocated.com.google.common.base.Strings;
+import org.apache.iceberg.relocated.com.google.common.collect.ImmutableList;
+import org.apache.iceberg.relocated.com.google.common.collect.ImmutableMap;
+import org.apache.iceberg.relocated.com.google.common.collect.Maps;
+import org.apache.iceberg.util.LocationUtil;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/** Iceberg Bigquery Metastore Catalog implementation. */
+public class BigQueryMetastoreCatalog extends BaseMetastoreCatalog
+    implements SupportsNamespaces, Configurable {
+
+  // User provided properties.
+  public static final String PROJECT_ID = "gcp-project";
+  public static final String GCP_LOCATION = "gcp-location";
+  public static final String LIST_ALL_TABLES = "list-all-tables";
+
+  public static final String HIVE_METASTORE_WAREHOUSE_DIR = "hive.metastore.warehouse.dir";
+
+  private static final Logger LOG = LoggerFactory.getLogger(BigQueryMetastoreCatalog.class);
+
+  private static final String DEFAULT_GCP_LOCATION = "us";
+
+  private String catalogName;
+  private Map<String, String> catalogProperties;
+  private FileIO fileIO;
+  private Configuration conf;
+  private String projectId;
+  private String location;
+  private BigQueryMetastoreClient client;
+  private boolean listAllTables;
+
+  // Must have a no-arg constructor to be dynamically loaded
+  // initialize(String name, Map<String, String> properties) will be called to complete
+  // initialization
+  public BigQueryMetastoreCatalog() {}
+
+  @Override
+  public void initialize(String inputName, Map<String, String> properties) {
+    Preconditions.checkArgument(
+        properties.containsKey(PROJECT_ID),
+        "Invalid GCP project: %s must be specified",
+        PROJECT_ID);
+
+    projectId = properties.get(PROJECT_ID);
+    location = properties.getOrDefault(GCP_LOCATION, DEFAULT_GCP_LOCATION);
+
+    BigQueryOptions options =
+        BigQueryOptions.newBuilder()
+            .setProjectId(projectId)
+            .setLocation(location)
+            .setRetrySettings(ServiceOptions.getDefaultRetrySettings())
+            .build();
+
+    try {
+      client = new BigQueryMetastoreClientImpl(options);
+    } catch (IOException e) {
+      throw new ServiceFailureException(e, "Creating BigQuery client failed");
+    } catch (GeneralSecurityException e) {
+      throw new ValidationException(e, "Creating BigQuery client failed due to a security issue");
+    }
+
+    initialize(inputName, properties, projectId, location, client);
+  }
+
+  @VisibleForTesting
+  void initialize(
+      String inputName,
+      Map<String, String> properties,
+      String initialProjectId,
+      String initialLocation,
+      BigQueryMetastoreClient bigQueryMetaStoreClient) {
+    Preconditions.checkArgument(bigQueryMetaStoreClient != null, "Invalid BigQuery client: null");
+    this.catalogName = inputName;
+    this.catalogProperties = ImmutableMap.copyOf(properties);
+    this.projectId = initialProjectId;
+    this.location = initialLocation;
+    this.client = bigQueryMetaStoreClient;
+
+    if (this.conf == null) {
+      LOG.warn("No configuration was set, using the default environment Configuration");
+      this.conf = new Configuration();
+    }
+
+    LOG.info("Using BigQuery Metastore Iceberg Catalog: {}", inputName);
+
+    if (properties.containsKey(CatalogProperties.WAREHOUSE_LOCATION)) {
+      this.conf.set(
+          HIVE_METASTORE_WAREHOUSE_DIR,
+          LocationUtil.stripTrailingSlash(properties.get(CatalogProperties.WAREHOUSE_LOCATION)));
+    }
+
+    String fileIoImpl =

Review Comment:
   nit: can probably be just inlined



-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org

For queries about this service, please contact Infrastructure at:
users@infra.apache.org


---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For additional commands, e-mail: issues-help@iceberg.apache.org


Re: [PR] Catalog: Add BigQuery Metastore Catalog Support [iceberg]

Posted by "nastra (via GitHub)" <gi...@apache.org>.
nastra commented on code in PR #12808:
URL: https://github.com/apache/iceberg/pull/12808#discussion_r2065997639


##########
bigquery/src/main/java/org/apache/iceberg/gcp/bigquery/BigQueryMetastoreCatalog.java:
##########
@@ -0,0 +1,405 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.iceberg.gcp.bigquery;
+
+import com.google.api.services.bigquery.model.Dataset;
+import com.google.api.services.bigquery.model.DatasetList.Datasets;
+import com.google.api.services.bigquery.model.DatasetReference;
+import com.google.api.services.bigquery.model.ExternalCatalogDatasetOptions;
+import com.google.api.services.bigquery.model.TableReference;
+import com.google.cloud.ServiceOptions;
+import com.google.cloud.bigquery.BigQueryOptions;
+import java.io.IOException;
+import java.security.GeneralSecurityException;
+import java.util.List;
+import java.util.Locale;
+import java.util.Map;
+import java.util.Objects;
+import java.util.Set;
+import org.apache.hadoop.conf.Configurable;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.iceberg.BaseMetastoreCatalog;
+import org.apache.iceberg.CatalogProperties;
+import org.apache.iceberg.CatalogUtil;
+import org.apache.iceberg.TableMetadata;
+import org.apache.iceberg.TableOperations;
+import org.apache.iceberg.catalog.Namespace;
+import org.apache.iceberg.catalog.SupportsNamespaces;
+import org.apache.iceberg.catalog.TableIdentifier;
+import org.apache.iceberg.exceptions.AlreadyExistsException;
+import org.apache.iceberg.exceptions.NoSuchNamespaceException;
+import org.apache.iceberg.exceptions.NoSuchTableException;
+import org.apache.iceberg.exceptions.ServiceFailureException;
+import org.apache.iceberg.exceptions.ValidationException;
+import org.apache.iceberg.io.FileIO;
+import org.apache.iceberg.relocated.com.google.common.annotations.VisibleForTesting;
+import org.apache.iceberg.relocated.com.google.common.base.Preconditions;
+import org.apache.iceberg.relocated.com.google.common.base.Strings;
+import org.apache.iceberg.relocated.com.google.common.collect.ImmutableList;
+import org.apache.iceberg.relocated.com.google.common.collect.ImmutableMap;
+import org.apache.iceberg.relocated.com.google.common.collect.Maps;
+import org.apache.iceberg.util.LocationUtil;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/** Iceberg Bigquery Metastore Catalog implementation. */
+public class BigQueryMetastoreCatalog extends BaseMetastoreCatalog
+    implements SupportsNamespaces, Configurable {
+
+  // User provided properties.
+  public static final String PROJECT_ID = "gcp-project";
+  public static final String GCP_LOCATION = "gcp-location";
+  public static final String LIST_ALL_TABLES = "list-all-tables";
+
+  public static final String HIVE_METASTORE_WAREHOUSE_DIR = "hive.metastore.warehouse.dir";
+
+  private static final Logger LOG = LoggerFactory.getLogger(BigQueryMetastoreCatalog.class);
+
+  private static final String DEFAULT_GCP_LOCATION = "us";
+
+  private String catalogName;
+  private Map<String, String> catalogProperties;
+  private FileIO fileIO;
+  private Configuration conf;
+  private String projectId;
+  private String location;
+  private BigQueryMetastoreClient client;
+  private boolean listAllTables;
+
+  // Must have a no-arg constructor to be dynamically loaded
+  // initialize(String name, Map<String, String> properties) will be called to complete
+  // initialization
+  public BigQueryMetastoreCatalog() {}
+
+  @Override
+  public void initialize(String inputName, Map<String, String> properties) {
+    Preconditions.checkArgument(
+        properties.containsKey(PROJECT_ID),
+        "Invalid GCP project: %s must be specified",
+        PROJECT_ID);
+
+    projectId = properties.get(PROJECT_ID);

Review Comment:
   ```suggestion
       this.projectId = properties.get(PROJECT_ID);
   ```
   sorry I missed this during previous review rounds



##########
bigquery/src/main/java/org/apache/iceberg/gcp/bigquery/BigQueryMetastoreCatalog.java:
##########
@@ -0,0 +1,405 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.iceberg.gcp.bigquery;
+
+import com.google.api.services.bigquery.model.Dataset;
+import com.google.api.services.bigquery.model.DatasetList.Datasets;
+import com.google.api.services.bigquery.model.DatasetReference;
+import com.google.api.services.bigquery.model.ExternalCatalogDatasetOptions;
+import com.google.api.services.bigquery.model.TableReference;
+import com.google.cloud.ServiceOptions;
+import com.google.cloud.bigquery.BigQueryOptions;
+import java.io.IOException;
+import java.security.GeneralSecurityException;
+import java.util.List;
+import java.util.Locale;
+import java.util.Map;
+import java.util.Objects;
+import java.util.Set;
+import org.apache.hadoop.conf.Configurable;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.iceberg.BaseMetastoreCatalog;
+import org.apache.iceberg.CatalogProperties;
+import org.apache.iceberg.CatalogUtil;
+import org.apache.iceberg.TableMetadata;
+import org.apache.iceberg.TableOperations;
+import org.apache.iceberg.catalog.Namespace;
+import org.apache.iceberg.catalog.SupportsNamespaces;
+import org.apache.iceberg.catalog.TableIdentifier;
+import org.apache.iceberg.exceptions.AlreadyExistsException;
+import org.apache.iceberg.exceptions.NoSuchNamespaceException;
+import org.apache.iceberg.exceptions.NoSuchTableException;
+import org.apache.iceberg.exceptions.ServiceFailureException;
+import org.apache.iceberg.exceptions.ValidationException;
+import org.apache.iceberg.io.FileIO;
+import org.apache.iceberg.relocated.com.google.common.annotations.VisibleForTesting;
+import org.apache.iceberg.relocated.com.google.common.base.Preconditions;
+import org.apache.iceberg.relocated.com.google.common.base.Strings;
+import org.apache.iceberg.relocated.com.google.common.collect.ImmutableList;
+import org.apache.iceberg.relocated.com.google.common.collect.ImmutableMap;
+import org.apache.iceberg.relocated.com.google.common.collect.Maps;
+import org.apache.iceberg.util.LocationUtil;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/** Iceberg Bigquery Metastore Catalog implementation. */
+public class BigQueryMetastoreCatalog extends BaseMetastoreCatalog
+    implements SupportsNamespaces, Configurable {
+
+  // User provided properties.
+  public static final String PROJECT_ID = "gcp-project";
+  public static final String GCP_LOCATION = "gcp-location";
+  public static final String LIST_ALL_TABLES = "list-all-tables";
+
+  public static final String HIVE_METASTORE_WAREHOUSE_DIR = "hive.metastore.warehouse.dir";
+
+  private static final Logger LOG = LoggerFactory.getLogger(BigQueryMetastoreCatalog.class);
+
+  private static final String DEFAULT_GCP_LOCATION = "us";
+
+  private String catalogName;
+  private Map<String, String> catalogProperties;
+  private FileIO fileIO;
+  private Configuration conf;
+  private String projectId;
+  private String location;
+  private BigQueryMetastoreClient client;
+  private boolean listAllTables;
+
+  // Must have a no-arg constructor to be dynamically loaded
+  // initialize(String name, Map<String, String> properties) will be called to complete
+  // initialization
+  public BigQueryMetastoreCatalog() {}
+
+  @Override
+  public void initialize(String inputName, Map<String, String> properties) {
+    Preconditions.checkArgument(
+        properties.containsKey(PROJECT_ID),
+        "Invalid GCP project: %s must be specified",
+        PROJECT_ID);
+
+    projectId = properties.get(PROJECT_ID);
+    location = properties.getOrDefault(GCP_LOCATION, DEFAULT_GCP_LOCATION);

Review Comment:
   ```suggestion
       this.location = properties.getOrDefault(GCP_LOCATION, DEFAULT_GCP_LOCATION);
   ```



-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org

For queries about this service, please contact Infrastructure at:
users@infra.apache.org


---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For additional commands, e-mail: issues-help@iceberg.apache.org


Re: [PR] Catalog: Add BigQuery Metastore Catalog Support [iceberg]

Posted by "danielcweeks (via GitHub)" <gi...@apache.org>.
danielcweeks commented on code in PR #12808:
URL: https://github.com/apache/iceberg/pull/12808#discussion_r2067559631


##########
bigquery/src/main/java/org/apache/iceberg/gcp/bigquery/BigQueryMetastoreCatalog.java:
##########
@@ -0,0 +1,405 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.iceberg.gcp.bigquery;
+
+import com.google.api.services.bigquery.model.Dataset;
+import com.google.api.services.bigquery.model.DatasetList.Datasets;
+import com.google.api.services.bigquery.model.DatasetReference;
+import com.google.api.services.bigquery.model.ExternalCatalogDatasetOptions;
+import com.google.api.services.bigquery.model.TableReference;
+import com.google.cloud.ServiceOptions;
+import com.google.cloud.bigquery.BigQueryOptions;
+import java.io.IOException;
+import java.security.GeneralSecurityException;
+import java.util.List;
+import java.util.Locale;
+import java.util.Map;
+import java.util.Objects;
+import java.util.Set;
+import org.apache.hadoop.conf.Configurable;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.iceberg.BaseMetastoreCatalog;
+import org.apache.iceberg.CatalogProperties;
+import org.apache.iceberg.CatalogUtil;
+import org.apache.iceberg.TableMetadata;
+import org.apache.iceberg.TableOperations;
+import org.apache.iceberg.catalog.Namespace;
+import org.apache.iceberg.catalog.SupportsNamespaces;
+import org.apache.iceberg.catalog.TableIdentifier;
+import org.apache.iceberg.exceptions.AlreadyExistsException;
+import org.apache.iceberg.exceptions.NoSuchNamespaceException;
+import org.apache.iceberg.exceptions.NoSuchTableException;
+import org.apache.iceberg.exceptions.ServiceFailureException;
+import org.apache.iceberg.exceptions.ValidationException;
+import org.apache.iceberg.io.FileIO;
+import org.apache.iceberg.relocated.com.google.common.annotations.VisibleForTesting;
+import org.apache.iceberg.relocated.com.google.common.base.Preconditions;
+import org.apache.iceberg.relocated.com.google.common.base.Strings;
+import org.apache.iceberg.relocated.com.google.common.collect.ImmutableList;
+import org.apache.iceberg.relocated.com.google.common.collect.ImmutableMap;
+import org.apache.iceberg.relocated.com.google.common.collect.Maps;
+import org.apache.iceberg.util.LocationUtil;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/** Iceberg Bigquery Metastore Catalog implementation. */
+public class BigQueryMetastoreCatalog extends BaseMetastoreCatalog
+    implements SupportsNamespaces, Configurable {
+
+  // User provided properties.
+  public static final String PROJECT_ID = "gcp-project";
+  public static final String GCP_LOCATION = "gcp-location";
+  public static final String LIST_ALL_TABLES = "list-all-tables";
+
+  public static final String HIVE_METASTORE_WAREHOUSE_DIR = "hive.metastore.warehouse.dir";
+
+  private static final Logger LOG = LoggerFactory.getLogger(BigQueryMetastoreCatalog.class);
+
+  private static final String DEFAULT_GCP_LOCATION = "us";
+
+  private String catalogName;
+  private Map<String, String> catalogProperties;
+  private FileIO fileIO;
+  private Configuration conf;
+  private String projectId;
+  private String location;
+  private BigQueryMetastoreClient client;
+  private boolean listAllTables;
+
+  // Must have a no-arg constructor to be dynamically loaded
+  // initialize(String name, Map<String, String> properties) will be called to complete
+  // initialization
+  public BigQueryMetastoreCatalog() {}
+
+  @Override
+  public void initialize(String inputName, Map<String, String> properties) {
+    Preconditions.checkArgument(
+        properties.containsKey(PROJECT_ID),
+        "Invalid GCP project: %s must be specified",
+        PROJECT_ID);
+
+    projectId = properties.get(PROJECT_ID);
+    location = properties.getOrDefault(GCP_LOCATION, DEFAULT_GCP_LOCATION);
+
+    BigQueryOptions options =
+        BigQueryOptions.newBuilder()
+            .setProjectId(projectId)
+            .setLocation(location)
+            .setRetrySettings(ServiceOptions.getDefaultRetrySettings())
+            .build();
+
+    try {
+      client = new BigQueryMetastoreClientImpl(options);
+    } catch (IOException e) {
+      throw new ServiceFailureException(e, "Creating BigQuery client failed");
+    } catch (GeneralSecurityException e) {
+      throw new ValidationException(e, "Creating BigQuery client failed due to a security issue");
+    }
+
+    initialize(inputName, properties, projectId, location, client);
+  }
+
+  @VisibleForTesting
+  void initialize(
+      String inputName,
+      Map<String, String> properties,
+      String initialProjectId,
+      String initialLocation,
+      BigQueryMetastoreClient bigQueryMetaStoreClient) {
+    Preconditions.checkArgument(bigQueryMetaStoreClient != null, "Invalid BigQuery client: null");
+    this.catalogName = inputName;
+    this.catalogProperties = ImmutableMap.copyOf(properties);
+    this.projectId = initialProjectId;
+    this.location = initialLocation;
+    this.client = bigQueryMetaStoreClient;
+
+    if (this.conf == null) {
+      LOG.warn("No configuration was set, using the default environment Configuration");
+      this.conf = new Configuration();
+    }
+
+    LOG.info("Using BigQuery Metastore Iceberg Catalog: {}", inputName);
+
+    if (properties.containsKey(CatalogProperties.WAREHOUSE_LOCATION)) {
+      this.conf.set(
+          HIVE_METASTORE_WAREHOUSE_DIR,
+          LocationUtil.stripTrailingSlash(properties.get(CatalogProperties.WAREHOUSE_LOCATION)));
+    }
+
+    String fileIoImpl =
+        properties.getOrDefault(
+            CatalogProperties.FILE_IO_IMPL, "org.apache.iceberg.hadoop.HadoopFileIO");
+    this.fileIO = CatalogUtil.loadFileIO(fileIoImpl, properties, conf);
+
+    this.listAllTables = Boolean.parseBoolean(properties.getOrDefault(LIST_ALL_TABLES, "true"));
+  }
+
+  @Override
+  protected TableOperations newTableOps(TableIdentifier identifier) {
+    return new BigQueryTableOperations(client, fileIO, toTableReference(identifier), conf);
+  }
+
+  @Override
+  protected String defaultWarehouseLocation(TableIdentifier identifier) {
+    String locationUri = null;
+    DatasetReference datasetReference = toDatasetReference(identifier.namespace());
+    Dataset dataset = client.load(datasetReference);
+    if (dataset != null && dataset.getExternalCatalogDatasetOptions() != null) {
+      locationUri = dataset.getExternalCatalogDatasetOptions().getDefaultStorageLocationUri();
+    }
+
+    return String.format(
+        "%s/%s",
+        Strings.isNullOrEmpty(locationUri)
+            ? createDefaultStorageLocationUri(datasetReference.getDatasetId())
+            : LocationUtil.stripTrailingSlash(locationUri),
+        identifier.name());
+  }
+
+  @Override
+  public List<TableIdentifier> listTables(Namespace namespace) {
+    validateNamespace(namespace);
+
+    return client.list(toDatasetReference(namespace), listAllTables).stream()
+        .map(
+            table -> TableIdentifier.of(namespace.level(0), table.getTableReference().getTableId()))
+        .collect(ImmutableList.toImmutableList());
+  }
+
+  @Override
+  public boolean dropTable(TableIdentifier identifier, boolean purge) {
+    try {
+      TableOperations ops = newTableOps(identifier);
+      TableMetadata lastMetadata = ops.current();
+
+      client.delete(toTableReference(identifier));
+
+      if (purge && lastMetadata != null) {
+        CatalogUtil.dropTableData(ops.io(), lastMetadata);
+      }
+    } catch (NoSuchTableException e) {
+      return false;
+    }
+
+    return true;
+  }
+
+  @Override
+  public void renameTable(TableIdentifier from, TableIdentifier to) {
+    validateNamespace(from.namespace());
+    validateNamespace(to.namespace());
+
+    if (!namespaceExists(to.namespace())) {
+      throw new NoSuchNamespaceException(
+          "Cannot rename %s to %s. Namespace does not exist: %s",
+          from.name(), to.name(), to.namespace());
+    }
+
+    if (!namespaceExists(from.namespace())) {
+      throw new NoSuchNamespaceException(
+          "Cannot rename %s to %s. Namespace does not exist: %s",
+          from.name(), to.name(), from.namespace());
+    }
+
+    if (!tableExists(from)) {
+      throw new NoSuchTableException(
+          "Cannot rename %s to %s. Table does not exist: %s", from.name(), to.name(), from.name());
+    }
+
+    if (!from.namespace().equals(to.namespace())) {
+      throw new ValidationException(
+          "Cannot rename %s to %s. New table name must be in the same namespace",
+          from.name(), to.name());
+    }
+
+    if (tableExists(to)) {
+      throw new AlreadyExistsException(
+          "Cannot rename %s to %s. Table already exists: %s", from.name(), to.name(), to.name());
+    }
+
+    // TODO: Enable once supported by BigQuery API.
+    throw new UnsupportedOperationException("Table rename operation is unsupported.");
+  }
+
+  @Override
+  public void createNamespace(Namespace namespace, Map<String, String> metadata) {
+    Dataset builder = new Dataset();
+    DatasetReference datasetReference = toDatasetReference(namespace);
+    builder.setLocation(this.location);
+    builder.setDatasetReference(datasetReference);
+    builder.setExternalCatalogDatasetOptions(
+        BigQueryMetastoreUtils.createExternalCatalogDatasetOptions(
+            createDefaultStorageLocationUri(datasetReference.getDatasetId()), metadata));
+
+    client.create(builder);
+  }
+
+  @Override
+  public List<Namespace> listNamespaces() {
+    try {
+      return listNamespaces(Namespace.empty());
+    } catch (NoSuchNamespaceException e) {
+      return ImmutableList.of();
+    }
+  }
+
+  /**
+   * Since this catalog only supports one-level namespaces, it always returns an empty list unless
+   * passed an empty namespace to list all namespaces within the catalog.
+   */
+  @Override
+  public List<Namespace> listNamespaces(Namespace namespace) {
+    List<Datasets> allDatasets = client.list(projectId);
+
+    ImmutableList<Namespace> namespaces =
+        allDatasets.stream()
+            .map(BigQueryMetastoreCatalog::toNamespace)
+            .collect(ImmutableList.toImmutableList());
+    if (namespaces.isEmpty()) {
+      throw new NoSuchNamespaceException("Namespace does not exist: %s", namespace);
+    }
+
+    return namespaces;
+  }
+
+  @Override
+  public boolean dropNamespace(Namespace namespace) {
+    try {
+      client.delete(toDatasetReference(namespace));
+      // We don't delete the data folder for safety, which aligns with Hive Metastore's default
+      // behavior.
+      // We can support database or catalog level config controlling file deletion in the future.
+      return true;
+    } catch (NoSuchNamespaceException e) {
+      return false;
+    }
+  }
+
+  @Override
+  public boolean setProperties(Namespace namespace, Map<String, String> properties) {
+    Dataset dataset = client.load(toDatasetReference(namespace));
+
+    ExternalCatalogDatasetOptions existingOptions = dataset.getExternalCatalogDatasetOptions();
+    Map<String, String> existingParameters =
+        existingOptions != null ? existingOptions.getParameters() : null;
+
+    Map<String, String> newParameters = Maps.newHashMap();
+    if (existingParameters != null) {
+      newParameters.putAll(existingParameters);
+    }
+
+    newParameters.putAll(properties);
+
+    if (Objects.equals(existingParameters, newParameters)) {
+      // No change in parameters detected
+      return false;
+    }
+
+    client.setParameters(toDatasetReference(namespace), properties);
+    return true;
+  }
+
+  @Override
+  public boolean removeProperties(Namespace namespace, Set<String> properties) {
+    client.removeParameters(toDatasetReference(namespace), properties);
+    return true;
+  }
+
+  @SuppressWarnings("FormatStringAnnotation")
+  @Override
+  public Map<String, String> loadNamespaceMetadata(Namespace namespace) {
+    try {
+      return toMetadata(client.load(toDatasetReference(namespace)));
+    } catch (IllegalArgumentException e) {
+      throw new NoSuchNamespaceException(e.getMessage());
+    }
+  }
+
+  @Override
+  public String name() {
+    return catalogName;
+  }
+
+  @Override
+  protected Map<String, String> properties() {
+    return catalogProperties == null ? ImmutableMap.of() : catalogProperties;
+  }
+
+  @Override
+  public void setConf(Configuration conf) {
+    this.conf = new Configuration(conf);
+  }
+
+  @Override
+  public Configuration getConf() {
+    return this.conf;
+  }
+
+  private String createDefaultStorageLocationUri(String dbId) {
+    String warehouseLocation = conf.get(HIVE_METASTORE_WAREHOUSE_DIR);

Review Comment:
   Similar to above comment, we should just have the `warehouseLocation` a property of the class and not get/set it via the hadoop conf.



-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org

For queries about this service, please contact Infrastructure at:
users@infra.apache.org


---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For additional commands, e-mail: issues-help@iceberg.apache.org


Re: [PR] Catalog: Add BigQuery Metastore Catalog Support [iceberg]

Posted by "talatuyarer (via GitHub)" <gi...@apache.org>.
talatuyarer commented on code in PR #12808:
URL: https://github.com/apache/iceberg/pull/12808#discussion_r2067934911


##########
bigquery/src/main/java/org/apache/iceberg/gcp/bigquery/BigQueryMetastoreClientImpl.java:
##########
@@ -0,0 +1,594 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.iceberg.gcp.bigquery;
+
+import com.google.api.client.googleapis.javanet.GoogleNetHttpTransport;
+import com.google.api.client.http.HttpHeaders;
+import com.google.api.client.http.HttpResponse;
+import com.google.api.client.http.HttpResponseException;
+import com.google.api.client.http.HttpStatusCodes;
+import com.google.api.client.json.gson.GsonFactory;
+import com.google.api.client.util.Data;
+import com.google.api.services.bigquery.Bigquery;
+import com.google.api.services.bigquery.BigqueryScopes;
+import com.google.api.services.bigquery.model.Dataset;
+import com.google.api.services.bigquery.model.DatasetList;
+import com.google.api.services.bigquery.model.DatasetList.Datasets;
+import com.google.api.services.bigquery.model.DatasetReference;
+import com.google.api.services.bigquery.model.ExternalCatalogDatasetOptions;
+import com.google.api.services.bigquery.model.ExternalCatalogTableOptions;
+import com.google.api.services.bigquery.model.Table;
+import com.google.api.services.bigquery.model.TableList;
+import com.google.api.services.bigquery.model.TableList.Tables;
+import com.google.api.services.bigquery.model.TableReference;
+import com.google.api.services.bigquery.model.TableSchema;
+import com.google.auth.http.HttpCredentialsAdapter;
+import com.google.auth.oauth2.GoogleCredentials;
+import com.google.cloud.BaseServiceException;
+import com.google.cloud.ExceptionHandler;
+import com.google.cloud.bigquery.BigQueryErrorMessages;
+import com.google.cloud.bigquery.BigQueryOptions;
+import com.google.cloud.bigquery.BigQueryRetryConfig;
+import com.google.cloud.bigquery.BigQueryRetryHelper;
+import java.io.IOException;
+import java.nio.charset.StandardCharsets;
+import java.security.GeneralSecurityException;
+import java.util.List;
+import java.util.Locale;
+import java.util.Map;
+import java.util.Set;
+import java.util.stream.Collectors;
+import java.util.stream.Stream;
+import org.apache.iceberg.BaseMetastoreTableOperations;
+import org.apache.iceberg.exceptions.AlreadyExistsException;
+import org.apache.iceberg.exceptions.BadRequestException;
+import org.apache.iceberg.exceptions.ForbiddenException;
+import org.apache.iceberg.exceptions.NoSuchIcebergTableException;
+import org.apache.iceberg.exceptions.NoSuchNamespaceException;
+import org.apache.iceberg.exceptions.NoSuchTableException;
+import org.apache.iceberg.exceptions.NotAuthorizedException;
+import org.apache.iceberg.exceptions.RuntimeIOException;
+import org.apache.iceberg.exceptions.ServiceFailureException;
+import org.apache.iceberg.exceptions.ServiceUnavailableException;
+import org.apache.iceberg.exceptions.ValidationException;
+import org.apache.iceberg.relocated.com.google.common.base.Preconditions;
+import org.apache.iceberg.relocated.com.google.common.collect.Lists;
+import org.apache.iceberg.relocated.com.google.common.collect.Maps;
+
+/** A client of Google Bigquery Metastore functions over the BigQuery service. */
+public final class BigQueryMetastoreClientImpl implements BigQueryMetastoreClient {
+
+  public static final ExceptionHandler.Interceptor EXCEPTION_HANDLER_INTERCEPTOR =
+      new ExceptionHandler.Interceptor() {
+
+        private static final long serialVersionUID = -8429573486870467828L;
+
+        @Override
+        public RetryResult afterEval(Exception exception, RetryResult retryResult) {
+          return ExceptionHandler.Interceptor.RetryResult.CONTINUE_EVALUATION;
+        }
+
+        @Override
+        public RetryResult beforeEval(Exception exception) {
+          if (exception instanceof BaseServiceException) {
+            boolean retriable = ((BaseServiceException) exception).isRetryable();
+            return retriable
+                ? ExceptionHandler.Interceptor.RetryResult.RETRY
+                : ExceptionHandler.Interceptor.RetryResult.CONTINUE_EVALUATION;
+          }
+
+          return ExceptionHandler.Interceptor.RetryResult.CONTINUE_EVALUATION;
+        }
+      };
+
+  private final Bigquery client;
+  private final BigQueryOptions bigqueryOptions;
+
+  // Retry config with error messages and regex for rate limit exceeded errors.
+  private static final BigQueryRetryConfig DEFAULT_RETRY_CONFIG =
+      BigQueryRetryConfig.newBuilder()
+          .retryOnMessage(BigQueryErrorMessages.RATE_LIMIT_EXCEEDED_MSG)
+          .retryOnMessage(BigQueryErrorMessages.JOB_RATE_LIMIT_EXCEEDED_MSG)
+          .retryOnRegEx(BigQueryErrorMessages.RetryRegExPatterns.RATE_LIMIT_EXCEEDED_REGEX)
+          .build();
+
+  public static final ExceptionHandler BIGQUERY_EXCEPTION_HANDLER =
+      ExceptionHandler.newBuilder()
+          .abortOn(RuntimeException.class)
+          .retryOn(
+              java.net.ConnectException
+                  .class) // Retry on connection failures due to transient network issues.
+          .retryOn(
+              java.net.UnknownHostException
+                  .class) // Retry to recover from temporary DNS resolution failures.
+          .addInterceptors(EXCEPTION_HANDLER_INTERCEPTOR)
+          .build();
+
+  /** Constructs a client of the Google BigQuery service. */
+  public BigQueryMetastoreClientImpl(BigQueryOptions options)
+      throws IOException, GeneralSecurityException {
+    // Initialize client that will be used to send requests. This client only needs to be created
+    // once, and can be reused for multiple requests
+    HttpCredentialsAdapter httpCredentialsAdapter =
+        new HttpCredentialsAdapter(
+            GoogleCredentials.getApplicationDefault().createScoped(BigqueryScopes.all()));
+    this.client =
+        new Bigquery.Builder(
+                GoogleNetHttpTransport.newTrustedTransport(),
+                GsonFactory.getDefaultInstance(),
+                httpRequest -> {
+                  httpCredentialsAdapter.initialize(httpRequest);
+                  // Instead of throwing exceptions, analyze the HttpResponse object and inspect its
+                  // status code. This will allow BigQuery API errors to be converted into Iceberg
+                  // exceptions.
+                  httpRequest.setThrowExceptionOnExecuteError(false);
+                })
+            .setApplicationName("BigQuery Metastore Iceberg Catalog Plugin")
+            .build();
+    this.bigqueryOptions = options;
+  }
+
+  @Override
+  public Dataset create(Dataset dataset) {
+    Dataset response = null;
+    try {
+      response =
+          BigQueryRetryHelper.runWithRetries(
+              () -> internalCreate(dataset),
+              bigqueryOptions.getRetrySettings(),
+              BIGQUERY_EXCEPTION_HANDLER,
+              bigqueryOptions.getClock(),
+              DEFAULT_RETRY_CONFIG);
+    } catch (BigQueryRetryHelper.BigQueryRetryHelperException e) {
+      handleBigQueryRetryException(e);
+    }
+    return response;
+  }
+
+  @SuppressWarnings("FormatStringAnnotation")
+  private Dataset internalCreate(Dataset dataset) {
+    try {
+      HttpResponse response =
+          client
+              .datasets()
+              .insert(dataset.getDatasetReference().getProjectId(), dataset)
+              .executeUnparsed();
+      return convertExceptionIfUnsuccessful(response).parseAs(Dataset.class);
+    } catch (IOException e) {
+      throw new RuntimeIOException(e);
+    } catch (AlreadyExistsException e) {
+      throw new AlreadyExistsException("Namespace already exists: " + dataset.getId());
+    }
+  }
+
+  @Override
+  @SuppressWarnings("FormatStringAnnotation")

Review Comment:
   Done



##########
bigquery/src/main/java/org/apache/iceberg/gcp/bigquery/BigQueryMetastoreClientImpl.java:
##########
@@ -0,0 +1,594 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.iceberg.gcp.bigquery;
+
+import com.google.api.client.googleapis.javanet.GoogleNetHttpTransport;
+import com.google.api.client.http.HttpHeaders;
+import com.google.api.client.http.HttpResponse;
+import com.google.api.client.http.HttpResponseException;
+import com.google.api.client.http.HttpStatusCodes;
+import com.google.api.client.json.gson.GsonFactory;
+import com.google.api.client.util.Data;
+import com.google.api.services.bigquery.Bigquery;
+import com.google.api.services.bigquery.BigqueryScopes;
+import com.google.api.services.bigquery.model.Dataset;
+import com.google.api.services.bigquery.model.DatasetList;
+import com.google.api.services.bigquery.model.DatasetList.Datasets;
+import com.google.api.services.bigquery.model.DatasetReference;
+import com.google.api.services.bigquery.model.ExternalCatalogDatasetOptions;
+import com.google.api.services.bigquery.model.ExternalCatalogTableOptions;
+import com.google.api.services.bigquery.model.Table;
+import com.google.api.services.bigquery.model.TableList;
+import com.google.api.services.bigquery.model.TableList.Tables;
+import com.google.api.services.bigquery.model.TableReference;
+import com.google.api.services.bigquery.model.TableSchema;
+import com.google.auth.http.HttpCredentialsAdapter;
+import com.google.auth.oauth2.GoogleCredentials;
+import com.google.cloud.BaseServiceException;
+import com.google.cloud.ExceptionHandler;
+import com.google.cloud.bigquery.BigQueryErrorMessages;
+import com.google.cloud.bigquery.BigQueryOptions;
+import com.google.cloud.bigquery.BigQueryRetryConfig;
+import com.google.cloud.bigquery.BigQueryRetryHelper;
+import java.io.IOException;
+import java.nio.charset.StandardCharsets;
+import java.security.GeneralSecurityException;
+import java.util.List;
+import java.util.Locale;
+import java.util.Map;
+import java.util.Set;
+import java.util.stream.Collectors;
+import java.util.stream.Stream;
+import org.apache.iceberg.BaseMetastoreTableOperations;
+import org.apache.iceberg.exceptions.AlreadyExistsException;
+import org.apache.iceberg.exceptions.BadRequestException;
+import org.apache.iceberg.exceptions.ForbiddenException;
+import org.apache.iceberg.exceptions.NoSuchIcebergTableException;
+import org.apache.iceberg.exceptions.NoSuchNamespaceException;
+import org.apache.iceberg.exceptions.NoSuchTableException;
+import org.apache.iceberg.exceptions.NotAuthorizedException;
+import org.apache.iceberg.exceptions.RuntimeIOException;
+import org.apache.iceberg.exceptions.ServiceFailureException;
+import org.apache.iceberg.exceptions.ServiceUnavailableException;
+import org.apache.iceberg.exceptions.ValidationException;
+import org.apache.iceberg.relocated.com.google.common.base.Preconditions;
+import org.apache.iceberg.relocated.com.google.common.collect.Lists;
+import org.apache.iceberg.relocated.com.google.common.collect.Maps;
+
+/** A client of Google Bigquery Metastore functions over the BigQuery service. */
+public final class BigQueryMetastoreClientImpl implements BigQueryMetastoreClient {
+
+  public static final ExceptionHandler.Interceptor EXCEPTION_HANDLER_INTERCEPTOR =
+      new ExceptionHandler.Interceptor() {
+
+        private static final long serialVersionUID = -8429573486870467828L;
+
+        @Override
+        public RetryResult afterEval(Exception exception, RetryResult retryResult) {
+          return ExceptionHandler.Interceptor.RetryResult.CONTINUE_EVALUATION;
+        }
+
+        @Override
+        public RetryResult beforeEval(Exception exception) {
+          if (exception instanceof BaseServiceException) {
+            boolean retriable = ((BaseServiceException) exception).isRetryable();
+            return retriable
+                ? ExceptionHandler.Interceptor.RetryResult.RETRY
+                : ExceptionHandler.Interceptor.RetryResult.CONTINUE_EVALUATION;
+          }
+
+          return ExceptionHandler.Interceptor.RetryResult.CONTINUE_EVALUATION;
+        }
+      };
+
+  private final Bigquery client;
+  private final BigQueryOptions bigqueryOptions;
+
+  // Retry config with error messages and regex for rate limit exceeded errors.
+  private static final BigQueryRetryConfig DEFAULT_RETRY_CONFIG =
+      BigQueryRetryConfig.newBuilder()
+          .retryOnMessage(BigQueryErrorMessages.RATE_LIMIT_EXCEEDED_MSG)
+          .retryOnMessage(BigQueryErrorMessages.JOB_RATE_LIMIT_EXCEEDED_MSG)
+          .retryOnRegEx(BigQueryErrorMessages.RetryRegExPatterns.RATE_LIMIT_EXCEEDED_REGEX)
+          .build();
+
+  public static final ExceptionHandler BIGQUERY_EXCEPTION_HANDLER =
+      ExceptionHandler.newBuilder()
+          .abortOn(RuntimeException.class)
+          .retryOn(
+              java.net.ConnectException
+                  .class) // Retry on connection failures due to transient network issues.
+          .retryOn(
+              java.net.UnknownHostException
+                  .class) // Retry to recover from temporary DNS resolution failures.
+          .addInterceptors(EXCEPTION_HANDLER_INTERCEPTOR)
+          .build();
+
+  /** Constructs a client of the Google BigQuery service. */
+  public BigQueryMetastoreClientImpl(BigQueryOptions options)
+      throws IOException, GeneralSecurityException {
+    // Initialize client that will be used to send requests. This client only needs to be created
+    // once, and can be reused for multiple requests
+    HttpCredentialsAdapter httpCredentialsAdapter =
+        new HttpCredentialsAdapter(
+            GoogleCredentials.getApplicationDefault().createScoped(BigqueryScopes.all()));
+    this.client =
+        new Bigquery.Builder(
+                GoogleNetHttpTransport.newTrustedTransport(),
+                GsonFactory.getDefaultInstance(),
+                httpRequest -> {
+                  httpCredentialsAdapter.initialize(httpRequest);
+                  // Instead of throwing exceptions, analyze the HttpResponse object and inspect its
+                  // status code. This will allow BigQuery API errors to be converted into Iceberg
+                  // exceptions.
+                  httpRequest.setThrowExceptionOnExecuteError(false);
+                })
+            .setApplicationName("BigQuery Metastore Iceberg Catalog Plugin")
+            .build();
+    this.bigqueryOptions = options;
+  }
+
+  @Override
+  public Dataset create(Dataset dataset) {
+    Dataset response = null;
+    try {
+      response =
+          BigQueryRetryHelper.runWithRetries(
+              () -> internalCreate(dataset),
+              bigqueryOptions.getRetrySettings(),
+              BIGQUERY_EXCEPTION_HANDLER,
+              bigqueryOptions.getClock(),
+              DEFAULT_RETRY_CONFIG);
+    } catch (BigQueryRetryHelper.BigQueryRetryHelperException e) {
+      handleBigQueryRetryException(e);
+    }
+    return response;
+  }
+
+  @SuppressWarnings("FormatStringAnnotation")
+  private Dataset internalCreate(Dataset dataset) {
+    try {
+      HttpResponse response =
+          client
+              .datasets()
+              .insert(dataset.getDatasetReference().getProjectId(), dataset)
+              .executeUnparsed();
+      return convertExceptionIfUnsuccessful(response).parseAs(Dataset.class);
+    } catch (IOException e) {
+      throw new RuntimeIOException(e);
+    } catch (AlreadyExistsException e) {
+      throw new AlreadyExistsException("Namespace already exists: " + dataset.getId());
+    }
+  }
+
+  @Override
+  @SuppressWarnings("FormatStringAnnotation")
+  public Dataset load(DatasetReference datasetReference) {
+    try {
+      HttpResponse response =
+          client
+              .datasets()
+              .get(datasetReference.getProjectId(), datasetReference.getDatasetId())
+              .executeUnparsed();
+      if (response.getStatusCode() == HttpStatusCodes.STATUS_CODE_NOT_FOUND) {
+        throw new NoSuchNamespaceException(
+            "Namespace does not exist: " + datasetReference.getDatasetId());
+      }
+
+      return convertExceptionIfUnsuccessful(response).parseAs(Dataset.class);
+    } catch (IOException e) {
+      throw new RuntimeIOException(e);
+    }
+  }
+
+  @Override
+  @SuppressWarnings("FormatStringAnnotation")
+  public void delete(DatasetReference datasetReference) {
+    try {
+      HttpResponse response =
+          client
+              .datasets()
+              .delete(datasetReference.getProjectId(), datasetReference.getDatasetId())
+              .executeUnparsed();
+      if (response.getStatusCode() == HttpStatusCodes.STATUS_CODE_NOT_FOUND) {
+        throw new NoSuchNamespaceException(
+            "Namespace does not exist: " + datasetReference.getDatasetId());
+      }
+
+      convertExceptionIfUnsuccessful(response);
+    } catch (IOException e) {
+      throw new RuntimeIOException(e);
+    }
+  }
+
+  @Override
+  public Dataset setParameters(DatasetReference datasetReference, Map<String, String> parameters) {
+    Dataset dataset = load(datasetReference);
+    ExternalCatalogDatasetOptions externalCatalogDatasetOptions =
+        dataset.getExternalCatalogDatasetOptions() == null
+            ? new ExternalCatalogDatasetOptions()
+            : dataset.getExternalCatalogDatasetOptions();
+    Map<String, String> finalParameters =
+        externalCatalogDatasetOptions.getParameters() == null
+            ? Maps.newHashMap()
+            : externalCatalogDatasetOptions.getParameters();
+    finalParameters.putAll(parameters);
+
+    dataset.setExternalCatalogDatasetOptions(
+        externalCatalogDatasetOptions.setParameters(finalParameters));
+
+    return internalUpdate(dataset);
+  }
+
+  @Override
+  public Dataset removeParameters(DatasetReference datasetReference, Set<String> parameters) {
+    Dataset dataset = load(datasetReference);
+    ExternalCatalogDatasetOptions externalCatalogDatasetOptions =
+        dataset.getExternalCatalogDatasetOptions() == null
+            ? new ExternalCatalogDatasetOptions()
+            : dataset.getExternalCatalogDatasetOptions();
+    Map<String, String> finalParameters =
+        externalCatalogDatasetOptions.getParameters() == null
+            ? Maps.newHashMap()
+            : externalCatalogDatasetOptions.getParameters();
+    parameters.forEach(finalParameters::remove);
+
+    dataset.setExternalCatalogDatasetOptions(
+        externalCatalogDatasetOptions.setParameters(finalParameters));
+
+    return internalUpdate(dataset);
+  }
+
+  @Override
+  public List<Datasets> list(String projectId) {
+    try {
+      String nextPageToken = null;
+      List<Datasets> datasets = Lists.newArrayList();
+      do {
+        HttpResponse pageResponse =
+            client.datasets().list(projectId).setPageToken(nextPageToken).executeUnparsed();
+        DatasetList result =
+            convertExceptionIfUnsuccessful(pageResponse).parseAs(DatasetList.class);
+        nextPageToken = result.getNextPageToken();
+        if (result.getDatasets() != null) {
+          datasets.addAll(result.getDatasets());
+        }
+
+      } while (nextPageToken != null && !nextPageToken.isEmpty());
+      return datasets;
+    } catch (IOException e) {
+      throw new RuntimeIOException(e);
+    }
+  }
+
+  @Override
+  public Table create(Table table) {
+    // Ensure it is an Iceberg table supported by the BigQuery metastore catalog.
+    validateTable(table);
+    // TODO: Ensure table creation is idempotent when handling retries.
+    Table response = null;
+    try {
+      response =
+          BigQueryRetryHelper.runWithRetries(
+              () -> internalCreate(table),
+              bigqueryOptions.getRetrySettings(),
+              BIGQUERY_EXCEPTION_HANDLER,
+              bigqueryOptions.getClock(),
+              DEFAULT_RETRY_CONFIG);
+    } catch (BigQueryRetryHelper.BigQueryRetryHelperException e) {
+      handleBigQueryRetryException(e);
+    }
+
+    return response;
+  }
+
+  @SuppressWarnings("FormatStringAnnotation")

Review Comment:
   Removed all



-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org

For queries about this service, please contact Infrastructure at:
users@infra.apache.org


---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For additional commands, e-mail: issues-help@iceberg.apache.org


Re: [PR] Catalog: Add BigQuery Metastore Catalog Support [iceberg]

Posted by "talatuyarer (via GitHub)" <gi...@apache.org>.
talatuyarer commented on code in PR #12808:
URL: https://github.com/apache/iceberg/pull/12808#discussion_r2071955755


##########
bigquery/src/main/java/org/apache/iceberg/gcp/bigquery/BigQueryMetastoreClientImpl.java:
##########
@@ -0,0 +1,581 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.iceberg.gcp.bigquery;
+
+import com.google.api.client.googleapis.javanet.GoogleNetHttpTransport;
+import com.google.api.client.http.HttpHeaders;
+import com.google.api.client.http.HttpResponse;
+import com.google.api.client.http.HttpResponseException;
+import com.google.api.client.http.HttpStatusCodes;
+import com.google.api.client.json.gson.GsonFactory;
+import com.google.api.client.util.Data;
+import com.google.api.services.bigquery.Bigquery;
+import com.google.api.services.bigquery.BigqueryScopes;
+import com.google.api.services.bigquery.model.Dataset;
+import com.google.api.services.bigquery.model.DatasetList;
+import com.google.api.services.bigquery.model.DatasetList.Datasets;
+import com.google.api.services.bigquery.model.DatasetReference;
+import com.google.api.services.bigquery.model.ExternalCatalogDatasetOptions;
+import com.google.api.services.bigquery.model.ExternalCatalogTableOptions;
+import com.google.api.services.bigquery.model.Table;
+import com.google.api.services.bigquery.model.TableList;
+import com.google.api.services.bigquery.model.TableList.Tables;
+import com.google.api.services.bigquery.model.TableReference;
+import com.google.api.services.bigquery.model.TableSchema;
+import com.google.auth.http.HttpCredentialsAdapter;
+import com.google.auth.oauth2.GoogleCredentials;
+import com.google.cloud.BaseServiceException;
+import com.google.cloud.ExceptionHandler;
+import com.google.cloud.bigquery.BigQueryErrorMessages;
+import com.google.cloud.bigquery.BigQueryOptions;
+import com.google.cloud.bigquery.BigQueryRetryConfig;
+import com.google.cloud.bigquery.BigQueryRetryHelper;
+import java.io.IOException;
+import java.nio.charset.StandardCharsets;
+import java.security.GeneralSecurityException;
+import java.util.List;
+import java.util.Locale;
+import java.util.Map;
+import java.util.Set;
+import java.util.stream.Collectors;
+import java.util.stream.Stream;
+import org.apache.iceberg.BaseMetastoreTableOperations;
+import org.apache.iceberg.exceptions.AlreadyExistsException;
+import org.apache.iceberg.exceptions.BadRequestException;
+import org.apache.iceberg.exceptions.ForbiddenException;
+import org.apache.iceberg.exceptions.NoSuchIcebergTableException;
+import org.apache.iceberg.exceptions.NoSuchNamespaceException;
+import org.apache.iceberg.exceptions.NoSuchTableException;
+import org.apache.iceberg.exceptions.NotAuthorizedException;
+import org.apache.iceberg.exceptions.RuntimeIOException;
+import org.apache.iceberg.exceptions.ServiceFailureException;
+import org.apache.iceberg.exceptions.ServiceUnavailableException;
+import org.apache.iceberg.exceptions.ValidationException;
+import org.apache.iceberg.relocated.com.google.common.base.Preconditions;
+import org.apache.iceberg.relocated.com.google.common.collect.Lists;
+import org.apache.iceberg.relocated.com.google.common.collect.Maps;
+
+/** A client of Google Bigquery Metastore functions over the BigQuery service. */
+public final class BigQueryMetastoreClientImpl implements BigQueryMetastoreClient {
+
+  private final Bigquery client;
+  private final BigQueryOptions bigqueryOptions;
+
+  public static final ExceptionHandler.Interceptor EXCEPTION_HANDLER_INTERCEPTOR =
+      new ExceptionHandler.Interceptor() {
+
+        private static final long serialVersionUID = -8429573486870467828L;

Review Comment:
   I put serialVersionUID because it implements Serializable. I agree there is no serialization for this class. I am going to remove this



-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org

For queries about this service, please contact Infrastructure at:
users@infra.apache.org


---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For additional commands, e-mail: issues-help@iceberg.apache.org


Re: [PR] Catalog: Add BigQuery Metastore Catalog Support [iceberg]

Posted by "danielcweeks (via GitHub)" <gi...@apache.org>.
danielcweeks commented on code in PR #12808:
URL: https://github.com/apache/iceberg/pull/12808#discussion_r2072156366


##########
bigquery/src/main/java/org/apache/iceberg/gcp/bigquery/BigQueryMetastoreCatalog.java:
##########
@@ -0,0 +1,366 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.iceberg.gcp.bigquery;
+
+import com.google.api.services.bigquery.model.Dataset;
+import com.google.api.services.bigquery.model.DatasetList.Datasets;
+import com.google.api.services.bigquery.model.DatasetReference;
+import com.google.api.services.bigquery.model.ExternalCatalogDatasetOptions;
+import com.google.api.services.bigquery.model.TableReference;
+import com.google.cloud.ServiceOptions;
+import com.google.cloud.bigquery.BigQueryOptions;
+import java.io.IOException;
+import java.io.UncheckedIOException;
+import java.security.GeneralSecurityException;
+import java.util.List;
+import java.util.Locale;
+import java.util.Map;
+import java.util.Objects;
+import java.util.Set;
+import org.apache.hadoop.conf.Configurable;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.iceberg.BaseMetastoreCatalog;
+import org.apache.iceberg.CatalogProperties;
+import org.apache.iceberg.CatalogUtil;
+import org.apache.iceberg.TableMetadata;
+import org.apache.iceberg.TableOperations;
+import org.apache.iceberg.catalog.Namespace;
+import org.apache.iceberg.catalog.SupportsNamespaces;
+import org.apache.iceberg.catalog.TableIdentifier;
+import org.apache.iceberg.exceptions.NoSuchNamespaceException;
+import org.apache.iceberg.exceptions.NoSuchTableException;
+import org.apache.iceberg.io.FileIO;
+import org.apache.iceberg.relocated.com.google.common.annotations.VisibleForTesting;
+import org.apache.iceberg.relocated.com.google.common.base.Preconditions;
+import org.apache.iceberg.relocated.com.google.common.base.Strings;
+import org.apache.iceberg.relocated.com.google.common.collect.ImmutableList;
+import org.apache.iceberg.relocated.com.google.common.collect.ImmutableMap;
+import org.apache.iceberg.relocated.com.google.common.collect.Maps;
+import org.apache.iceberg.util.LocationUtil;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/** Iceberg Bigquery Metastore Catalog implementation. */
+public class BigQueryMetastoreCatalog extends BaseMetastoreCatalog
+    implements SupportsNamespaces, Configurable {
+
+  // User provided properties.
+  public static final String PROJECT_ID = "gcp.bigquery.project-id";
+  public static final String GCP_LOCATION = "gcp.bigquery.location";
+  public static final String LIST_ALL_TABLES = "gcp.bigquery.list-all-tables";
+
+  private static final Logger LOG = LoggerFactory.getLogger(BigQueryMetastoreCatalog.class);
+
+  private static final String DEFAULT_GCP_LOCATION = "us";
+
+  private String catalogName;
+  private Map<String, String> catalogProperties;
+  private FileIO fileIO;
+  private Configuration conf;
+  private String projectId;
+  private String projectLocation;
+  private BigQueryMetastoreClient client;
+  private boolean listAllTables;
+  private String warehouseLocation;
+
+  public BigQueryMetastoreCatalog() {}
+
+  @Override
+  public void initialize(String name, Map<String, String> properties) {
+    Preconditions.checkArgument(
+        properties.containsKey(PROJECT_ID),
+        "Invalid GCP project: %s must be specified",
+        PROJECT_ID);
+
+    this.projectId = properties.get(PROJECT_ID);
+    this.projectLocation = properties.getOrDefault(GCP_LOCATION, DEFAULT_GCP_LOCATION);
+
+    BigQueryOptions options =
+        BigQueryOptions.newBuilder()
+            .setProjectId(projectId)
+            .setLocation(projectLocation)
+            .setRetrySettings(ServiceOptions.getDefaultRetrySettings())
+            .build();
+
+    try {
+      client = new BigQueryMetastoreClientImpl(options);
+    } catch (IOException e) {
+      throw new UncheckedIOException("Creating BigQuery client failed", e);
+    } catch (GeneralSecurityException e) {
+      throw new RuntimeException("Creating BigQuery client failed due to a security issue", e);
+    }
+
+    initialize(name, properties, projectId, projectLocation, client);
+  }
+
+  @VisibleForTesting
+  void initialize(
+      String name,
+      Map<String, String> properties,
+      String initialProjectId,
+      String initialLocation,
+      BigQueryMetastoreClient bigQueryMetaStoreClient) {
+    Preconditions.checkArgument(bigQueryMetaStoreClient != null, "Invalid BigQuery client: null");
+    this.catalogName = name;
+    this.catalogProperties = ImmutableMap.copyOf(properties);
+    this.projectId = initialProjectId;
+    this.projectLocation = initialLocation;
+    this.client = bigQueryMetaStoreClient;
+
+    if (this.conf == null) {
+      LOG.warn("No configuration was set, using the default environment Configuration");
+      this.conf = new Configuration();
+    }
+
+    LOG.info("Using BigQuery Metastore Iceberg Catalog: {}", name);
+
+    if (properties.containsKey(CatalogProperties.WAREHOUSE_LOCATION)) {
+      this.warehouseLocation =
+          LocationUtil.stripTrailingSlash(properties.get(CatalogProperties.WAREHOUSE_LOCATION));
+    }
+
+    this.fileIO =
+        CatalogUtil.loadFileIO(
+            properties.getOrDefault(
+                CatalogProperties.FILE_IO_IMPL, "org.apache.iceberg.io.ResolvingFileIO"),
+            properties,
+            conf);
+
+    this.listAllTables = Boolean.parseBoolean(properties.getOrDefault(LIST_ALL_TABLES, "true"));
+  }
+
+  @Override
+  protected TableOperations newTableOps(TableIdentifier identifier) {
+    return new BigQueryTableOperations(client, fileIO, toTableReference(identifier), conf);
+  }
+
+  @Override
+  protected String defaultWarehouseLocation(TableIdentifier identifier) {
+    String locationUri = null;
+    DatasetReference datasetReference = toDatasetReference(identifier.namespace());
+    Dataset dataset = client.load(datasetReference);
+    if (dataset != null && dataset.getExternalCatalogDatasetOptions() != null) {
+      locationUri = dataset.getExternalCatalogDatasetOptions().getDefaultStorageLocationUri();
+    }
+
+    return String.format(
+        "%s/%s",
+        Strings.isNullOrEmpty(locationUri)
+            ? createDefaultStorageLocationUri(datasetReference.getDatasetId())
+            : LocationUtil.stripTrailingSlash(locationUri),
+        identifier.name());
+  }
+
+  @Override
+  public List<TableIdentifier> listTables(Namespace namespace) {
+    validateNamespace(namespace);
+
+    return client.list(toDatasetReference(namespace), listAllTables).stream()
+        .map(
+            table -> TableIdentifier.of(namespace.level(0), table.getTableReference().getTableId()))
+        .collect(ImmutableList.toImmutableList());
+  }
+
+  @Override
+  public boolean dropTable(TableIdentifier identifier, boolean purge) {
+    try {
+      TableOperations ops = newTableOps(identifier);
+      TableMetadata lastMetadata = ops.current();
+
+      client.delete(toTableReference(identifier));
+
+      if (purge && lastMetadata != null) {
+        CatalogUtil.dropTableData(ops.io(), lastMetadata);
+      }
+    } catch (NoSuchTableException e) {
+      return false;
+    }
+
+    return true;
+  }
+
+  @Override
+  public void renameTable(TableIdentifier from, TableIdentifier to) {
+    // TODO: Enable once supported by BigQuery API.
+    throw new UnsupportedOperationException("Table rename operation is unsupported.");
+  }
+
+  @Override
+  public void createNamespace(Namespace namespace, Map<String, String> metadata) {
+    Dataset builder = new Dataset();
+    DatasetReference datasetReference = toDatasetReference(namespace);
+    builder.setLocation(this.projectLocation);
+    builder.setDatasetReference(datasetReference);
+    builder.setExternalCatalogDatasetOptions(
+        BigQueryMetastoreUtils.createExternalCatalogDatasetOptions(
+            createDefaultStorageLocationUri(datasetReference.getDatasetId()), metadata));
+
+    client.create(builder);
+  }
+
+  @Override
+  public List<Namespace> listNamespaces() {
+    try {
+      return listNamespaces(Namespace.empty());
+    } catch (NoSuchNamespaceException e) {
+      return ImmutableList.of();
+    }
+  }
+
+  /**
+   * Since this catalog only supports one-level namespaces, it always returns an empty list unless
+   * passed an empty namespace to list all namespaces within the catalog.
+   */
+  @Override
+  public List<Namespace> listNamespaces(Namespace namespace) {
+    List<Datasets> allDatasets = client.list(projectId);
+
+    ImmutableList<Namespace> namespaces =

Review Comment:
   I'm concerned this is going to result in confusing behavior with spark.  We're ignoring the input namespace, which I assume will cause incorrect behavior.  If I'm in a namespace and list that namespace, I will get the top-level namespace listing as opposed to nothing.  Same if I explicitly call show namespaces on a namespace.  I think that's why HiveCatalog returns an empty list if the namespace is not empty.



-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org

For queries about this service, please contact Infrastructure at:
users@infra.apache.org


---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For additional commands, e-mail: issues-help@iceberg.apache.org


Re: [PR] Catalog: Add BigQuery Metastore Catalog Support [iceberg]

Posted by "danielcweeks (via GitHub)" <gi...@apache.org>.
danielcweeks commented on code in PR #12808:
URL: https://github.com/apache/iceberg/pull/12808#discussion_r2072164906


##########
bigquery/src/main/java/org/apache/iceberg/gcp/bigquery/BigQueryMetastoreCatalog.java:
##########
@@ -0,0 +1,366 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.iceberg.gcp.bigquery;
+
+import com.google.api.services.bigquery.model.Dataset;
+import com.google.api.services.bigquery.model.DatasetList.Datasets;
+import com.google.api.services.bigquery.model.DatasetReference;
+import com.google.api.services.bigquery.model.ExternalCatalogDatasetOptions;
+import com.google.api.services.bigquery.model.TableReference;
+import com.google.cloud.ServiceOptions;
+import com.google.cloud.bigquery.BigQueryOptions;
+import java.io.IOException;
+import java.io.UncheckedIOException;
+import java.security.GeneralSecurityException;
+import java.util.List;
+import java.util.Locale;
+import java.util.Map;
+import java.util.Objects;
+import java.util.Set;
+import org.apache.hadoop.conf.Configurable;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.iceberg.BaseMetastoreCatalog;
+import org.apache.iceberg.CatalogProperties;
+import org.apache.iceberg.CatalogUtil;
+import org.apache.iceberg.TableMetadata;
+import org.apache.iceberg.TableOperations;
+import org.apache.iceberg.catalog.Namespace;
+import org.apache.iceberg.catalog.SupportsNamespaces;
+import org.apache.iceberg.catalog.TableIdentifier;
+import org.apache.iceberg.exceptions.NoSuchNamespaceException;
+import org.apache.iceberg.exceptions.NoSuchTableException;
+import org.apache.iceberg.io.FileIO;
+import org.apache.iceberg.relocated.com.google.common.annotations.VisibleForTesting;
+import org.apache.iceberg.relocated.com.google.common.base.Preconditions;
+import org.apache.iceberg.relocated.com.google.common.base.Strings;
+import org.apache.iceberg.relocated.com.google.common.collect.ImmutableList;
+import org.apache.iceberg.relocated.com.google.common.collect.ImmutableMap;
+import org.apache.iceberg.relocated.com.google.common.collect.Maps;
+import org.apache.iceberg.util.LocationUtil;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/** Iceberg Bigquery Metastore Catalog implementation. */
+public class BigQueryMetastoreCatalog extends BaseMetastoreCatalog
+    implements SupportsNamespaces, Configurable {
+
+  // User provided properties.
+  public static final String PROJECT_ID = "gcp.bigquery.project-id";
+  public static final String GCP_LOCATION = "gcp.bigquery.location";
+  public static final String LIST_ALL_TABLES = "gcp.bigquery.list-all-tables";
+
+  private static final Logger LOG = LoggerFactory.getLogger(BigQueryMetastoreCatalog.class);
+
+  private static final String DEFAULT_GCP_LOCATION = "us";
+
+  private String catalogName;
+  private Map<String, String> catalogProperties;
+  private FileIO fileIO;
+  private Configuration conf;
+  private String projectId;
+  private String projectLocation;
+  private BigQueryMetastoreClient client;
+  private boolean listAllTables;
+  private String warehouseLocation;
+
+  public BigQueryMetastoreCatalog() {}
+
+  @Override
+  public void initialize(String name, Map<String, String> properties) {
+    Preconditions.checkArgument(
+        properties.containsKey(PROJECT_ID),
+        "Invalid GCP project: %s must be specified",
+        PROJECT_ID);
+
+    this.projectId = properties.get(PROJECT_ID);
+    this.projectLocation = properties.getOrDefault(GCP_LOCATION, DEFAULT_GCP_LOCATION);
+
+    BigQueryOptions options =
+        BigQueryOptions.newBuilder()
+            .setProjectId(projectId)
+            .setLocation(projectLocation)
+            .setRetrySettings(ServiceOptions.getDefaultRetrySettings())
+            .build();
+
+    try {
+      client = new BigQueryMetastoreClientImpl(options);
+    } catch (IOException e) {
+      throw new UncheckedIOException("Creating BigQuery client failed", e);
+    } catch (GeneralSecurityException e) {
+      throw new RuntimeException("Creating BigQuery client failed due to a security issue", e);
+    }
+
+    initialize(name, properties, projectId, projectLocation, client);
+  }
+
+  @VisibleForTesting
+  void initialize(
+      String name,
+      Map<String, String> properties,
+      String initialProjectId,
+      String initialLocation,
+      BigQueryMetastoreClient bigQueryMetaStoreClient) {
+    Preconditions.checkArgument(bigQueryMetaStoreClient != null, "Invalid BigQuery client: null");
+    this.catalogName = name;
+    this.catalogProperties = ImmutableMap.copyOf(properties);
+    this.projectId = initialProjectId;
+    this.projectLocation = initialLocation;
+    this.client = bigQueryMetaStoreClient;
+
+    if (this.conf == null) {
+      LOG.warn("No configuration was set, using the default environment Configuration");
+      this.conf = new Configuration();
+    }
+
+    LOG.info("Using BigQuery Metastore Iceberg Catalog: {}", name);
+
+    if (properties.containsKey(CatalogProperties.WAREHOUSE_LOCATION)) {
+      this.warehouseLocation =
+          LocationUtil.stripTrailingSlash(properties.get(CatalogProperties.WAREHOUSE_LOCATION));
+    }
+
+    this.fileIO =
+        CatalogUtil.loadFileIO(
+            properties.getOrDefault(
+                CatalogProperties.FILE_IO_IMPL, "org.apache.iceberg.io.ResolvingFileIO"),
+            properties,
+            conf);
+
+    this.listAllTables = Boolean.parseBoolean(properties.getOrDefault(LIST_ALL_TABLES, "true"));
+  }
+
+  @Override
+  protected TableOperations newTableOps(TableIdentifier identifier) {
+    return new BigQueryTableOperations(client, fileIO, toTableReference(identifier), conf);
+  }
+
+  @Override
+  protected String defaultWarehouseLocation(TableIdentifier identifier) {
+    String locationUri = null;
+    DatasetReference datasetReference = toDatasetReference(identifier.namespace());
+    Dataset dataset = client.load(datasetReference);
+    if (dataset != null && dataset.getExternalCatalogDatasetOptions() != null) {
+      locationUri = dataset.getExternalCatalogDatasetOptions().getDefaultStorageLocationUri();
+    }
+
+    return String.format(
+        "%s/%s",
+        Strings.isNullOrEmpty(locationUri)
+            ? createDefaultStorageLocationUri(datasetReference.getDatasetId())
+            : LocationUtil.stripTrailingSlash(locationUri),
+        identifier.name());
+  }
+
+  @Override
+  public List<TableIdentifier> listTables(Namespace namespace) {
+    validateNamespace(namespace);
+
+    return client.list(toDatasetReference(namespace), listAllTables).stream()
+        .map(
+            table -> TableIdentifier.of(namespace.level(0), table.getTableReference().getTableId()))
+        .collect(ImmutableList.toImmutableList());
+  }
+
+  @Override
+  public boolean dropTable(TableIdentifier identifier, boolean purge) {
+    try {
+      TableOperations ops = newTableOps(identifier);
+      TableMetadata lastMetadata = ops.current();
+
+      client.delete(toTableReference(identifier));
+
+      if (purge && lastMetadata != null) {
+        CatalogUtil.dropTableData(ops.io(), lastMetadata);
+      }
+    } catch (NoSuchTableException e) {
+      return false;
+    }
+
+    return true;
+  }
+
+  @Override
+  public void renameTable(TableIdentifier from, TableIdentifier to) {
+    // TODO: Enable once supported by BigQuery API.
+    throw new UnsupportedOperationException("Table rename operation is unsupported.");
+  }
+
+  @Override
+  public void createNamespace(Namespace namespace, Map<String, String> metadata) {
+    Dataset builder = new Dataset();
+    DatasetReference datasetReference = toDatasetReference(namespace);
+    builder.setLocation(this.projectLocation);
+    builder.setDatasetReference(datasetReference);
+    builder.setExternalCatalogDatasetOptions(
+        BigQueryMetastoreUtils.createExternalCatalogDatasetOptions(
+            createDefaultStorageLocationUri(datasetReference.getDatasetId()), metadata));
+
+    client.create(builder);
+  }
+
+  @Override
+  public List<Namespace> listNamespaces() {
+    try {
+      return listNamespaces(Namespace.empty());
+    } catch (NoSuchNamespaceException e) {
+      return ImmutableList.of();
+    }
+  }
+
+  /**
+   * Since this catalog only supports one-level namespaces, it always returns an empty list unless
+   * passed an empty namespace to list all namespaces within the catalog.
+   */
+  @Override
+  public List<Namespace> listNamespaces(Namespace namespace) {
+    List<Datasets> allDatasets = client.list(projectId);
+
+    ImmutableList<Namespace> namespaces =
+        allDatasets.stream().map(this::toNamespace).collect(ImmutableList.toImmutableList());
+    if (namespaces.isEmpty()) {
+      throw new NoSuchNamespaceException("Namespace does not exist: %s", namespace);
+    }
+
+    return namespaces;
+  }
+
+  @Override
+  public boolean dropNamespace(Namespace namespace) {
+    try {
+      client.delete(toDatasetReference(namespace));
+      // We don't delete the data folder for safety, which aligns with Hive Metastore's default
+      // behavior.
+      // We can support database or catalog level config controlling file deletion in the future.
+      return true;
+    } catch (NoSuchNamespaceException e) {
+      return false;
+    }
+  }
+
+  @Override
+  public boolean setProperties(Namespace namespace, Map<String, String> properties) {
+    Dataset dataset = client.load(toDatasetReference(namespace));
+
+    ExternalCatalogDatasetOptions existingOptions = dataset.getExternalCatalogDatasetOptions();
+    Map<String, String> existingParameters =
+        existingOptions != null ? existingOptions.getParameters() : null;
+
+    Map<String, String> newParameters = Maps.newHashMap();
+    if (existingParameters != null) {
+      newParameters.putAll(existingParameters);
+    }
+
+    newParameters.putAll(properties);
+
+    if (Objects.equals(existingParameters, newParameters)) {
+      // No change in parameters detected
+      return false;
+    }
+
+    client.setParameters(toDatasetReference(namespace), properties);
+    return true;
+  }
+
+  @Override
+  public boolean removeProperties(Namespace namespace, Set<String> properties) {
+    client.removeParameters(toDatasetReference(namespace), properties);
+    return true;
+  }
+
+  @Override
+  public Map<String, String> loadNamespaceMetadata(Namespace namespace) {
+    try {
+      return toMetadata(client.load(toDatasetReference(namespace)));
+    } catch (IllegalArgumentException e) {
+      throw new NoSuchNamespaceException("%s", e.getMessage());
+    }
+  }
+
+  @Override
+  public String name() {
+    return catalogName;
+  }
+
+  @Override
+  protected Map<String, String> properties() {
+    return catalogProperties == null ? ImmutableMap.of() : catalogProperties;
+  }
+
+  @Override
+  public void setConf(Configuration conf) {
+    this.conf = new Configuration(conf);
+  }
+
+  @Override
+  public Configuration getConf() {
+    return this.conf;
+  }
+
+  private String createDefaultStorageLocationUri(String dbId) {
+    Preconditions.checkArgument(
+        this.warehouseLocation != null,
+        String.format(
+            "Invalid data warehouse location: %s not set", CatalogProperties.WAREHOUSE_LOCATION));
+    return String.format("%s/%s.db", LocationUtil.stripTrailingSlash(warehouseLocation), dbId);
+  }
+
+  private Namespace toNamespace(Datasets dataset) {
+    return Namespace.of(dataset.getDatasetReference().getDatasetId());
+  }
+
+  private DatasetReference toDatasetReference(Namespace namespace) {
+    validateNamespace(namespace);
+    return new DatasetReference().setProjectId(projectId).setDatasetId(namespace.level(0));
+  }
+
+  private TableReference toTableReference(TableIdentifier tableIdentifier) {
+    DatasetReference datasetReference = toDatasetReference(tableIdentifier.namespace());
+    return new TableReference()
+        .setProjectId(datasetReference.getProjectId())
+        .setDatasetId(datasetReference.getDatasetId())
+        .setTableId(tableIdentifier.name());
+  }
+
+  private Map<String, String> toMetadata(Dataset dataset) {
+    ExternalCatalogDatasetOptions options = dataset.getExternalCatalogDatasetOptions();
+    Map<String, String> metadata = Maps.newHashMap();
+    if (options != null) {
+      if (options.getParameters() != null) {
+        metadata.putAll(options.getParameters());
+      }
+
+      if (!Strings.isNullOrEmpty(options.getDefaultStorageLocationUri())) {
+        metadata.put("location", options.getDefaultStorageLocationUri());
+      }
+    }
+
+    return metadata;
+  }
+
+  private void validateNamespace(Namespace namespace) {
+    Preconditions.checkArgument(

Review Comment:
   Are there any character set validations we should be doing here?  Glue has some examples of this and I believe there limitations imposed by BigQuery Metastore as well.



-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org

For queries about this service, please contact Infrastructure at:
users@infra.apache.org


---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For additional commands, e-mail: issues-help@iceberg.apache.org


Re: [PR] Catalog: Add BigQuery Metastore Catalog Support [iceberg]

Posted by "danielcweeks (via GitHub)" <gi...@apache.org>.
danielcweeks commented on code in PR #12808:
URL: https://github.com/apache/iceberg/pull/12808#discussion_r2072181229


##########
bigquery/src/main/java/org/apache/iceberg/gcp/bigquery/BigQueryTableOperations.java:
##########
@@ -0,0 +1,296 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.iceberg.gcp.bigquery;
+
+import com.google.api.services.bigquery.model.ExternalCatalogTableOptions;
+import com.google.api.services.bigquery.model.Table;
+import com.google.api.services.bigquery.model.TableReference;
+import java.util.Locale;
+import java.util.Map;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hive.common.StatsSetupConst;
+import org.apache.iceberg.BaseMetastoreOperations;
+import org.apache.iceberg.BaseMetastoreTableOperations;
+import org.apache.iceberg.SnapshotSummary;
+import org.apache.iceberg.TableMetadata;
+import org.apache.iceberg.TableProperties;
+import org.apache.iceberg.exceptions.AlreadyExistsException;
+import org.apache.iceberg.exceptions.CommitFailedException;
+import org.apache.iceberg.exceptions.CommitStateUnknownException;
+import org.apache.iceberg.exceptions.NoSuchTableException;
+import org.apache.iceberg.exceptions.ValidationException;
+import org.apache.iceberg.hadoop.ConfigProperties;
+import org.apache.iceberg.io.FileIO;
+import org.apache.iceberg.relocated.com.google.common.collect.Maps;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/** Handles BigQuery metastore table operations. */
+public final class BigQueryTableOperations extends BaseMetastoreTableOperations {
+
+  private static final Logger LOG = LoggerFactory.getLogger(BigQueryTableOperations.class);
+
+  private static final String TABLE_PROPERTIES_BQ_CONNECTION = "bq_connection";
+
+  private final BigQueryMetastoreClient client;
+  private final FileIO fileIO;
+  private final TableReference tableReference;
+  private final Configuration conf;
+
+  BigQueryTableOperations(
+      BigQueryMetastoreClient client,
+      FileIO fileIO,
+      TableReference tableReference,
+      Configuration conf) {
+    this.client = client;
+    this.fileIO = fileIO;
+    this.tableReference = tableReference;
+    this.conf = conf;
+  }
+
+  // The doRefresh method should provide implementation on how to get the metadata location.
+  @Override
+  public void doRefresh() {
+    // Must default to null.
+    String metadataLocation = null;
+    try {
+      metadataLocation =
+          loadMetadataLocationOrThrow(
+              client.load(this.tableReference).getExternalCatalogTableOptions());
+    } catch (NoSuchTableException e) {
+      if (currentMetadataLocation() != null) {
+        // Re-throws the exception because the table must exist in this case.

Review Comment:
   Do we mean "table must _not_ exist"? Otherwise, why are we throwing NSTE?



-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org

For queries about this service, please contact Infrastructure at:
users@infra.apache.org


---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For additional commands, e-mail: issues-help@iceberg.apache.org


Re: [PR] Catalog: Add BigQuery Metastore Catalog Support [iceberg]

Posted by "talatuyarer (via GitHub)" <gi...@apache.org>.
talatuyarer commented on code in PR #12808:
URL: https://github.com/apache/iceberg/pull/12808#discussion_r2072241375


##########
bigquery/src/main/java/org/apache/iceberg/gcp/bigquery/BigQueryTableOperations.java:
##########
@@ -0,0 +1,296 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.iceberg.gcp.bigquery;
+
+import com.google.api.services.bigquery.model.ExternalCatalogTableOptions;
+import com.google.api.services.bigquery.model.Table;
+import com.google.api.services.bigquery.model.TableReference;
+import java.util.Locale;
+import java.util.Map;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hive.common.StatsSetupConst;
+import org.apache.iceberg.BaseMetastoreOperations;
+import org.apache.iceberg.BaseMetastoreTableOperations;
+import org.apache.iceberg.SnapshotSummary;
+import org.apache.iceberg.TableMetadata;
+import org.apache.iceberg.TableProperties;
+import org.apache.iceberg.exceptions.AlreadyExistsException;
+import org.apache.iceberg.exceptions.CommitFailedException;
+import org.apache.iceberg.exceptions.CommitStateUnknownException;
+import org.apache.iceberg.exceptions.NoSuchTableException;
+import org.apache.iceberg.exceptions.ValidationException;
+import org.apache.iceberg.hadoop.ConfigProperties;
+import org.apache.iceberg.io.FileIO;
+import org.apache.iceberg.relocated.com.google.common.collect.Maps;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/** Handles BigQuery metastore table operations. */
+public final class BigQueryTableOperations extends BaseMetastoreTableOperations {
+
+  private static final Logger LOG = LoggerFactory.getLogger(BigQueryTableOperations.class);
+
+  private static final String TABLE_PROPERTIES_BQ_CONNECTION = "bq_connection";

Review Comment:
   This is BQ's concept. Rather than giving credential user can create "connection" which store necessary permission for GCS bucket. We use this in addConnectionIfProvided to set table to right permission.



-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org

For queries about this service, please contact Infrastructure at:
users@infra.apache.org


---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For additional commands, e-mail: issues-help@iceberg.apache.org


Re: [PR] Catalog: Add BigQuery Metastore Catalog Support [iceberg]

Posted by "danielcweeks (via GitHub)" <gi...@apache.org>.
danielcweeks commented on code in PR #12808:
URL: https://github.com/apache/iceberg/pull/12808#discussion_r2067556600


##########
bigquery/src/main/java/org/apache/iceberg/gcp/bigquery/BigQueryMetastoreCatalog.java:
##########
@@ -0,0 +1,405 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.iceberg.gcp.bigquery;
+
+import com.google.api.services.bigquery.model.Dataset;
+import com.google.api.services.bigquery.model.DatasetList.Datasets;
+import com.google.api.services.bigquery.model.DatasetReference;
+import com.google.api.services.bigquery.model.ExternalCatalogDatasetOptions;
+import com.google.api.services.bigquery.model.TableReference;
+import com.google.cloud.ServiceOptions;
+import com.google.cloud.bigquery.BigQueryOptions;
+import java.io.IOException;
+import java.security.GeneralSecurityException;
+import java.util.List;
+import java.util.Locale;
+import java.util.Map;
+import java.util.Objects;
+import java.util.Set;
+import org.apache.hadoop.conf.Configurable;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.iceberg.BaseMetastoreCatalog;
+import org.apache.iceberg.CatalogProperties;
+import org.apache.iceberg.CatalogUtil;
+import org.apache.iceberg.TableMetadata;
+import org.apache.iceberg.TableOperations;
+import org.apache.iceberg.catalog.Namespace;
+import org.apache.iceberg.catalog.SupportsNamespaces;
+import org.apache.iceberg.catalog.TableIdentifier;
+import org.apache.iceberg.exceptions.AlreadyExistsException;
+import org.apache.iceberg.exceptions.NoSuchNamespaceException;
+import org.apache.iceberg.exceptions.NoSuchTableException;
+import org.apache.iceberg.exceptions.ServiceFailureException;
+import org.apache.iceberg.exceptions.ValidationException;
+import org.apache.iceberg.io.FileIO;
+import org.apache.iceberg.relocated.com.google.common.annotations.VisibleForTesting;
+import org.apache.iceberg.relocated.com.google.common.base.Preconditions;
+import org.apache.iceberg.relocated.com.google.common.base.Strings;
+import org.apache.iceberg.relocated.com.google.common.collect.ImmutableList;
+import org.apache.iceberg.relocated.com.google.common.collect.ImmutableMap;
+import org.apache.iceberg.relocated.com.google.common.collect.Maps;
+import org.apache.iceberg.util.LocationUtil;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/** Iceberg Bigquery Metastore Catalog implementation. */
+public class BigQueryMetastoreCatalog extends BaseMetastoreCatalog
+    implements SupportsNamespaces, Configurable {
+
+  // User provided properties.
+  public static final String PROJECT_ID = "gcp-project";
+  public static final String GCP_LOCATION = "gcp-location";
+  public static final String LIST_ALL_TABLES = "list-all-tables";
+
+  public static final String HIVE_METASTORE_WAREHOUSE_DIR = "hive.metastore.warehouse.dir";
+
+  private static final Logger LOG = LoggerFactory.getLogger(BigQueryMetastoreCatalog.class);
+
+  private static final String DEFAULT_GCP_LOCATION = "us";
+
+  private String catalogName;
+  private Map<String, String> catalogProperties;
+  private FileIO fileIO;
+  private Configuration conf;
+  private String projectId;
+  private String location;

Review Comment:
   I think we probably want to change the name of this field.  `location` is really overloaded and can refer to the catalog/warehouse location, namespace location, table location which are all used throughout this class, but this actually is the `projectLocation`.   



-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org

For queries about this service, please contact Infrastructure at:
users@infra.apache.org


---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For additional commands, e-mail: issues-help@iceberg.apache.org


Re: [PR] Catalog: Add BigQuery Metastore Catalog Support [iceberg]

Posted by "danielcweeks (via GitHub)" <gi...@apache.org>.
danielcweeks commented on code in PR #12808:
URL: https://github.com/apache/iceberg/pull/12808#discussion_r2070910387


##########
bigquery/src/main/java/org/apache/iceberg/gcp/bigquery/BigQueryMetastoreCatalog.java:
##########
@@ -0,0 +1,370 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.iceberg.gcp.bigquery;
+
+import com.google.api.services.bigquery.model.Dataset;
+import com.google.api.services.bigquery.model.DatasetList.Datasets;
+import com.google.api.services.bigquery.model.DatasetReference;
+import com.google.api.services.bigquery.model.ExternalCatalogDatasetOptions;
+import com.google.api.services.bigquery.model.TableReference;
+import com.google.cloud.ServiceOptions;
+import com.google.cloud.bigquery.BigQueryOptions;
+import java.io.IOException;
+import java.security.GeneralSecurityException;
+import java.util.List;
+import java.util.Locale;
+import java.util.Map;
+import java.util.Objects;
+import java.util.Set;
+import org.apache.hadoop.conf.Configurable;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.iceberg.BaseMetastoreCatalog;
+import org.apache.iceberg.CatalogProperties;
+import org.apache.iceberg.CatalogUtil;
+import org.apache.iceberg.TableMetadata;
+import org.apache.iceberg.TableOperations;
+import org.apache.iceberg.catalog.Namespace;
+import org.apache.iceberg.catalog.SupportsNamespaces;
+import org.apache.iceberg.catalog.TableIdentifier;
+import org.apache.iceberg.exceptions.NoSuchNamespaceException;
+import org.apache.iceberg.exceptions.NoSuchTableException;
+import org.apache.iceberg.exceptions.ServiceFailureException;
+import org.apache.iceberg.exceptions.ValidationException;
+import org.apache.iceberg.io.FileIO;
+import org.apache.iceberg.relocated.com.google.common.annotations.VisibleForTesting;
+import org.apache.iceberg.relocated.com.google.common.base.Preconditions;
+import org.apache.iceberg.relocated.com.google.common.base.Strings;
+import org.apache.iceberg.relocated.com.google.common.collect.ImmutableList;
+import org.apache.iceberg.relocated.com.google.common.collect.ImmutableMap;
+import org.apache.iceberg.relocated.com.google.common.collect.Maps;
+import org.apache.iceberg.util.LocationUtil;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/** Iceberg Bigquery Metastore Catalog implementation. */
+public class BigQueryMetastoreCatalog extends BaseMetastoreCatalog
+    implements SupportsNamespaces, Configurable {
+
+  // User provided properties.
+  public static final String PROJECT_ID = "gcp.bigquery.project-id";
+  public static final String GCP_LOCATION = "gcp.bigquery.location";
+  public static final String LIST_ALL_TABLES = "gcp.bigquery.list-all-tables";
+
+  private static final Logger LOG = LoggerFactory.getLogger(BigQueryMetastoreCatalog.class);
+
+  private static final String DEFAULT_GCP_LOCATION = "us";
+
+  private String catalogName;
+  private Map<String, String> catalogProperties;
+  private FileIO fileIO;
+  private Configuration conf;
+  private String projectId;
+  private String projectLocation;
+  private BigQueryMetastoreClient client;
+  private boolean listAllTables;
+  private String warehouseLocation;
+
+  // Must have a no-arg constructor to be dynamically loaded
+  // initialize(String name, Map<String, String> properties) will be called to complete
+  // initialization
+  public BigQueryMetastoreCatalog() {}
+
+  @Override
+  public void initialize(String name, Map<String, String> properties) {
+    Preconditions.checkArgument(
+        properties.containsKey(PROJECT_ID),
+        "Invalid GCP project: %s must be specified",
+        PROJECT_ID);
+
+    this.projectId = properties.get(PROJECT_ID);
+    this.projectLocation = properties.getOrDefault(GCP_LOCATION, DEFAULT_GCP_LOCATION);
+
+    BigQueryOptions options =
+        BigQueryOptions.newBuilder()
+            .setProjectId(projectId)
+            .setLocation(projectLocation)
+            .setRetrySettings(ServiceOptions.getDefaultRetrySettings())
+            .build();
+
+    try {
+      client = new BigQueryMetastoreClientImpl(options);
+    } catch (IOException e) {
+      throw new ServiceFailureException(e, "Creating BigQuery client failed");
+    } catch (GeneralSecurityException e) {
+      throw new ValidationException(e, "Creating BigQuery client failed due to a security issue");
+    }
+
+    initialize(name, properties, projectId, projectLocation, client);
+  }
+
+  @VisibleForTesting
+  void initialize(
+      String inputName,

Review Comment:
   I think I commented on this before with the `initialize` method, but `inputName` is somewhat misleading.  It should just be `name` or `catalogName` if you want to be more specific.



-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org

For queries about this service, please contact Infrastructure at:
users@infra.apache.org


---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For additional commands, e-mail: issues-help@iceberg.apache.org


Re: [PR] Catalog: Add BigQuery Metastore Catalog Support [iceberg]

Posted by "talatuyarer (via GitHub)" <gi...@apache.org>.
talatuyarer commented on code in PR #12808:
URL: https://github.com/apache/iceberg/pull/12808#discussion_r2070953481


##########
bigquery/src/main/java/org/apache/iceberg/gcp/bigquery/BigQueryMetastoreCatalog.java:
##########
@@ -0,0 +1,370 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.iceberg.gcp.bigquery;
+
+import com.google.api.services.bigquery.model.Dataset;
+import com.google.api.services.bigquery.model.DatasetList.Datasets;
+import com.google.api.services.bigquery.model.DatasetReference;
+import com.google.api.services.bigquery.model.ExternalCatalogDatasetOptions;
+import com.google.api.services.bigquery.model.TableReference;
+import com.google.cloud.ServiceOptions;
+import com.google.cloud.bigquery.BigQueryOptions;
+import java.io.IOException;
+import java.security.GeneralSecurityException;
+import java.util.List;
+import java.util.Locale;
+import java.util.Map;
+import java.util.Objects;
+import java.util.Set;
+import org.apache.hadoop.conf.Configurable;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.iceberg.BaseMetastoreCatalog;
+import org.apache.iceberg.CatalogProperties;
+import org.apache.iceberg.CatalogUtil;
+import org.apache.iceberg.TableMetadata;
+import org.apache.iceberg.TableOperations;
+import org.apache.iceberg.catalog.Namespace;
+import org.apache.iceberg.catalog.SupportsNamespaces;
+import org.apache.iceberg.catalog.TableIdentifier;
+import org.apache.iceberg.exceptions.NoSuchNamespaceException;
+import org.apache.iceberg.exceptions.NoSuchTableException;
+import org.apache.iceberg.exceptions.ServiceFailureException;
+import org.apache.iceberg.exceptions.ValidationException;
+import org.apache.iceberg.io.FileIO;
+import org.apache.iceberg.relocated.com.google.common.annotations.VisibleForTesting;
+import org.apache.iceberg.relocated.com.google.common.base.Preconditions;
+import org.apache.iceberg.relocated.com.google.common.base.Strings;
+import org.apache.iceberg.relocated.com.google.common.collect.ImmutableList;
+import org.apache.iceberg.relocated.com.google.common.collect.ImmutableMap;
+import org.apache.iceberg.relocated.com.google.common.collect.Maps;
+import org.apache.iceberg.util.LocationUtil;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/** Iceberg Bigquery Metastore Catalog implementation. */
+public class BigQueryMetastoreCatalog extends BaseMetastoreCatalog
+    implements SupportsNamespaces, Configurable {
+
+  // User provided properties.
+  public static final String PROJECT_ID = "gcp.bigquery.project-id";
+  public static final String GCP_LOCATION = "gcp.bigquery.location";
+  public static final String LIST_ALL_TABLES = "gcp.bigquery.list-all-tables";
+
+  private static final Logger LOG = LoggerFactory.getLogger(BigQueryMetastoreCatalog.class);
+
+  private static final String DEFAULT_GCP_LOCATION = "us";
+
+  private String catalogName;
+  private Map<String, String> catalogProperties;
+  private FileIO fileIO;
+  private Configuration conf;
+  private String projectId;
+  private String projectLocation;
+  private BigQueryMetastoreClient client;
+  private boolean listAllTables;
+  private String warehouseLocation;
+
+  // Must have a no-arg constructor to be dynamically loaded
+  // initialize(String name, Map<String, String> properties) will be called to complete
+  // initialization
+  public BigQueryMetastoreCatalog() {}
+
+  @Override
+  public void initialize(String name, Map<String, String> properties) {
+    Preconditions.checkArgument(
+        properties.containsKey(PROJECT_ID),
+        "Invalid GCP project: %s must be specified",
+        PROJECT_ID);
+
+    this.projectId = properties.get(PROJECT_ID);
+    this.projectLocation = properties.getOrDefault(GCP_LOCATION, DEFAULT_GCP_LOCATION);
+
+    BigQueryOptions options =
+        BigQueryOptions.newBuilder()
+            .setProjectId(projectId)
+            .setLocation(projectLocation)
+            .setRetrySettings(ServiceOptions.getDefaultRetrySettings())
+            .build();
+
+    try {
+      client = new BigQueryMetastoreClientImpl(options);
+    } catch (IOException e) {
+      throw new ServiceFailureException(e, "Creating BigQuery client failed");
+    } catch (GeneralSecurityException e) {
+      throw new ValidationException(e, "Creating BigQuery client failed due to a security issue");
+    }
+
+    initialize(name, properties, projectId, projectLocation, client);
+  }
+
+  @VisibleForTesting
+  void initialize(
+      String inputName,

Review Comment:
   Sorry I used refactoring in Idea. Looks like it does not count all inputName. I changed it



-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org

For queries about this service, please contact Infrastructure at:
users@infra.apache.org


---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For additional commands, e-mail: issues-help@iceberg.apache.org


Re: [PR] Catalog: Add BigQuery Metastore Catalog Support [iceberg]

Posted by "danielcweeks (via GitHub)" <gi...@apache.org>.
danielcweeks commented on code in PR #12808:
URL: https://github.com/apache/iceberg/pull/12808#discussion_r2080090426


##########
bigquery/src/main/java/org/apache/iceberg/gcp/bigquery/BigQueryTableOperations.java:
##########
@@ -0,0 +1,296 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.iceberg.gcp.bigquery;
+
+import com.google.api.services.bigquery.model.ExternalCatalogTableOptions;
+import com.google.api.services.bigquery.model.Table;
+import com.google.api.services.bigquery.model.TableReference;
+import java.util.Locale;
+import java.util.Map;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hive.common.StatsSetupConst;
+import org.apache.iceberg.BaseMetastoreOperations;
+import org.apache.iceberg.BaseMetastoreTableOperations;
+import org.apache.iceberg.SnapshotSummary;
+import org.apache.iceberg.TableMetadata;
+import org.apache.iceberg.TableProperties;
+import org.apache.iceberg.exceptions.AlreadyExistsException;
+import org.apache.iceberg.exceptions.CommitFailedException;
+import org.apache.iceberg.exceptions.CommitStateUnknownException;
+import org.apache.iceberg.exceptions.NoSuchTableException;
+import org.apache.iceberg.exceptions.ValidationException;
+import org.apache.iceberg.hadoop.ConfigProperties;
+import org.apache.iceberg.io.FileIO;
+import org.apache.iceberg.relocated.com.google.common.collect.Maps;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/** Handles BigQuery metastore table operations. */
+public final class BigQueryTableOperations extends BaseMetastoreTableOperations {
+
+  private static final Logger LOG = LoggerFactory.getLogger(BigQueryTableOperations.class);
+
+  private static final String TABLE_PROPERTIES_BQ_CONNECTION = "bq_connection";
+
+  private final BigQueryMetastoreClient client;
+  private final FileIO fileIO;
+  private final TableReference tableReference;
+  private final Configuration conf;
+
+  BigQueryTableOperations(
+      BigQueryMetastoreClient client,
+      FileIO fileIO,
+      TableReference tableReference,
+      Configuration conf) {
+    this.client = client;
+    this.fileIO = fileIO;
+    this.tableReference = tableReference;
+    this.conf = conf;
+  }
+
+  // The doRefresh method should provide implementation on how to get the metadata location.
+  @Override
+  public void doRefresh() {
+    // Must default to null.
+    String metadataLocation = null;
+    try {
+      metadataLocation =
+          loadMetadataLocationOrThrow(
+              client.load(this.tableReference).getExternalCatalogTableOptions());
+    } catch (NoSuchTableException e) {
+      if (currentMetadataLocation() != null) {
+        // Re-throws the exception because the table must exist in this case.
+        throw e;
+      }
+    }
+    refreshFromMetadataLocation(metadataLocation);
+  }
+
+  // The doCommit method should provide implementation on how to update with metadata location
+  // atomically
+  @Override
+  public void doCommit(TableMetadata base, TableMetadata metadata) {
+    String newMetadataLocation =
+        base == null && metadata.metadataFileLocation() != null
+            ? metadata.metadataFileLocation()
+            : writeNewMetadata(metadata, currentVersion() + 1);
+    BaseMetastoreOperations.CommitStatus commitStatus =
+        BaseMetastoreOperations.CommitStatus.FAILURE;
+    try {
+      if (base == null) {
+        createTable(newMetadataLocation, metadata);
+      } else {
+        updateTable(base.metadataFileLocation(), newMetadataLocation, metadata);
+      }
+      commitStatus = BaseMetastoreOperations.CommitStatus.SUCCESS;
+    } catch (CommitFailedException | CommitStateUnknownException e) {
+      throw e;
+    } catch (Throwable e) {
+      LOG.error("Exception thrown on commit: ", e);
+      if (e instanceof AlreadyExistsException) {
+        throw e;
+      }
+      commitStatus =
+          BaseMetastoreOperations.CommitStatus.valueOf(
+              checkCommitStatus(newMetadataLocation, metadata).name());
+      if (commitStatus == BaseMetastoreOperations.CommitStatus.FAILURE) {
+        throw new CommitFailedException(e, "Failed to commit");
+      }
+      if (commitStatus == BaseMetastoreOperations.CommitStatus.UNKNOWN) {
+        throw new CommitStateUnknownException(e);
+      }
+    } finally {
+      try {
+        if (commitStatus == BaseMetastoreOperations.CommitStatus.FAILURE) {
+          LOG.warn("Failed to commit updates to table {}", tableName());
+          io().deleteFile(newMetadataLocation);
+        }
+      } catch (RuntimeException e) {
+        LOG.error(
+            "Failed to cleanup metadata file at {} for table {}",
+            newMetadataLocation,
+            tableName(),
+            e);
+      }
+    }
+  }
+
+  @Override
+  public String tableName() {
+    return String.format("%s.%s", tableReference.getDatasetId(), tableReference.getTableId());
+  }
+
+  @Override
+  public FileIO io() {
+    return fileIO;
+  }
+
+  private void createTable(String newMetadataLocation, TableMetadata metadata) {
+    LOG.debug("Creating a new Iceberg table: {}", tableName());
+    Table tableBuilder = makeNewTable(metadata, newMetadataLocation);
+    tableBuilder.setTableReference(this.tableReference);
+    addConnectionIfProvided(tableBuilder, metadata.properties());
+
+    client.create(tableBuilder);
+  }
+
+  private void addConnectionIfProvided(Table tableBuilder, Map<String, String> metadataProperties) {
+    if (metadataProperties.containsKey(TABLE_PROPERTIES_BQ_CONNECTION)) {
+      tableBuilder
+          .getExternalCatalogTableOptions()
+          .setConnectionId(metadataProperties.get(TABLE_PROPERTIES_BQ_CONNECTION));
+    }
+  }
+
+  /** Update table properties with concurrent update detection using etag. */
+  private void updateTable(
+      String oldMetadataLocation, String newMetadataLocation, TableMetadata metadata) {
+    Table table = client.load(this.tableReference);
+    if (table.getEtag().isEmpty()) {
+      throw new ValidationException(
+          "Etag of legacy table %s is empty, manually update the table via the BigQuery API or"
+              + " recreate and retry",
+          tableName());
+    }
+    ExternalCatalogTableOptions options = table.getExternalCatalogTableOptions();
+    addConnectionIfProvided(table, metadata.properties());
+
+    // If `metadataLocationFromMetastore` is different from metadata location of base, it means
+    // someone has updated metadata location in metastore, which is a conflict update.
+    String metadataLocationFromMetastore =
+        options.getParameters().getOrDefault(METADATA_LOCATION_PROP, "");
+    if (!metadataLocationFromMetastore.isEmpty()
+        && !metadataLocationFromMetastore.equals(oldMetadataLocation)) {
+      throw new CommitFailedException(
+          "Cannot commit base metadata location '%s' is not same as the current table metadata location '%s' for"
+              + " %s.%s",
+          oldMetadataLocation,
+          metadataLocationFromMetastore,
+          tableReference.getDatasetId(),
+          tableReference.getTableId());
+    }
+
+    options.setParameters(buildTableParameters(newMetadataLocation, metadata));
+    try {
+      client.update(tableReference, table);
+    } catch (ValidationException e) {
+      if (e.getMessage().toLowerCase(Locale.ENGLISH).contains("etag mismatch")) {
+        throw new CommitFailedException(
+            "Updating table failed due to conflict updates (etag mismatch). Retry the update");
+      }
+
+      throw e;
+    }
+  }
+
+  // To make the table queryable from Hive, the user would likely be setting the HIVE_ENGINE_ENABLED
+  // parameter.
+  //
+  // TODO: We need to make a decision on how to make the table queryable from Hive.
+  // (could be a server side change or a client side change - that's TBD).
+  private Table makeNewTable(TableMetadata metadata, String metadataFileLocation) {
+    return new Table()
+        .setExternalCatalogTableOptions(
+            BigQueryMetastoreUtils.createExternalCatalogTableOptions(
+                metadata.location(),
+                buildTableParameters(metadataFileLocation, metadata),
+                hiveEngineEnabled(metadata)));

Review Comment:
   I don't know what that means, and my cursory look into it, I'm not sure it's safe.  I'd rather we start without this complexity so we can get a clean implementation in and then worry about adding this in the future.



-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org

For queries about this service, please contact Infrastructure at:
users@infra.apache.org


---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For additional commands, e-mail: issues-help@iceberg.apache.org


Re: [PR] Catalog: Add BigQuery Metastore Catalog Support [iceberg]

Posted by "nastra (via GitHub)" <gi...@apache.org>.
nastra commented on code in PR #12808:
URL: https://github.com/apache/iceberg/pull/12808#discussion_r2063949068


##########
build.gradle:
##########
@@ -657,6 +657,47 @@ project(':iceberg-delta-lake') {
   }
 }
 
+project(':iceberg-bigquery') {
+  test {
+    useJUnitPlatform()
+  }
+
+  dependencies {
+    implementation project(path: ':iceberg-bundled-guava', configuration: 'shadow')
+    implementation project(':iceberg-gcp')

Review Comment:
   does the catalog actually directly depend on this?



-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org

For queries about this service, please contact Infrastructure at:
users@infra.apache.org


---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For additional commands, e-mail: issues-help@iceberg.apache.org


Re: [PR] Catalog: Add BigQuery Metastore Catalog Support [iceberg]

Posted by "nastra (via GitHub)" <gi...@apache.org>.
nastra commented on code in PR #12808:
URL: https://github.com/apache/iceberg/pull/12808#discussion_r2063980222


##########
bigquery/src/main/java/org/apache/iceberg/gcp/bigquery/BigQueryMetastoreCatalog.java:
##########
@@ -0,0 +1,400 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.iceberg.gcp.bigquery;
+
+import com.google.api.services.bigquery.model.Dataset;
+import com.google.api.services.bigquery.model.DatasetList.Datasets;
+import com.google.api.services.bigquery.model.DatasetReference;
+import com.google.api.services.bigquery.model.ExternalCatalogDatasetOptions;
+import com.google.api.services.bigquery.model.TableReference;
+import com.google.cloud.ServiceOptions;
+import com.google.cloud.bigquery.BigQueryOptions;
+import java.io.IOException;
+import java.security.GeneralSecurityException;
+import java.util.List;
+import java.util.Locale;
+import java.util.Map;
+import java.util.Objects;
+import java.util.Set;
+import org.apache.hadoop.conf.Configurable;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.iceberg.BaseMetastoreCatalog;
+import org.apache.iceberg.CatalogProperties;
+import org.apache.iceberg.CatalogUtil;
+import org.apache.iceberg.TableMetadata;
+import org.apache.iceberg.TableOperations;
+import org.apache.iceberg.catalog.Namespace;
+import org.apache.iceberg.catalog.SupportsNamespaces;
+import org.apache.iceberg.catalog.TableIdentifier;
+import org.apache.iceberg.exceptions.AlreadyExistsException;
+import org.apache.iceberg.exceptions.NoSuchNamespaceException;
+import org.apache.iceberg.exceptions.NoSuchTableException;
+import org.apache.iceberg.exceptions.ServiceFailureException;
+import org.apache.iceberg.exceptions.ValidationException;
+import org.apache.iceberg.io.FileIO;
+import org.apache.iceberg.relocated.com.google.common.annotations.VisibleForTesting;
+import org.apache.iceberg.relocated.com.google.common.base.Preconditions;
+import org.apache.iceberg.relocated.com.google.common.base.Strings;
+import org.apache.iceberg.relocated.com.google.common.collect.ImmutableList;
+import org.apache.iceberg.relocated.com.google.common.collect.ImmutableMap;
+import org.apache.iceberg.relocated.com.google.common.collect.Maps;
+import org.apache.iceberg.util.LocationUtil;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/** Iceberg Bigquery Metastore Catalog implementation. */
+public final class BigQueryMetastoreCatalog extends BaseMetastoreCatalog
+    implements SupportsNamespaces, Configurable {
+
+  // User provided properties.
+  public static final String PROJECT_ID = "gcp-project";
+  public static final String GCP_LOCATION = "gcp-location";
+  public static final String FILTER_UNSUPPORTED_TABLES = "filter-unsupported-tables";
+  public static final String TESTING_ENABLED = "testing-enabled";
+
+  public static final String HIVE_METASTORE_WAREHOUSE_DIR = "hive.metastore.warehouse.dir";
+
+  private static final Logger LOG = LoggerFactory.getLogger(BigQueryMetastoreCatalog.class);
+
+  private static final String DEFAULT_GCP_LOCATION = "us";
+
+  private String catalogName;
+  private Map<String, String> catalogProperties;
+  private FileIO fileIO;
+  private Configuration conf;
+  private String projectId;
+  private String location;
+  private BigQueryMetastoreClient client;
+  private boolean filterUnsupportedTables;
+
+  // Must have a no-arg constructor to be dynamically loaded
+  // initialize(String name, Map<String, String> properties) will be called to complete
+  // initialization
+  public BigQueryMetastoreCatalog() {}
+
+  @Override
+  public void initialize(String inputName, Map<String, String> properties) {
+    Preconditions.checkArgument(
+        properties.containsKey(PROJECT_ID), "GCP project must be specified");
+
+    projectId = properties.get(PROJECT_ID);
+    location = properties.getOrDefault(GCP_LOCATION, DEFAULT_GCP_LOCATION);
+    boolean testingEnabled =
+        Boolean.parseBoolean(properties.getOrDefault(TESTING_ENABLED, "false"));
+
+    BigQueryOptions options =
+        BigQueryOptions.newBuilder()
+            .setProjectId(projectId)
+            .setLocation(location)
+            .setRetrySettings(ServiceOptions.getDefaultRetrySettings())
+            .build();
+
+    try {
+      if (testingEnabled) {
+        client = new FakeBigQueryMetastoreClient(options);
+
+      } else {
+        client = new BigQueryMetastoreClientImpl(options);
+      }
+    } catch (IOException e) {
+      throw new ServiceFailureException(e, "Creating BigQuery client failed");
+    } catch (GeneralSecurityException e) {
+      throw new ValidationException(e, "Creating BigQuery client failed due to a security issue");
+    }
+    initialize(inputName, properties, projectId, location, client);
+  }
+
+  @VisibleForTesting
+  void initialize(
+      String inputName,
+      Map<String, String> properties,
+      String initialProjectId,
+      String initialLocation,
+      BigQueryMetastoreClient bigQueryMetaStoreClient) {
+    Preconditions.checkArgument(bigQueryMetaStoreClient != null, "BigQuery client can not be null");
+    this.catalogName = inputName;
+    this.catalogProperties = ImmutableMap.copyOf(properties);
+    this.projectId = initialProjectId;
+    this.location = initialLocation;
+    this.client = bigQueryMetaStoreClient;
+
+    if (this.conf == null) {
+      LOG.warn("No configuration was set, using the default environment Configuration");
+      this.conf = new Configuration();
+    }
+
+    LOG.info("Using BigQuery Metastore Iceberg Catalog: {}", inputName);
+
+    if (properties.containsKey(CatalogProperties.WAREHOUSE_LOCATION)) {
+      // Iceberg always removes trailing slash to avoid paths like "<folder>//data" in file systems
+      // like s3.
+      this.conf.set(
+          HIVE_METASTORE_WAREHOUSE_DIR,
+          LocationUtil.stripTrailingSlash(properties.get(CatalogProperties.WAREHOUSE_LOCATION)));
+    }
+
+    String fileIoImpl =
+        properties.getOrDefault(
+            CatalogProperties.FILE_IO_IMPL, "org.apache.iceberg.hadoop.HadoopFileIO");
+    this.fileIO = CatalogUtil.loadFileIO(fileIoImpl, properties, conf);
+
+    this.filterUnsupportedTables =
+        Boolean.parseBoolean(properties.getOrDefault(FILTER_UNSUPPORTED_TABLES, "false"));
+  }
+
+  @Override
+  protected TableOperations newTableOps(TableIdentifier identifier) {
+    return new BigQueryTableOperations(
+        client,
+        fileIO,
+        projectId,
+        // Sometimes extensions have the namespace contain the table name too, so we are forced to
+        // allow invalid namespace and just take the first part here like other catalog
+        // implementations do.
+        identifier.namespace().level(0),
+        identifier.name(),
+        conf);
+  }
+
+  @Override
+  protected String defaultWarehouseLocation(TableIdentifier identifier) {
+    String locationUri = null;
+    DatasetReference datasetReference = toDatasetReference(identifier.namespace());
+    Dataset dataset = client.getDataset(datasetReference);
+    if (dataset != null && dataset.getExternalCatalogDatasetOptions() != null) {
+      locationUri = dataset.getExternalCatalogDatasetOptions().getDefaultStorageLocationUri();
+    }
+    return String.format(
+        "%s/%s",
+        Strings.isNullOrEmpty(locationUri)
+            ? datasetReference.getDatasetId()
+            : LocationUtil.stripTrailingSlash(locationUri),
+        identifier.name());
+  }
+
+  @Override
+  public List<TableIdentifier> listTables(Namespace namespace) {
+    validateNamespace(namespace);
+
+    return client.listTables(toDatasetReference(namespace), filterUnsupportedTables).stream()
+        .map(
+            table -> TableIdentifier.of(namespace.level(0), table.getTableReference().getTableId()))
+        .collect(ImmutableList.toImmutableList());
+  }
+
+  @Override
+  public boolean dropTable(TableIdentifier identifier, boolean purge) {
+    try {
+      TableOperations ops = newTableOps(identifier);
+      TableMetadata lastMetadata = ops.current();
+
+      client.deleteTable(toTableReference(identifier));
+
+      if (purge && lastMetadata != null) {
+        CatalogUtil.dropTableData(ops.io(), lastMetadata);
+      }
+    } catch (NoSuchTableException e) { // Not catching a NoSuchIcebergTableException on purpose
+      return false; // The documentation says just return false in this case
+    }
+    return true;
+  }
+
+  @Override
+  public void renameTable(TableIdentifier from, TableIdentifier to) {
+    validateNamespace(from.namespace());
+    validateNamespace(to.namespace());
+
+    try {
+      client.getDataset(toDatasetReference(to.namespace()));
+      client.getDataset(toDatasetReference(from.namespace()));
+    } catch (NoSuchNamespaceException e) {
+      throw e;
+    }
+
+    TableReference fromTableRef = toTableReference(from);
+    TableReference toTableRef = toTableReference(to);
+
+    // Check if the source table exists
+    try {
+      client.getTable(fromTableRef);
+    } catch (NoSuchTableException e) {
+      throw new NoSuchTableException("Table does not exist: %s", from.name());
+    }
+
+    // Check if the destination table already exists
+    try {
+      client.getTable(toTableRef);
+      // If getTable succeeds, the destination table exists
+      throw new AlreadyExistsException("Table already exists: %s", to.name());
+    } catch (NoSuchTableException e) {
+      // Destination table does not exist, proceed with throwing exception
+    }
+
+    if (!from.namespace().equals(to.namespace())) {
+      throw new ValidationException("New table name must be in the same namespace");
+    }
+
+    // TODO(b/354981675): Enable once supported by the API.
+    throw new UnsupportedOperationException("Table rename operation is unsupported.");
+  }
+
+  @Override
+  public void createNamespace(Namespace namespace, Map<String, String> metadata) {
+    Dataset builder = new Dataset();
+    DatasetReference datasetReference = toDatasetReference(namespace);
+    builder.setLocation(this.location);
+    builder.setDatasetReference(datasetReference);
+    builder.setExternalCatalogDatasetOptions(
+        BigQueryMetastoreUtils.createExternalCatalogDatasetOptions(
+            datasetReference.getDatasetId(), metadata));
+
+    client.createDataset(builder);
+  }
+
+  /**
+   * Since this catalog only supports one-level namespaces, it always returns an empty list unless
+   * passed an empty namespace to list all namespaces within the catalog.
+   */
+  @Override
+  public List<Namespace> listNamespaces(Namespace namespace) {
+    if (namespace.levels().length != 0) {
+      // BQMS does not support namespaces under database or tables, returns empty.
+      // It is called when dropping a namespace to make sure it's empty (listTables is called as
+      // well), returns empty to unblock deletion.
+      return ImmutableList.of();
+    }
+    return client.listDatasets(projectId).stream()
+        .map(BigQueryMetastoreCatalog::getNamespace)
+        .collect(ImmutableList.toImmutableList());
+  }
+
+  @Override
+  public boolean dropNamespace(Namespace namespace) {
+    try {
+      client.deleteDataset(toDatasetReference(namespace));
+      // We don't delete the data folder for safety, which aligns with Hive Metastore's default
+      // behavior.
+      // We can support database or catalog level config controlling file deletion in the future.
+      return true;
+    } catch (NoSuchNamespaceException e) {
+      return false;
+    }
+  }
+
+  @Override
+  public boolean setProperties(Namespace namespace, Map<String, String> properties) {
+    Dataset dataset = client.getDataset(toDatasetReference(namespace));
+
+    ExternalCatalogDatasetOptions existingOptions = dataset.getExternalCatalogDatasetOptions();
+    Map<String, String> existingParameters =
+        existingOptions != null ? existingOptions.getParameters() : null;
+
+    Map<String, String> newParameters = Maps.newHashMap();
+    if (existingParameters != null) {
+      newParameters.putAll(existingParameters);
+    }
+    newParameters.putAll(properties);
+
+    if (Objects.equals(existingParameters, newParameters)) {
+      // No change in parameters detected
+      return false;
+    }
+
+    client.setDatasetParameters(toDatasetReference(namespace), properties);
+    return true;
+  }
+
+  @Override
+  public boolean removeProperties(Namespace namespace, Set<String> properties) {
+    client.removeDatasetParameters(toDatasetReference(namespace), properties);
+    return true;
+  }
+
+  @SuppressWarnings("FormatStringAnnotation")
+  @Override
+  public Map<String, String> loadNamespaceMetadata(Namespace namespace) {
+    try {
+      return getMetadata(client.getDataset(toDatasetReference(namespace)));
+    } catch (IllegalArgumentException e) {

Review Comment:
   so the IAE is something that the BigQuery client is throwing? I'm a bit worried that if the IAE happens for a different reason that it ends up as a NSNE at the user



-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org

For queries about this service, please contact Infrastructure at:
users@infra.apache.org


---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For additional commands, e-mail: issues-help@iceberg.apache.org


Re: [PR] Catalog: Add BigQuery Metastore Catalog Support [iceberg]

Posted by "nastra (via GitHub)" <gi...@apache.org>.
nastra commented on code in PR #12808:
URL: https://github.com/apache/iceberg/pull/12808#discussion_r2063988105


##########
bigquery/src/main/java/org/apache/iceberg/gcp/bigquery/BigQueryMetastoreCatalog.java:
##########
@@ -0,0 +1,394 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.iceberg.gcp.bigquery;
+
+import com.google.api.services.bigquery.model.Dataset;
+import com.google.api.services.bigquery.model.DatasetList.Datasets;
+import com.google.api.services.bigquery.model.DatasetReference;
+import com.google.api.services.bigquery.model.ExternalCatalogDatasetOptions;
+import com.google.api.services.bigquery.model.TableReference;
+import com.google.cloud.ServiceOptions;
+import com.google.cloud.bigquery.BigQueryOptions;
+import java.io.IOException;
+import java.security.GeneralSecurityException;
+import java.util.List;
+import java.util.Locale;
+import java.util.Map;
+import java.util.Objects;
+import java.util.Set;
+import org.apache.hadoop.conf.Configurable;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.iceberg.BaseMetastoreCatalog;
+import org.apache.iceberg.CatalogProperties;
+import org.apache.iceberg.CatalogUtil;
+import org.apache.iceberg.TableMetadata;
+import org.apache.iceberg.TableOperations;
+import org.apache.iceberg.catalog.Namespace;
+import org.apache.iceberg.catalog.SupportsNamespaces;
+import org.apache.iceberg.catalog.TableIdentifier;
+import org.apache.iceberg.exceptions.AlreadyExistsException;
+import org.apache.iceberg.exceptions.NoSuchNamespaceException;
+import org.apache.iceberg.exceptions.NoSuchTableException;
+import org.apache.iceberg.exceptions.ServiceFailureException;
+import org.apache.iceberg.exceptions.ValidationException;
+import org.apache.iceberg.io.FileIO;
+import org.apache.iceberg.relocated.com.google.common.annotations.VisibleForTesting;
+import org.apache.iceberg.relocated.com.google.common.base.Preconditions;
+import org.apache.iceberg.relocated.com.google.common.base.Strings;
+import org.apache.iceberg.relocated.com.google.common.collect.ImmutableList;
+import org.apache.iceberg.relocated.com.google.common.collect.ImmutableMap;
+import org.apache.iceberg.relocated.com.google.common.collect.Maps;
+import org.apache.iceberg.util.LocationUtil;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/** Iceberg Bigquery Metastore Catalog implementation. */
+public class BigQueryMetastoreCatalog extends BaseMetastoreCatalog
+    implements SupportsNamespaces, Configurable {
+
+  // User provided properties.
+  public static final String PROJECT_ID = "gcp-project";
+  public static final String GCP_LOCATION = "gcp-location";
+  public static final String LIST_ALL_TABLES = "list-all-tables";
+
+  public static final String HIVE_METASTORE_WAREHOUSE_DIR = "hive.metastore.warehouse.dir";
+
+  private static final Logger LOG = LoggerFactory.getLogger(BigQueryMetastoreCatalog.class);
+
+  private static final String DEFAULT_GCP_LOCATION = "us";
+
+  private String catalogName;
+  private Map<String, String> catalogProperties;
+  private FileIO fileIO;
+  private Configuration conf;
+  private String projectId;
+  private String location;
+  private BigQueryMetastoreClient client;
+  private boolean listAllTables;
+
+  // Must have a no-arg constructor to be dynamically loaded
+  // initialize(String name, Map<String, String> properties) will be called to complete
+  // initialization
+  public BigQueryMetastoreCatalog() {}
+
+  @Override
+  public void initialize(String inputName, Map<String, String> properties) {
+    Preconditions.checkArgument(
+        properties.containsKey(PROJECT_ID), "GCP project must be specified");
+
+    projectId = properties.get(PROJECT_ID);
+    location = properties.getOrDefault(GCP_LOCATION, DEFAULT_GCP_LOCATION);
+
+    BigQueryOptions options =
+        BigQueryOptions.newBuilder()
+            .setProjectId(projectId)
+            .setLocation(location)
+            .setRetrySettings(ServiceOptions.getDefaultRetrySettings())
+            .build();
+
+    try {
+      client = new BigQueryMetastoreClientImpl(options);
+    } catch (IOException e) {
+      throw new ServiceFailureException(e, "Creating BigQuery client failed");
+    } catch (GeneralSecurityException e) {
+      throw new ValidationException(e, "Creating BigQuery client failed due to a security issue");
+    }
+
+    initialize(inputName, properties, projectId, location, client);
+  }
+
+  @VisibleForTesting
+  void initialize(
+      String inputName,
+      Map<String, String> properties,
+      String initialProjectId,
+      String initialLocation,
+      BigQueryMetastoreClient bigQueryMetaStoreClient) {
+    Preconditions.checkArgument(bigQueryMetaStoreClient != null, "BigQuery client can not be null");
+    this.catalogName = inputName;
+    this.catalogProperties = ImmutableMap.copyOf(properties);
+    this.projectId = initialProjectId;
+    this.location = initialLocation;
+    this.client = bigQueryMetaStoreClient;
+
+    if (this.conf == null) {
+      LOG.warn("No configuration was set, using the default environment Configuration");
+      this.conf = new Configuration();
+    }
+
+    LOG.info("Using BigQuery Metastore Iceberg Catalog: {}", inputName);
+
+    if (properties.containsKey(CatalogProperties.WAREHOUSE_LOCATION)) {
+      this.conf.set(
+          HIVE_METASTORE_WAREHOUSE_DIR,
+          LocationUtil.stripTrailingSlash(properties.get(CatalogProperties.WAREHOUSE_LOCATION)));
+    }
+
+    String fileIoImpl =
+        properties.getOrDefault(
+            CatalogProperties.FILE_IO_IMPL, "org.apache.iceberg.hadoop.HadoopFileIO");
+    this.fileIO = CatalogUtil.loadFileIO(fileIoImpl, properties, conf);
+
+    this.listAllTables = Boolean.parseBoolean(properties.getOrDefault(LIST_ALL_TABLES, "true"));
+  }
+
+  @Override
+  protected TableOperations newTableOps(TableIdentifier identifier) {
+    return new BigQueryTableOperations(client, fileIO, toTableReference(identifier), conf);
+  }
+
+  @Override
+  protected String defaultWarehouseLocation(TableIdentifier identifier) {
+    String locationUri = null;
+    DatasetReference datasetReference = toDatasetReference(identifier.namespace());
+    Dataset dataset = client.load(datasetReference);
+    if (dataset != null && dataset.getExternalCatalogDatasetOptions() != null) {
+      locationUri = dataset.getExternalCatalogDatasetOptions().getDefaultStorageLocationUri();
+    }
+
+    return String.format(
+        "%s/%s",
+        Strings.isNullOrEmpty(locationUri)
+            ? createDefaultStorageLocationUri(datasetReference.getDatasetId())
+            : LocationUtil.stripTrailingSlash(locationUri),
+        identifier.name());
+  }
+
+  @Override
+  public List<TableIdentifier> listTables(Namespace namespace) {
+    validateNamespace(namespace);
+
+    return client.list(toDatasetReference(namespace), listAllTables).stream()
+        .map(
+            table -> TableIdentifier.of(namespace.level(0), table.getTableReference().getTableId()))
+        .collect(ImmutableList.toImmutableList());
+  }
+
+  @Override
+  public boolean dropTable(TableIdentifier identifier, boolean purge) {
+    try {
+      TableOperations ops = newTableOps(identifier);
+      TableMetadata lastMetadata = ops.current();
+
+      client.delete(toTableReference(identifier));
+
+      if (purge && lastMetadata != null) {
+        CatalogUtil.dropTableData(ops.io(), lastMetadata);
+      }
+    } catch (NoSuchTableException e) {
+      return false;
+    }
+
+    return true;
+  }
+
+  @Override
+  public void renameTable(TableIdentifier from, TableIdentifier to) {
+    validateNamespace(from.namespace());
+    validateNamespace(to.namespace());
+
+    if (!namespaceExists(to.namespace())) {
+      throw new NoSuchNamespaceException("Namespace does not exist: %s", to.namespace());
+    }
+
+    if (!namespaceExists(from.namespace())) {
+      throw new NoSuchNamespaceException("Namespace does not exist: %s", from.namespace());
+    }
+
+    // Check if the source table exists
+    if (!tableExists(from)) {
+      throw new NoSuchTableException("Table does not exist: %s", from.name());
+    }
+
+    if (!from.namespace().equals(to.namespace())) {
+      throw new ValidationException("New table name must be in the same namespace");
+    }
+
+    // Check if the destination table already exists
+    if (tableExists(to)) {
+      throw new AlreadyExistsException("Table already exists: %s", to.name());
+    }
+
+    // TODO: Enable once supported by BigQuery API.
+    throw new UnsupportedOperationException("Table rename operation is unsupported.");
+  }
+
+  @Override
+  public void createNamespace(Namespace namespace, Map<String, String> metadata) {
+    Dataset builder = new Dataset();
+    DatasetReference datasetReference = toDatasetReference(namespace);
+    builder.setLocation(this.location);
+    builder.setDatasetReference(datasetReference);
+    builder.setExternalCatalogDatasetOptions(
+        BigQueryMetastoreUtils.createExternalCatalogDatasetOptions(
+            createDefaultStorageLocationUri(datasetReference.getDatasetId()), metadata));
+
+    client.create(builder);
+  }
+
+  @Override
+  public List<Namespace> listNamespaces() {
+    try {
+      return listNamespaces(Namespace.empty());
+    } catch (NoSuchNamespaceException e) {
+      return ImmutableList.of();
+    }
+  }
+
+  /**
+   * Since this catalog only supports one-level namespaces, it always returns an empty list unless
+   * passed an empty namespace to list all namespaces within the catalog.
+   */
+  @Override
+  public List<Namespace> listNamespaces(Namespace namespace) {
+    List<Datasets> allDatasets = client.list(projectId);
+
+    ImmutableList<Namespace> namespaces =
+        allDatasets.stream()
+            .map(BigQueryMetastoreCatalog::toNamespace)
+            .collect(ImmutableList.toImmutableList());
+    if (namespaces.isEmpty()) {
+      throw new NoSuchNamespaceException("Namespace does not exist: %s", namespace);
+    }
+
+    return namespaces;
+  }
+
+  @Override
+  public boolean dropNamespace(Namespace namespace) {
+    try {
+      client.delete(toDatasetReference(namespace));
+      // We don't delete the data folder for safety, which aligns with Hive Metastore's default
+      // behavior.
+      // We can support database or catalog level config controlling file deletion in the future.
+      return true;
+    } catch (NoSuchNamespaceException e) {
+      return false;
+    }
+  }
+
+  @Override
+  public boolean setProperties(Namespace namespace, Map<String, String> properties) {
+    Dataset dataset = client.load(toDatasetReference(namespace));
+
+    ExternalCatalogDatasetOptions existingOptions = dataset.getExternalCatalogDatasetOptions();
+    Map<String, String> existingParameters =
+        existingOptions != null ? existingOptions.getParameters() : null;
+
+    Map<String, String> newParameters = Maps.newHashMap();
+    if (existingParameters != null) {
+      newParameters.putAll(existingParameters);
+    }
+
+    newParameters.putAll(properties);
+
+    if (Objects.equals(existingParameters, newParameters)) {
+      // No change in parameters detected
+      return false;
+    }
+
+    client.setParameters(toDatasetReference(namespace), properties);
+    return true;
+  }
+
+  @Override
+  public boolean removeProperties(Namespace namespace, Set<String> properties) {
+    client.removeParameters(toDatasetReference(namespace), properties);
+    return true;
+  }
+
+  @SuppressWarnings("FormatStringAnnotation")
+  @Override
+  public Map<String, String> loadNamespaceMetadata(Namespace namespace) {
+    try {
+      return toMetadata(client.load(toDatasetReference(namespace)));
+    } catch (IllegalArgumentException e) {
+      throw new NoSuchNamespaceException(e.getMessage());
+    }
+  }
+
+  @Override
+  public String name() {
+    return catalogName;
+  }
+
+  @Override
+  protected Map<String, String> properties() {
+    return catalogProperties == null ? ImmutableMap.of() : catalogProperties;
+  }
+
+  @Override
+  public void setConf(Configuration conf) {
+    this.conf = new Configuration(conf);
+  }
+
+  @Override
+  public Configuration getConf() {
+    return this.conf;
+  }
+
+  private String createDefaultStorageLocationUri(String dbId) {
+    String warehouseLocation = conf.get(HIVE_METASTORE_WAREHOUSE_DIR);
+    Preconditions.checkNotNull(warehouseLocation, "Data warehouse location is not set");
+    return String.format("%s/%s.db", LocationUtil.stripTrailingSlash(warehouseLocation), dbId);
+  }
+
+  private static Namespace toNamespace(Datasets dataset) {
+    return Namespace.of(dataset.getDatasetReference().getDatasetId());
+  }
+
+  private DatasetReference toDatasetReference(Namespace namespace) {
+    validateNamespace(namespace);
+    return new DatasetReference().setProjectId(projectId).setDatasetId(namespace.level(0));
+  }
+
+  private TableReference toTableReference(TableIdentifier tableIdentifier) {
+    DatasetReference datasetReference = toDatasetReference(tableIdentifier.namespace());
+    return new TableReference()
+        .setProjectId(datasetReference.getProjectId())
+        .setDatasetId(datasetReference.getDatasetId())
+        .setTableId(tableIdentifier.name());
+  }
+
+  private Map<String, String> toMetadata(Dataset dataset) {
+    ExternalCatalogDatasetOptions options = dataset.getExternalCatalogDatasetOptions();
+    Map<String, String> metadata = Maps.newHashMap();
+    if (options != null) {
+      if (options.getParameters() != null) {
+        metadata.putAll(options.getParameters());
+      }
+      if (!Strings.isNullOrEmpty(options.getDefaultStorageLocationUri())) {
+        metadata.put("location", options.getDefaultStorageLocationUri());
+      }
+    }
+
+    return metadata;
+  }
+
+  private static void validateNamespace(Namespace namespace) {

Review Comment:
   I think this can be non-static



-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org

For queries about this service, please contact Infrastructure at:
users@infra.apache.org


---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For additional commands, e-mail: issues-help@iceberg.apache.org


Re: [PR] Catalog: Add BigQuery Metastore Catalog Support [iceberg]

Posted by "nastra (via GitHub)" <gi...@apache.org>.
nastra commented on code in PR #12808:
URL: https://github.com/apache/iceberg/pull/12808#discussion_r2063986064


##########
bigquery/src/main/java/org/apache/iceberg/gcp/bigquery/BigQueryMetastoreCatalog.java:
##########
@@ -0,0 +1,394 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.iceberg.gcp.bigquery;
+
+import com.google.api.services.bigquery.model.Dataset;
+import com.google.api.services.bigquery.model.DatasetList.Datasets;
+import com.google.api.services.bigquery.model.DatasetReference;
+import com.google.api.services.bigquery.model.ExternalCatalogDatasetOptions;
+import com.google.api.services.bigquery.model.TableReference;
+import com.google.cloud.ServiceOptions;
+import com.google.cloud.bigquery.BigQueryOptions;
+import java.io.IOException;
+import java.security.GeneralSecurityException;
+import java.util.List;
+import java.util.Locale;
+import java.util.Map;
+import java.util.Objects;
+import java.util.Set;
+import org.apache.hadoop.conf.Configurable;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.iceberg.BaseMetastoreCatalog;
+import org.apache.iceberg.CatalogProperties;
+import org.apache.iceberg.CatalogUtil;
+import org.apache.iceberg.TableMetadata;
+import org.apache.iceberg.TableOperations;
+import org.apache.iceberg.catalog.Namespace;
+import org.apache.iceberg.catalog.SupportsNamespaces;
+import org.apache.iceberg.catalog.TableIdentifier;
+import org.apache.iceberg.exceptions.AlreadyExistsException;
+import org.apache.iceberg.exceptions.NoSuchNamespaceException;
+import org.apache.iceberg.exceptions.NoSuchTableException;
+import org.apache.iceberg.exceptions.ServiceFailureException;
+import org.apache.iceberg.exceptions.ValidationException;
+import org.apache.iceberg.io.FileIO;
+import org.apache.iceberg.relocated.com.google.common.annotations.VisibleForTesting;
+import org.apache.iceberg.relocated.com.google.common.base.Preconditions;
+import org.apache.iceberg.relocated.com.google.common.base.Strings;
+import org.apache.iceberg.relocated.com.google.common.collect.ImmutableList;
+import org.apache.iceberg.relocated.com.google.common.collect.ImmutableMap;
+import org.apache.iceberg.relocated.com.google.common.collect.Maps;
+import org.apache.iceberg.util.LocationUtil;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/** Iceberg Bigquery Metastore Catalog implementation. */
+public class BigQueryMetastoreCatalog extends BaseMetastoreCatalog
+    implements SupportsNamespaces, Configurable {
+
+  // User provided properties.
+  public static final String PROJECT_ID = "gcp-project";
+  public static final String GCP_LOCATION = "gcp-location";
+  public static final String LIST_ALL_TABLES = "list-all-tables";
+
+  public static final String HIVE_METASTORE_WAREHOUSE_DIR = "hive.metastore.warehouse.dir";
+
+  private static final Logger LOG = LoggerFactory.getLogger(BigQueryMetastoreCatalog.class);
+
+  private static final String DEFAULT_GCP_LOCATION = "us";
+
+  private String catalogName;
+  private Map<String, String> catalogProperties;
+  private FileIO fileIO;
+  private Configuration conf;
+  private String projectId;
+  private String location;
+  private BigQueryMetastoreClient client;
+  private boolean listAllTables;
+
+  // Must have a no-arg constructor to be dynamically loaded
+  // initialize(String name, Map<String, String> properties) will be called to complete
+  // initialization
+  public BigQueryMetastoreCatalog() {}
+
+  @Override
+  public void initialize(String inputName, Map<String, String> properties) {
+    Preconditions.checkArgument(
+        properties.containsKey(PROJECT_ID), "GCP project must be specified");
+
+    projectId = properties.get(PROJECT_ID);
+    location = properties.getOrDefault(GCP_LOCATION, DEFAULT_GCP_LOCATION);
+
+    BigQueryOptions options =
+        BigQueryOptions.newBuilder()
+            .setProjectId(projectId)
+            .setLocation(location)
+            .setRetrySettings(ServiceOptions.getDefaultRetrySettings())
+            .build();
+
+    try {
+      client = new BigQueryMetastoreClientImpl(options);
+    } catch (IOException e) {
+      throw new ServiceFailureException(e, "Creating BigQuery client failed");
+    } catch (GeneralSecurityException e) {
+      throw new ValidationException(e, "Creating BigQuery client failed due to a security issue");
+    }
+
+    initialize(inputName, properties, projectId, location, client);
+  }
+
+  @VisibleForTesting
+  void initialize(
+      String inputName,
+      Map<String, String> properties,
+      String initialProjectId,
+      String initialLocation,
+      BigQueryMetastoreClient bigQueryMetaStoreClient) {
+    Preconditions.checkArgument(bigQueryMetaStoreClient != null, "BigQuery client can not be null");
+    this.catalogName = inputName;
+    this.catalogProperties = ImmutableMap.copyOf(properties);
+    this.projectId = initialProjectId;
+    this.location = initialLocation;
+    this.client = bigQueryMetaStoreClient;
+
+    if (this.conf == null) {
+      LOG.warn("No configuration was set, using the default environment Configuration");
+      this.conf = new Configuration();
+    }
+
+    LOG.info("Using BigQuery Metastore Iceberg Catalog: {}", inputName);
+
+    if (properties.containsKey(CatalogProperties.WAREHOUSE_LOCATION)) {
+      this.conf.set(
+          HIVE_METASTORE_WAREHOUSE_DIR,
+          LocationUtil.stripTrailingSlash(properties.get(CatalogProperties.WAREHOUSE_LOCATION)));
+    }
+
+    String fileIoImpl =
+        properties.getOrDefault(
+            CatalogProperties.FILE_IO_IMPL, "org.apache.iceberg.hadoop.HadoopFileIO");
+    this.fileIO = CatalogUtil.loadFileIO(fileIoImpl, properties, conf);
+
+    this.listAllTables = Boolean.parseBoolean(properties.getOrDefault(LIST_ALL_TABLES, "true"));
+  }
+
+  @Override
+  protected TableOperations newTableOps(TableIdentifier identifier) {
+    return new BigQueryTableOperations(client, fileIO, toTableReference(identifier), conf);
+  }
+
+  @Override
+  protected String defaultWarehouseLocation(TableIdentifier identifier) {
+    String locationUri = null;
+    DatasetReference datasetReference = toDatasetReference(identifier.namespace());
+    Dataset dataset = client.load(datasetReference);
+    if (dataset != null && dataset.getExternalCatalogDatasetOptions() != null) {
+      locationUri = dataset.getExternalCatalogDatasetOptions().getDefaultStorageLocationUri();
+    }
+
+    return String.format(
+        "%s/%s",
+        Strings.isNullOrEmpty(locationUri)
+            ? createDefaultStorageLocationUri(datasetReference.getDatasetId())
+            : LocationUtil.stripTrailingSlash(locationUri),
+        identifier.name());
+  }
+
+  @Override
+  public List<TableIdentifier> listTables(Namespace namespace) {
+    validateNamespace(namespace);
+
+    return client.list(toDatasetReference(namespace), listAllTables).stream()
+        .map(
+            table -> TableIdentifier.of(namespace.level(0), table.getTableReference().getTableId()))
+        .collect(ImmutableList.toImmutableList());
+  }
+
+  @Override
+  public boolean dropTable(TableIdentifier identifier, boolean purge) {
+    try {
+      TableOperations ops = newTableOps(identifier);
+      TableMetadata lastMetadata = ops.current();
+
+      client.delete(toTableReference(identifier));
+
+      if (purge && lastMetadata != null) {
+        CatalogUtil.dropTableData(ops.io(), lastMetadata);
+      }
+    } catch (NoSuchTableException e) {
+      return false;
+    }
+
+    return true;
+  }
+
+  @Override
+  public void renameTable(TableIdentifier from, TableIdentifier to) {
+    validateNamespace(from.namespace());
+    validateNamespace(to.namespace());
+
+    if (!namespaceExists(to.namespace())) {
+      throw new NoSuchNamespaceException("Namespace does not exist: %s", to.namespace());
+    }
+
+    if (!namespaceExists(from.namespace())) {
+      throw new NoSuchNamespaceException("Namespace does not exist: %s", from.namespace());
+    }
+
+    // Check if the source table exists
+    if (!tableExists(from)) {
+      throw new NoSuchTableException("Table does not exist: %s", from.name());
+    }
+
+    if (!from.namespace().equals(to.namespace())) {
+      throw new ValidationException("New table name must be in the same namespace");
+    }
+
+    // Check if the destination table already exists
+    if (tableExists(to)) {
+      throw new AlreadyExistsException("Table already exists: %s", to.name());
+    }
+
+    // TODO: Enable once supported by BigQuery API.
+    throw new UnsupportedOperationException("Table rename operation is unsupported.");
+  }
+
+  @Override
+  public void createNamespace(Namespace namespace, Map<String, String> metadata) {
+    Dataset builder = new Dataset();
+    DatasetReference datasetReference = toDatasetReference(namespace);
+    builder.setLocation(this.location);
+    builder.setDatasetReference(datasetReference);
+    builder.setExternalCatalogDatasetOptions(
+        BigQueryMetastoreUtils.createExternalCatalogDatasetOptions(
+            createDefaultStorageLocationUri(datasetReference.getDatasetId()), metadata));
+
+    client.create(builder);
+  }
+
+  @Override
+  public List<Namespace> listNamespaces() {
+    try {
+      return listNamespaces(Namespace.empty());
+    } catch (NoSuchNamespaceException e) {
+      return ImmutableList.of();
+    }
+  }
+
+  /**
+   * Since this catalog only supports one-level namespaces, it always returns an empty list unless
+   * passed an empty namespace to list all namespaces within the catalog.
+   */
+  @Override
+  public List<Namespace> listNamespaces(Namespace namespace) {
+    List<Datasets> allDatasets = client.list(projectId);
+
+    ImmutableList<Namespace> namespaces =
+        allDatasets.stream()
+            .map(BigQueryMetastoreCatalog::toNamespace)
+            .collect(ImmutableList.toImmutableList());
+    if (namespaces.isEmpty()) {
+      throw new NoSuchNamespaceException("Namespace does not exist: %s", namespace);
+    }
+
+    return namespaces;
+  }
+
+  @Override
+  public boolean dropNamespace(Namespace namespace) {
+    try {
+      client.delete(toDatasetReference(namespace));
+      // We don't delete the data folder for safety, which aligns with Hive Metastore's default
+      // behavior.
+      // We can support database or catalog level config controlling file deletion in the future.
+      return true;
+    } catch (NoSuchNamespaceException e) {
+      return false;
+    }
+  }
+
+  @Override
+  public boolean setProperties(Namespace namespace, Map<String, String> properties) {
+    Dataset dataset = client.load(toDatasetReference(namespace));
+
+    ExternalCatalogDatasetOptions existingOptions = dataset.getExternalCatalogDatasetOptions();
+    Map<String, String> existingParameters =
+        existingOptions != null ? existingOptions.getParameters() : null;
+
+    Map<String, String> newParameters = Maps.newHashMap();
+    if (existingParameters != null) {
+      newParameters.putAll(existingParameters);
+    }
+
+    newParameters.putAll(properties);
+
+    if (Objects.equals(existingParameters, newParameters)) {
+      // No change in parameters detected
+      return false;
+    }
+
+    client.setParameters(toDatasetReference(namespace), properties);
+    return true;
+  }
+
+  @Override
+  public boolean removeProperties(Namespace namespace, Set<String> properties) {
+    client.removeParameters(toDatasetReference(namespace), properties);
+    return true;
+  }
+
+  @SuppressWarnings("FormatStringAnnotation")
+  @Override
+  public Map<String, String> loadNamespaceMetadata(Namespace namespace) {
+    try {
+      return toMetadata(client.load(toDatasetReference(namespace)));
+    } catch (IllegalArgumentException e) {
+      throw new NoSuchNamespaceException(e.getMessage());
+    }
+  }
+
+  @Override
+  public String name() {
+    return catalogName;
+  }
+
+  @Override
+  protected Map<String, String> properties() {
+    return catalogProperties == null ? ImmutableMap.of() : catalogProperties;
+  }
+
+  @Override
+  public void setConf(Configuration conf) {
+    this.conf = new Configuration(conf);
+  }
+
+  @Override
+  public Configuration getConf() {
+    return this.conf;
+  }
+
+  private String createDefaultStorageLocationUri(String dbId) {
+    String warehouseLocation = conf.get(HIVE_METASTORE_WAREHOUSE_DIR);
+    Preconditions.checkNotNull(warehouseLocation, "Data warehouse location is not set");

Review Comment:
   probably better to handle this via IAE and update the error msg to `Invalid data warehouse location: < HIVE_METASTORE_WAREHOUSE_DIR> not set` as we typically would want to mention the actual property that isn't set



-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org

For queries about this service, please contact Infrastructure at:
users@infra.apache.org


---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For additional commands, e-mail: issues-help@iceberg.apache.org


Re: [PR] Catalog: Add BigQuery Metastore Catalog Support [iceberg]

Posted by "nastra (via GitHub)" <gi...@apache.org>.
nastra commented on code in PR #12808:
URL: https://github.com/apache/iceberg/pull/12808#discussion_r2063897797


##########
bigquery/src/test/java/org/apache/iceberg/gcp/bigquery/BigQueryMetastoreTestUtils.java:
##########
@@ -0,0 +1,87 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.iceberg.gcp.bigquery;
+
+import static org.apache.iceberg.types.Types.NestedField.required;
+import static org.assertj.core.api.Assertions.assertThat;
+
+import com.google.api.services.bigquery.model.ExternalCatalogTableOptions;
+import com.google.api.services.bigquery.model.StorageDescriptor;
+import com.google.api.services.bigquery.model.Table;
+import com.google.api.services.bigquery.model.TableReference;
+import java.io.File;
+import java.io.IOException;
+import java.util.Collections;
+import java.util.Optional;
+import org.apache.commons.io.FileUtils;
+import org.apache.commons.io.filefilter.TrueFileFilter;
+import org.apache.iceberg.Schema;
+import org.apache.iceberg.catalog.TableIdentifier;
+import org.apache.iceberg.types.Types;
+
+/** Test utility methods for BigQuery Metastore Iceberg catalog. */
+public final class BigQueryMetastoreTestUtils {
+  private BigQueryMetastoreTestUtils() {}
+
+  public static final String METADATA_LOCATION_PROP = "metadata_location";
+
+  public static Schema getTestSchema() {
+    return new Schema(
+        required(1, "id", Types.IntegerType.get(), "unique ID"),
+        required(2, "data", Types.StringType.get()));
+  }
+
+  public static Table createTestTable(
+      File tempFolder,
+      BigQueryMetastoreCatalog bigQueryMetastoreCatalog,
+      TableReference tableReference)
+      throws IOException {
+    Schema schema = getTestSchema();
+    TableIdentifier tableIdentifier =
+        TableIdentifier.of(tableReference.getDatasetId(), tableReference.getTableId());
+    String tableDir = tempFolder.toPath().resolve(tableReference.getTableId()).toString();
+
+    bigQueryMetastoreCatalog
+        .buildTable(tableIdentifier, schema)
+        .withLocation(tableDir)
+        .createTransaction()
+        .commitTransaction();
+
+    Optional<String> metadataLocation = getIcebergMetadataFilePath(tableDir);
+    assertThat(metadataLocation).isPresent();
+    return new Table()
+        .setTableReference(tableReference)
+        .setExternalCatalogTableOptions(
+            new ExternalCatalogTableOptions()
+                .setStorageDescriptor(new StorageDescriptor().setLocationUri(tableDir))
+                .setParameters(
+                    Collections.singletonMap(METADATA_LOCATION_PROP, metadataLocation.get())));
+  }
+
+  public static Optional<String> getIcebergMetadataFilePath(String tableDir) throws IOException {

Review Comment:
   uses `get` prefix. Also I'm not sure we want to search for metadata given a table directory. There's `ReachableFileUtil#metadataFileLocations` that takes an actual `Table` instance



-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org

For queries about this service, please contact Infrastructure at:
users@infra.apache.org


---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For additional commands, e-mail: issues-help@iceberg.apache.org


Re: [PR] Catalog: Add BigQuery Metastore Catalog Support [iceberg]

Posted by "nastra (via GitHub)" <gi...@apache.org>.
nastra commented on code in PR #12808:
URL: https://github.com/apache/iceberg/pull/12808#discussion_r2063943146


##########
bigquery/src/test/java/org/apache/iceberg/gcp/bigquery/TestBigQueryCatalog.java:
##########
@@ -0,0 +1,191 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.iceberg.gcp.bigquery;
+
+import static org.apache.iceberg.CatalogUtil.ICEBERG_CATALOG_TYPE;
+import static org.apache.iceberg.CatalogUtil.ICEBERG_CATALOG_TYPE_BIGQUERY;
+import static org.apache.iceberg.gcp.bigquery.BigQueryMetastoreCatalog.PROJECT_ID;
+import static org.assertj.core.api.Assertions.assertThat;
+import static org.assertj.core.api.Assertions.assertThatThrownBy;
+
+import java.io.File;
+import java.util.List;
+import java.util.Map;
+import org.apache.iceberg.CatalogProperties;
+import org.apache.iceberg.PartitionSpec;
+import org.apache.iceberg.TableUtil;
+import org.apache.iceberg.catalog.CatalogTests;
+import org.apache.iceberg.catalog.Namespace;
+import org.apache.iceberg.catalog.TableIdentifier;
+import org.apache.iceberg.exceptions.NoSuchNamespaceException;
+import org.apache.iceberg.exceptions.NoSuchTableException;
+import org.apache.iceberg.relocated.com.google.common.collect.ImmutableMap;
+import org.junit.jupiter.api.AfterEach;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Disabled;
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.io.TempDir;
+import org.junit.jupiter.params.ParameterizedTest;
+import org.junit.jupiter.params.provider.ValueSource;
+
+public class TestBigQueryCatalog extends CatalogTests<BigQueryMetastoreCatalog> {
+  @TempDir private File tempFolder;
+  private BigQueryMetastoreCatalog catalog;
+  private String warehouseLocation;
+
+  @BeforeEach
+  public void before() throws Exception {
+    catalog = initCatalog("catalog-name", ImmutableMap.of());
+  }
+
+  @AfterEach
+  public void after() throws Exception {
+    // Drop all tables in all datasets
+    List<Namespace> namespaces = catalog().listNamespaces();
+    for (Namespace namespace : namespaces) {
+      List<TableIdentifier> tables = catalog().listTables(namespace);
+      for (TableIdentifier table : tables) {
+        try {
+          catalog().dropTable(table, true);
+        } catch (NoSuchTableException e) {
+          // Table might be dropped by a previous iteration, ignore
+        }
+      }
+
+      // Drop all datasets (except any initial ones)
+      try {
+        catalog().dropNamespace(namespace);
+      } catch (NoSuchNamespaceException e) {
+        // Namespace might be dropped by a previous iteration, ignore
+      }
+    }
+  }
+
+  @Override
+  protected boolean requiresNamespaceCreate() {
+    return true;
+  }
+
+  @Override
+  protected boolean supportsNamesWithSlashes() {
+    return false;
+  }
+
+  @Override
+  protected boolean supportsNamesWithDot() {
+    return false;
+  }
+
+  @Override
+  public BigQueryMetastoreCatalog catalog() {
+    return catalog;
+  }
+
+  @Override
+  protected BigQueryMetastoreCatalog initCatalog(
+      String catalogName, Map<String, String> additionalProperties) {
+
+    warehouseLocation = tempFolder.toPath().resolve("hive-warehouse").toString();
+    FakeBigQueryMetastoreClient fakeBigQueryClient = new FakeBigQueryMetastoreClient();
+
+    Map<String, String> properties =
+        Map.of(
+            ICEBERG_CATALOG_TYPE,
+            ICEBERG_CATALOG_TYPE_BIGQUERY,
+            PROJECT_ID,
+            "project-id",
+            CatalogProperties.WAREHOUSE_LOCATION,
+            warehouseLocation,
+            CatalogProperties.TABLE_DEFAULT_PREFIX + "default-key1",
+            "catalog-default-key1",
+            CatalogProperties.TABLE_DEFAULT_PREFIX + "default-key2",
+            "catalog-default-key2",
+            CatalogProperties.TABLE_DEFAULT_PREFIX + "override-key3",
+            "catalog-default-key3",
+            CatalogProperties.TABLE_OVERRIDE_PREFIX + "override-key3",
+            "catalog-override-key3",
+            CatalogProperties.TABLE_OVERRIDE_PREFIX + "override-key4",
+            "catalog-override-key4");
+
+    BigQueryMetastoreCatalog tmpCatalog = new BigQueryMetastoreCatalog();
+    tmpCatalog.initialize(
+        catalogName,
+        ImmutableMap.<String, String>builder()
+            .putAll(properties)
+            .putAll(additionalProperties)
+            .build(),
+        "project-id",
+        "us-central1",
+        fakeBigQueryClient);
+
+    return tmpCatalog;
+  }
+
+  @Test
+  public void testRenameTable() {

Review Comment:
   nit: I would also add `@Override` to this method as well



-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org

For queries about this service, please contact Infrastructure at:
users@infra.apache.org


---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For additional commands, e-mail: issues-help@iceberg.apache.org


Re: [PR] Catalog: Add BigQuery Metastore Catalog Support [iceberg]

Posted by "talatuyarer (via GitHub)" <gi...@apache.org>.
talatuyarer commented on code in PR #12808:
URL: https://github.com/apache/iceberg/pull/12808#discussion_r2059735874


##########
bigquery/src/main/java/org/apache/iceberg/gcp/bigquery/BigQueryMetastoreCatalog.java:
##########
@@ -0,0 +1,400 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.iceberg.gcp.bigquery;
+
+import com.google.api.services.bigquery.model.Dataset;
+import com.google.api.services.bigquery.model.DatasetList.Datasets;
+import com.google.api.services.bigquery.model.DatasetReference;
+import com.google.api.services.bigquery.model.ExternalCatalogDatasetOptions;
+import com.google.api.services.bigquery.model.TableReference;
+import com.google.cloud.ServiceOptions;
+import com.google.cloud.bigquery.BigQueryOptions;
+import java.io.IOException;
+import java.security.GeneralSecurityException;
+import java.util.List;
+import java.util.Locale;
+import java.util.Map;
+import java.util.Objects;
+import java.util.Set;
+import org.apache.hadoop.conf.Configurable;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.iceberg.BaseMetastoreCatalog;
+import org.apache.iceberg.CatalogProperties;
+import org.apache.iceberg.CatalogUtil;
+import org.apache.iceberg.TableMetadata;
+import org.apache.iceberg.TableOperations;
+import org.apache.iceberg.catalog.Namespace;
+import org.apache.iceberg.catalog.SupportsNamespaces;
+import org.apache.iceberg.catalog.TableIdentifier;
+import org.apache.iceberg.exceptions.AlreadyExistsException;
+import org.apache.iceberg.exceptions.NoSuchNamespaceException;
+import org.apache.iceberg.exceptions.NoSuchTableException;
+import org.apache.iceberg.exceptions.ServiceFailureException;
+import org.apache.iceberg.exceptions.ValidationException;
+import org.apache.iceberg.io.FileIO;
+import org.apache.iceberg.relocated.com.google.common.annotations.VisibleForTesting;
+import org.apache.iceberg.relocated.com.google.common.base.Preconditions;
+import org.apache.iceberg.relocated.com.google.common.base.Strings;
+import org.apache.iceberg.relocated.com.google.common.collect.ImmutableList;
+import org.apache.iceberg.relocated.com.google.common.collect.ImmutableMap;
+import org.apache.iceberg.relocated.com.google.common.collect.Maps;
+import org.apache.iceberg.util.LocationUtil;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/** Iceberg Bigquery Metastore Catalog implementation. */
+public final class BigQueryMetastoreCatalog extends BaseMetastoreCatalog
+    implements SupportsNamespaces, Configurable {
+
+  // User provided properties.
+  public static final String PROJECT_ID = "gcp-project";
+  public static final String GCP_LOCATION = "gcp-location";
+  public static final String FILTER_UNSUPPORTED_TABLES = "filter-unsupported-tables";
+  public static final String TESTING_ENABLED = "testing-enabled";
+
+  public static final String HIVE_METASTORE_WAREHOUSE_DIR = "hive.metastore.warehouse.dir";
+
+  private static final Logger LOG = LoggerFactory.getLogger(BigQueryMetastoreCatalog.class);
+
+  private static final String DEFAULT_GCP_LOCATION = "us";
+
+  private String catalogName;
+  private Map<String, String> catalogProperties;
+  private FileIO fileIO;
+  private Configuration conf;
+  private String projectId;
+  private String location;
+  private BigQueryMetastoreClient client;
+  private boolean filterUnsupportedTables;
+
+  // Must have a no-arg constructor to be dynamically loaded
+  // initialize(String name, Map<String, String> properties) will be called to complete
+  // initialization
+  public BigQueryMetastoreCatalog() {}
+
+  @Override
+  public void initialize(String inputName, Map<String, String> properties) {
+    Preconditions.checkArgument(
+        properties.containsKey(PROJECT_ID), "GCP project must be specified");
+
+    projectId = properties.get(PROJECT_ID);
+    location = properties.getOrDefault(GCP_LOCATION, DEFAULT_GCP_LOCATION);
+    boolean testingEnabled =

Review Comment:
   Actually I have that initialize method. Let me remove this 



-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org

For queries about this service, please contact Infrastructure at:
users@infra.apache.org


---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For additional commands, e-mail: issues-help@iceberg.apache.org


Re: [PR] Catalog: Add BigQuery Metastore Catalog Support [iceberg]

Posted by "nastra (via GitHub)" <gi...@apache.org>.
nastra commented on code in PR #12808:
URL: https://github.com/apache/iceberg/pull/12808#discussion_r2066013063


##########
bigquery/src/main/java/org/apache/iceberg/gcp/bigquery/BigQueryMetastoreClientImpl.java:
##########
@@ -0,0 +1,594 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.iceberg.gcp.bigquery;
+
+import com.google.api.client.googleapis.javanet.GoogleNetHttpTransport;
+import com.google.api.client.http.HttpHeaders;
+import com.google.api.client.http.HttpResponse;
+import com.google.api.client.http.HttpResponseException;
+import com.google.api.client.http.HttpStatusCodes;
+import com.google.api.client.json.gson.GsonFactory;
+import com.google.api.client.util.Data;
+import com.google.api.services.bigquery.Bigquery;
+import com.google.api.services.bigquery.BigqueryScopes;
+import com.google.api.services.bigquery.model.Dataset;
+import com.google.api.services.bigquery.model.DatasetList;
+import com.google.api.services.bigquery.model.DatasetList.Datasets;
+import com.google.api.services.bigquery.model.DatasetReference;
+import com.google.api.services.bigquery.model.ExternalCatalogDatasetOptions;
+import com.google.api.services.bigquery.model.ExternalCatalogTableOptions;
+import com.google.api.services.bigquery.model.Table;
+import com.google.api.services.bigquery.model.TableList;
+import com.google.api.services.bigquery.model.TableList.Tables;
+import com.google.api.services.bigquery.model.TableReference;
+import com.google.api.services.bigquery.model.TableSchema;
+import com.google.auth.http.HttpCredentialsAdapter;
+import com.google.auth.oauth2.GoogleCredentials;
+import com.google.cloud.BaseServiceException;
+import com.google.cloud.ExceptionHandler;
+import com.google.cloud.bigquery.BigQueryErrorMessages;
+import com.google.cloud.bigquery.BigQueryOptions;
+import com.google.cloud.bigquery.BigQueryRetryConfig;
+import com.google.cloud.bigquery.BigQueryRetryHelper;
+import java.io.IOException;
+import java.nio.charset.StandardCharsets;
+import java.security.GeneralSecurityException;
+import java.util.List;
+import java.util.Locale;
+import java.util.Map;
+import java.util.Set;
+import java.util.stream.Collectors;
+import java.util.stream.Stream;
+import org.apache.iceberg.BaseMetastoreTableOperations;
+import org.apache.iceberg.exceptions.AlreadyExistsException;
+import org.apache.iceberg.exceptions.BadRequestException;
+import org.apache.iceberg.exceptions.ForbiddenException;
+import org.apache.iceberg.exceptions.NoSuchIcebergTableException;
+import org.apache.iceberg.exceptions.NoSuchNamespaceException;
+import org.apache.iceberg.exceptions.NoSuchTableException;
+import org.apache.iceberg.exceptions.NotAuthorizedException;
+import org.apache.iceberg.exceptions.RuntimeIOException;
+import org.apache.iceberg.exceptions.ServiceFailureException;
+import org.apache.iceberg.exceptions.ServiceUnavailableException;
+import org.apache.iceberg.exceptions.ValidationException;
+import org.apache.iceberg.relocated.com.google.common.base.Preconditions;
+import org.apache.iceberg.relocated.com.google.common.collect.Lists;
+import org.apache.iceberg.relocated.com.google.common.collect.Maps;
+
+/** A client of Google Bigquery Metastore functions over the BigQuery service. */
+public final class BigQueryMetastoreClientImpl implements BigQueryMetastoreClient {
+
+  public static final ExceptionHandler.Interceptor EXCEPTION_HANDLER_INTERCEPTOR =
+      new ExceptionHandler.Interceptor() {
+
+        private static final long serialVersionUID = -8429573486870467828L;
+
+        @Override
+        public RetryResult afterEval(Exception exception, RetryResult retryResult) {
+          return ExceptionHandler.Interceptor.RetryResult.CONTINUE_EVALUATION;
+        }
+
+        @Override
+        public RetryResult beforeEval(Exception exception) {
+          if (exception instanceof BaseServiceException) {
+            boolean retriable = ((BaseServiceException) exception).isRetryable();
+            return retriable
+                ? ExceptionHandler.Interceptor.RetryResult.RETRY
+                : ExceptionHandler.Interceptor.RetryResult.CONTINUE_EVALUATION;
+          }
+
+          return ExceptionHandler.Interceptor.RetryResult.CONTINUE_EVALUATION;
+        }
+      };
+
+  private final Bigquery client;
+  private final BigQueryOptions bigqueryOptions;
+
+  // Retry config with error messages and regex for rate limit exceeded errors.
+  private static final BigQueryRetryConfig DEFAULT_RETRY_CONFIG =
+      BigQueryRetryConfig.newBuilder()
+          .retryOnMessage(BigQueryErrorMessages.RATE_LIMIT_EXCEEDED_MSG)
+          .retryOnMessage(BigQueryErrorMessages.JOB_RATE_LIMIT_EXCEEDED_MSG)
+          .retryOnRegEx(BigQueryErrorMessages.RetryRegExPatterns.RATE_LIMIT_EXCEEDED_REGEX)
+          .build();
+
+  public static final ExceptionHandler BIGQUERY_EXCEPTION_HANDLER =
+      ExceptionHandler.newBuilder()
+          .abortOn(RuntimeException.class)
+          .retryOn(
+              java.net.ConnectException
+                  .class) // Retry on connection failures due to transient network issues.
+          .retryOn(
+              java.net.UnknownHostException
+                  .class) // Retry to recover from temporary DNS resolution failures.
+          .addInterceptors(EXCEPTION_HANDLER_INTERCEPTOR)
+          .build();
+
+  /** Constructs a client of the Google BigQuery service. */
+  public BigQueryMetastoreClientImpl(BigQueryOptions options)
+      throws IOException, GeneralSecurityException {
+    // Initialize client that will be used to send requests. This client only needs to be created
+    // once, and can be reused for multiple requests
+    HttpCredentialsAdapter httpCredentialsAdapter =
+        new HttpCredentialsAdapter(
+            GoogleCredentials.getApplicationDefault().createScoped(BigqueryScopes.all()));
+    this.client =
+        new Bigquery.Builder(
+                GoogleNetHttpTransport.newTrustedTransport(),
+                GsonFactory.getDefaultInstance(),
+                httpRequest -> {
+                  httpCredentialsAdapter.initialize(httpRequest);
+                  // Instead of throwing exceptions, analyze the HttpResponse object and inspect its
+                  // status code. This will allow BigQuery API errors to be converted into Iceberg
+                  // exceptions.
+                  httpRequest.setThrowExceptionOnExecuteError(false);
+                })
+            .setApplicationName("BigQuery Metastore Iceberg Catalog Plugin")
+            .build();
+    this.bigqueryOptions = options;
+  }
+
+  @Override
+  public Dataset create(Dataset dataset) {
+    Dataset response = null;
+    try {
+      response =
+          BigQueryRetryHelper.runWithRetries(
+              () -> internalCreate(dataset),
+              bigqueryOptions.getRetrySettings(),
+              BIGQUERY_EXCEPTION_HANDLER,
+              bigqueryOptions.getClock(),
+              DEFAULT_RETRY_CONFIG);
+    } catch (BigQueryRetryHelper.BigQueryRetryHelperException e) {
+      handleBigQueryRetryException(e);
+    }
+    return response;
+  }
+
+  @SuppressWarnings("FormatStringAnnotation")
+  private Dataset internalCreate(Dataset dataset) {
+    try {
+      HttpResponse response =
+          client
+              .datasets()
+              .insert(dataset.getDatasetReference().getProjectId(), dataset)
+              .executeUnparsed();
+      return convertExceptionIfUnsuccessful(response).parseAs(Dataset.class);
+    } catch (IOException e) {
+      throw new RuntimeIOException(e);
+    } catch (AlreadyExistsException e) {
+      throw new AlreadyExistsException("Namespace already exists: " + dataset.getId());
+    }
+  }
+
+  @Override
+  @SuppressWarnings("FormatStringAnnotation")
+  public Dataset load(DatasetReference datasetReference) {
+    try {
+      HttpResponse response =
+          client
+              .datasets()
+              .get(datasetReference.getProjectId(), datasetReference.getDatasetId())
+              .executeUnparsed();
+      if (response.getStatusCode() == HttpStatusCodes.STATUS_CODE_NOT_FOUND) {
+        throw new NoSuchNamespaceException(
+            "Namespace does not exist: " + datasetReference.getDatasetId());
+      }
+
+      return convertExceptionIfUnsuccessful(response).parseAs(Dataset.class);
+    } catch (IOException e) {
+      throw new RuntimeIOException(e);
+    }
+  }
+
+  @Override
+  @SuppressWarnings("FormatStringAnnotation")
+  public void delete(DatasetReference datasetReference) {
+    try {
+      HttpResponse response =
+          client
+              .datasets()
+              .delete(datasetReference.getProjectId(), datasetReference.getDatasetId())
+              .executeUnparsed();
+      if (response.getStatusCode() == HttpStatusCodes.STATUS_CODE_NOT_FOUND) {
+        throw new NoSuchNamespaceException(
+            "Namespace does not exist: " + datasetReference.getDatasetId());
+      }
+
+      convertExceptionIfUnsuccessful(response);
+    } catch (IOException e) {
+      throw new RuntimeIOException(e);
+    }
+  }
+
+  @Override
+  public Dataset setParameters(DatasetReference datasetReference, Map<String, String> parameters) {
+    Dataset dataset = load(datasetReference);
+    ExternalCatalogDatasetOptions externalCatalogDatasetOptions =
+        dataset.getExternalCatalogDatasetOptions() == null
+            ? new ExternalCatalogDatasetOptions()
+            : dataset.getExternalCatalogDatasetOptions();
+    Map<String, String> finalParameters =
+        externalCatalogDatasetOptions.getParameters() == null
+            ? Maps.newHashMap()
+            : externalCatalogDatasetOptions.getParameters();
+    finalParameters.putAll(parameters);
+
+    dataset.setExternalCatalogDatasetOptions(
+        externalCatalogDatasetOptions.setParameters(finalParameters));
+
+    return internalUpdate(dataset);
+  }
+
+  @Override
+  public Dataset removeParameters(DatasetReference datasetReference, Set<String> parameters) {
+    Dataset dataset = load(datasetReference);
+    ExternalCatalogDatasetOptions externalCatalogDatasetOptions =
+        dataset.getExternalCatalogDatasetOptions() == null
+            ? new ExternalCatalogDatasetOptions()
+            : dataset.getExternalCatalogDatasetOptions();
+    Map<String, String> finalParameters =
+        externalCatalogDatasetOptions.getParameters() == null
+            ? Maps.newHashMap()
+            : externalCatalogDatasetOptions.getParameters();
+    parameters.forEach(finalParameters::remove);
+
+    dataset.setExternalCatalogDatasetOptions(
+        externalCatalogDatasetOptions.setParameters(finalParameters));
+
+    return internalUpdate(dataset);
+  }
+
+  @Override
+  public List<Datasets> list(String projectId) {
+    try {
+      String nextPageToken = null;
+      List<Datasets> datasets = Lists.newArrayList();
+      do {
+        HttpResponse pageResponse =
+            client.datasets().list(projectId).setPageToken(nextPageToken).executeUnparsed();
+        DatasetList result =
+            convertExceptionIfUnsuccessful(pageResponse).parseAs(DatasetList.class);
+        nextPageToken = result.getNextPageToken();
+        if (result.getDatasets() != null) {
+          datasets.addAll(result.getDatasets());
+        }
+
+      } while (nextPageToken != null && !nextPageToken.isEmpty());
+      return datasets;
+    } catch (IOException e) {
+      throw new RuntimeIOException(e);
+    }
+  }
+
+  @Override
+  public Table create(Table table) {
+    // Ensure it is an Iceberg table supported by the BigQuery metastore catalog.
+    validateTable(table);
+    // TODO: Ensure table creation is idempotent when handling retries.
+    Table response = null;
+    try {
+      response =
+          BigQueryRetryHelper.runWithRetries(
+              () -> internalCreate(table),
+              bigqueryOptions.getRetrySettings(),
+              BIGQUERY_EXCEPTION_HANDLER,
+              bigqueryOptions.getClock(),
+              DEFAULT_RETRY_CONFIG);
+    } catch (BigQueryRetryHelper.BigQueryRetryHelperException e) {
+      handleBigQueryRetryException(e);
+    }
+
+    return response;
+  }
+
+  @SuppressWarnings("FormatStringAnnotation")

Review Comment:
   same as above



-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org

For queries about this service, please contact Infrastructure at:
users@infra.apache.org


---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For additional commands, e-mail: issues-help@iceberg.apache.org


Re: [PR] Catalog: Add BigQuery Metastore Catalog Support [iceberg]

Posted by "nastra (via GitHub)" <gi...@apache.org>.
nastra commented on code in PR #12808:
URL: https://github.com/apache/iceberg/pull/12808#discussion_r2066022013


##########
bigquery/src/main/java/org/apache/iceberg/gcp/bigquery/BigQueryTableOperations.java:
##########
@@ -0,0 +1,297 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.iceberg.gcp.bigquery;
+
+import com.google.api.services.bigquery.model.ExternalCatalogTableOptions;
+import com.google.api.services.bigquery.model.Table;
+import com.google.api.services.bigquery.model.TableReference;
+import java.util.Locale;
+import java.util.Map;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hive.common.StatsSetupConst;
+import org.apache.iceberg.BaseMetastoreOperations;
+import org.apache.iceberg.BaseMetastoreTableOperations;
+import org.apache.iceberg.SnapshotSummary;
+import org.apache.iceberg.TableMetadata;
+import org.apache.iceberg.TableProperties;
+import org.apache.iceberg.exceptions.AlreadyExistsException;
+import org.apache.iceberg.exceptions.CommitFailedException;
+import org.apache.iceberg.exceptions.CommitStateUnknownException;
+import org.apache.iceberg.exceptions.NoSuchTableException;
+import org.apache.iceberg.exceptions.ValidationException;
+import org.apache.iceberg.hadoop.ConfigProperties;
+import org.apache.iceberg.io.FileIO;
+import org.apache.iceberg.relocated.com.google.common.collect.Maps;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/** Handles BigQuery metastore table operations. */
+public final class BigQueryTableOperations extends BaseMetastoreTableOperations {
+
+  private static final Logger LOG = LoggerFactory.getLogger(BigQueryTableOperations.class);
+
+  private static final String TABLE_PROPERTIES_BQ_CONNECTION = "bq_connection";
+
+  private final BigQueryMetastoreClient client;
+  private final FileIO fileIO;
+  private final TableReference tableReference;
+  private final Configuration conf;
+
+  BigQueryTableOperations(
+      BigQueryMetastoreClient client,
+      FileIO fileIO,
+      TableReference tableReference,
+      Configuration conf) {
+    this.client = client;
+    this.fileIO = fileIO;
+    this.tableReference = tableReference;
+    this.conf = conf;
+  }
+
+  // The doRefresh method should provide implementation on how to get the metadata location.
+  @Override
+  public void doRefresh() {
+    // Must default to null.
+    String metadataLocation = null;
+    try {
+      metadataLocation =
+          getMetadataLocationOrThrow(
+              client.load(this.tableReference).getExternalCatalogTableOptions());
+    } catch (NoSuchTableException e) {
+      if (currentMetadataLocation() != null) {
+        // Re-throws the exception because the table must exist in this case.
+        throw e;
+      }
+    }
+    refreshFromMetadataLocation(metadataLocation);
+  }
+
+  // The doCommit method should provide implementation on how to update with metadata location
+  // atomically
+  @Override
+  public void doCommit(TableMetadata base, TableMetadata metadata) {
+    String newMetadataLocation =
+        base == null && metadata.metadataFileLocation() != null
+            ? metadata.metadataFileLocation()
+            : writeNewMetadata(metadata, currentVersion() + 1);
+    BaseMetastoreOperations.CommitStatus commitStatus =
+        BaseMetastoreOperations.CommitStatus.FAILURE;
+    try {
+      if (base == null) {
+        createTable(newMetadataLocation, metadata);
+      } else {
+        updateTable(base.metadataFileLocation(), newMetadataLocation, metadata);
+      }
+      commitStatus = BaseMetastoreOperations.CommitStatus.SUCCESS;
+    } catch (CommitFailedException | CommitStateUnknownException e) {
+      throw e;
+    } catch (Throwable e) {
+      LOG.error("Exception thrown on commit: ", e);
+      if (e instanceof AlreadyExistsException) {
+        throw e;
+      }
+      commitStatus =
+          BaseMetastoreOperations.CommitStatus.valueOf(
+              checkCommitStatus(newMetadataLocation, metadata).name());
+      if (commitStatus == BaseMetastoreOperations.CommitStatus.FAILURE) {
+        throw new CommitFailedException(e, "Failed to commit");
+      }
+      if (commitStatus == BaseMetastoreOperations.CommitStatus.UNKNOWN) {
+        throw new CommitStateUnknownException(e);
+      }
+    } finally {
+      try {
+        if (commitStatus == BaseMetastoreOperations.CommitStatus.FAILURE) {
+          LOG.warn("Failed to commit updates to table {}", tableName());
+          io().deleteFile(newMetadataLocation);
+        }
+      } catch (RuntimeException e) {
+        LOG.error(
+            "Failed to cleanup metadata file at {} for table {}",
+            newMetadataLocation,
+            tableName(),
+            e);
+      }
+    }
+  }
+
+  @Override
+  public String tableName() {
+    return String.format("%s.%s", tableReference.getDatasetId(), tableReference.getTableId());
+  }
+
+  @Override
+  public FileIO io() {
+    return fileIO;
+  }
+
+  private void createTable(String newMetadataLocation, TableMetadata metadata) {
+    LOG.debug("Creating a new Iceberg table: {}", tableName());
+    Table tableBuilder = makeNewTable(metadata, newMetadataLocation);
+    tableBuilder.setTableReference(this.tableReference);
+    addConnectionIfProvided(tableBuilder, metadata.properties());
+
+    client.create(tableBuilder);
+  }
+
+  private void addConnectionIfProvided(Table tableBuilder, Map<String, String> metadataProperties) {
+    if (metadataProperties.containsKey(TABLE_PROPERTIES_BQ_CONNECTION)) {
+      tableBuilder
+          .getExternalCatalogTableOptions()
+          .setConnectionId(metadataProperties.get(TABLE_PROPERTIES_BQ_CONNECTION));
+    }
+  }
+
+  /** Update table properties with concurrent update detection using etag. */
+  private void updateTable(
+      String oldMetadataLocation, String newMetadataLocation, TableMetadata metadata) {
+    Table table = client.load(this.tableReference);
+    if (table.getEtag().isEmpty()) {
+      throw new ValidationException(
+          "Etag of legacy table %s is empty, manually update the table via the BigQuery API or"
+              + " recreate and retry",
+          tableName());
+    }
+    ExternalCatalogTableOptions options = table.getExternalCatalogTableOptions();
+    addConnectionIfProvided(table, metadata.properties());
+
+    // If `metadataLocationFromMetastore` is different from metadata location of base, it means
+    // someone has updated metadata location in metastore, which is a conflict update.
+    String metadataLocationFromMetastore =
+        options.getParameters().getOrDefault(METADATA_LOCATION_PROP, "");
+    if (!metadataLocationFromMetastore.isEmpty()
+        && !metadataLocationFromMetastore.equals(oldMetadataLocation)) {
+      throw new CommitFailedException(
+          "Cannot commit base metadata location '%s' is not same as the current table metadata location '%s' for"
+              + " %s.%s",
+          oldMetadataLocation,
+          metadataLocationFromMetastore,
+          tableReference.getDatasetId(),
+          tableReference.getTableId());
+    }
+
+    options.setParameters(buildTableParameters(newMetadataLocation, metadata));
+    try {
+      client.update(tableReference, table);
+    } catch (ValidationException e) {
+      if (e.getMessage().toLowerCase(Locale.ENGLISH).contains("etag mismatch")) {
+        throw new CommitFailedException(
+            "Updating table failed due to conflict updates (etag mismatch). Retry the update");
+      }
+
+      throw e;
+    }
+  }
+
+  // To make the table queryable from Hive, the user would likely be setting the HIVE_ENGINE_ENABLED
+  // parameter.
+  //
+  // TODO: We need to make a decision on how to make the table queryable from Hive.
+  // (could be a server side change or a client side change - that's TBD).
+  private Table makeNewTable(TableMetadata metadata, String metadataFileLocation) {
+    boolean hiveEngineEnabled = hiveEngineEnabled(metadata);
+    return new Table()
+        .setExternalCatalogTableOptions(
+            BigQueryMetastoreUtils.createExternalCatalogTableOptions(
+                metadata.location(),
+                buildTableParameters(metadataFileLocation, metadata),
+                hiveEngineEnabled));
+  }
+
+  // Follow Iceberg's HiveTableOperations to populate more table parameters for HMS compatibility.
+  private Map<String, String> buildTableParameters(
+      String metadataFileLocation, TableMetadata metadata) {
+    Map<String, String> parameters = Maps.newHashMap(metadata.properties());
+    if (metadata.uuid() != null) {
+      parameters.put(TableProperties.UUID, metadata.uuid());
+    }
+    if (currentMetadataLocation() != null && !currentMetadataLocation().isEmpty()) {
+      parameters.put(PREVIOUS_METADATA_LOCATION_PROP, currentMetadataLocation());
+    }
+    parameters.put(METADATA_LOCATION_PROP, metadataFileLocation);
+    parameters.put(TABLE_TYPE_PROP, ICEBERG_TABLE_TYPE_VALUE);
+    // Follow HMS to use the EXTERNAL type.
+    parameters.put("EXTERNAL", "TRUE");
+
+    // Hive style basic statistics.
+    updateParametersWithSnapshotMetadata(metadata, parameters);
+    // More Iceberg metadata can be exposed, e.g., statistic, schema, partition spec, as HMS do. But
+    // we should be careful that these metadata could be huge and make the metadata API response
+    // less readable (e.g., list tables). Users can always inspect these metadata in Spark, so they
+    // are not set for now.
+    return parameters;
+  }
+
+  /** Adds Hive-style basic statistics from snapshot metadata if it exists. */
+  private static void updateParametersWithSnapshotMetadata(
+      TableMetadata metadata, Map<String, String> parameters) {
+    if (metadata.currentSnapshot() == null) {
+      return;
+    }
+
+    Map<String, String> summary = metadata.currentSnapshot().summary();
+    if (summary.get(SnapshotSummary.TOTAL_DATA_FILES_PROP) != null) {
+      parameters.put(StatsSetupConst.NUM_FILES, summary.get(SnapshotSummary.TOTAL_DATA_FILES_PROP));
+    }
+
+    if (summary.get(SnapshotSummary.TOTAL_RECORDS_PROP) != null) {
+      parameters.put(StatsSetupConst.ROW_COUNT, summary.get(SnapshotSummary.TOTAL_RECORDS_PROP));
+    }
+
+    if (summary.get(SnapshotSummary.TOTAL_FILE_SIZE_PROP) != null) {
+      parameters.put(StatsSetupConst.TOTAL_SIZE, summary.get(SnapshotSummary.TOTAL_FILE_SIZE_PROP));
+    }
+  }
+
+  private String getMetadataLocationOrThrow(ExternalCatalogTableOptions tableOptions) {

Review Comment:
   ```suggestion
     private String loadMetadataLocationOrThrow(ExternalCatalogTableOptions tableOptions) {
   ```



-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org

For queries about this service, please contact Infrastructure at:
users@infra.apache.org


---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For additional commands, e-mail: issues-help@iceberg.apache.org


Re: [PR] Catalog: Add BigQuery Metastore Catalog Support [iceberg]

Posted by "nastra (via GitHub)" <gi...@apache.org>.
nastra commented on code in PR #12808:
URL: https://github.com/apache/iceberg/pull/12808#discussion_r2063972491


##########
bigquery/src/main/java/org/apache/iceberg/gcp/bigquery/BigQueryMetastoreCatalog.java:
##########
@@ -0,0 +1,394 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.iceberg.gcp.bigquery;
+
+import com.google.api.services.bigquery.model.Dataset;
+import com.google.api.services.bigquery.model.DatasetList.Datasets;
+import com.google.api.services.bigquery.model.DatasetReference;
+import com.google.api.services.bigquery.model.ExternalCatalogDatasetOptions;
+import com.google.api.services.bigquery.model.TableReference;
+import com.google.cloud.ServiceOptions;
+import com.google.cloud.bigquery.BigQueryOptions;
+import java.io.IOException;
+import java.security.GeneralSecurityException;
+import java.util.List;
+import java.util.Locale;
+import java.util.Map;
+import java.util.Objects;
+import java.util.Set;
+import org.apache.hadoop.conf.Configurable;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.iceberg.BaseMetastoreCatalog;
+import org.apache.iceberg.CatalogProperties;
+import org.apache.iceberg.CatalogUtil;
+import org.apache.iceberg.TableMetadata;
+import org.apache.iceberg.TableOperations;
+import org.apache.iceberg.catalog.Namespace;
+import org.apache.iceberg.catalog.SupportsNamespaces;
+import org.apache.iceberg.catalog.TableIdentifier;
+import org.apache.iceberg.exceptions.AlreadyExistsException;
+import org.apache.iceberg.exceptions.NoSuchNamespaceException;
+import org.apache.iceberg.exceptions.NoSuchTableException;
+import org.apache.iceberg.exceptions.ServiceFailureException;
+import org.apache.iceberg.exceptions.ValidationException;
+import org.apache.iceberg.io.FileIO;
+import org.apache.iceberg.relocated.com.google.common.annotations.VisibleForTesting;
+import org.apache.iceberg.relocated.com.google.common.base.Preconditions;
+import org.apache.iceberg.relocated.com.google.common.base.Strings;
+import org.apache.iceberg.relocated.com.google.common.collect.ImmutableList;
+import org.apache.iceberg.relocated.com.google.common.collect.ImmutableMap;
+import org.apache.iceberg.relocated.com.google.common.collect.Maps;
+import org.apache.iceberg.util.LocationUtil;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/** Iceberg Bigquery Metastore Catalog implementation. */
+public class BigQueryMetastoreCatalog extends BaseMetastoreCatalog
+    implements SupportsNamespaces, Configurable {
+
+  // User provided properties.
+  public static final String PROJECT_ID = "gcp-project";
+  public static final String GCP_LOCATION = "gcp-location";
+  public static final String LIST_ALL_TABLES = "list-all-tables";
+
+  public static final String HIVE_METASTORE_WAREHOUSE_DIR = "hive.metastore.warehouse.dir";
+
+  private static final Logger LOG = LoggerFactory.getLogger(BigQueryMetastoreCatalog.class);
+
+  private static final String DEFAULT_GCP_LOCATION = "us";
+
+  private String catalogName;
+  private Map<String, String> catalogProperties;
+  private FileIO fileIO;
+  private Configuration conf;
+  private String projectId;
+  private String location;
+  private BigQueryMetastoreClient client;
+  private boolean listAllTables;
+
+  // Must have a no-arg constructor to be dynamically loaded
+  // initialize(String name, Map<String, String> properties) will be called to complete
+  // initialization
+  public BigQueryMetastoreCatalog() {}
+
+  @Override
+  public void initialize(String inputName, Map<String, String> properties) {
+    Preconditions.checkArgument(
+        properties.containsKey(PROJECT_ID), "GCP project must be specified");
+
+    projectId = properties.get(PROJECT_ID);
+    location = properties.getOrDefault(GCP_LOCATION, DEFAULT_GCP_LOCATION);
+
+    BigQueryOptions options =
+        BigQueryOptions.newBuilder()
+            .setProjectId(projectId)
+            .setLocation(location)
+            .setRetrySettings(ServiceOptions.getDefaultRetrySettings())
+            .build();
+
+    try {
+      client = new BigQueryMetastoreClientImpl(options);
+    } catch (IOException e) {
+      throw new ServiceFailureException(e, "Creating BigQuery client failed");
+    } catch (GeneralSecurityException e) {
+      throw new ValidationException(e, "Creating BigQuery client failed due to a security issue");
+    }
+
+    initialize(inputName, properties, projectId, location, client);
+  }
+
+  @VisibleForTesting
+  void initialize(
+      String inputName,
+      Map<String, String> properties,
+      String initialProjectId,
+      String initialLocation,
+      BigQueryMetastoreClient bigQueryMetaStoreClient) {
+    Preconditions.checkArgument(bigQueryMetaStoreClient != null, "BigQuery client can not be null");
+    this.catalogName = inputName;
+    this.catalogProperties = ImmutableMap.copyOf(properties);
+    this.projectId = initialProjectId;
+    this.location = initialLocation;
+    this.client = bigQueryMetaStoreClient;
+
+    if (this.conf == null) {
+      LOG.warn("No configuration was set, using the default environment Configuration");
+      this.conf = new Configuration();
+    }
+
+    LOG.info("Using BigQuery Metastore Iceberg Catalog: {}", inputName);
+
+    if (properties.containsKey(CatalogProperties.WAREHOUSE_LOCATION)) {
+      this.conf.set(
+          HIVE_METASTORE_WAREHOUSE_DIR,
+          LocationUtil.stripTrailingSlash(properties.get(CatalogProperties.WAREHOUSE_LOCATION)));
+    }
+
+    String fileIoImpl =
+        properties.getOrDefault(
+            CatalogProperties.FILE_IO_IMPL, "org.apache.iceberg.hadoop.HadoopFileIO");
+    this.fileIO = CatalogUtil.loadFileIO(fileIoImpl, properties, conf);
+
+    this.listAllTables = Boolean.parseBoolean(properties.getOrDefault(LIST_ALL_TABLES, "true"));
+  }
+
+  @Override
+  protected TableOperations newTableOps(TableIdentifier identifier) {
+    return new BigQueryTableOperations(client, fileIO, toTableReference(identifier), conf);
+  }
+
+  @Override
+  protected String defaultWarehouseLocation(TableIdentifier identifier) {
+    String locationUri = null;
+    DatasetReference datasetReference = toDatasetReference(identifier.namespace());
+    Dataset dataset = client.load(datasetReference);
+    if (dataset != null && dataset.getExternalCatalogDatasetOptions() != null) {
+      locationUri = dataset.getExternalCatalogDatasetOptions().getDefaultStorageLocationUri();
+    }
+
+    return String.format(
+        "%s/%s",
+        Strings.isNullOrEmpty(locationUri)
+            ? createDefaultStorageLocationUri(datasetReference.getDatasetId())
+            : LocationUtil.stripTrailingSlash(locationUri),
+        identifier.name());
+  }
+
+  @Override
+  public List<TableIdentifier> listTables(Namespace namespace) {
+    validateNamespace(namespace);
+
+    return client.list(toDatasetReference(namespace), listAllTables).stream()
+        .map(
+            table -> TableIdentifier.of(namespace.level(0), table.getTableReference().getTableId()))
+        .collect(ImmutableList.toImmutableList());
+  }
+
+  @Override
+  public boolean dropTable(TableIdentifier identifier, boolean purge) {
+    try {
+      TableOperations ops = newTableOps(identifier);
+      TableMetadata lastMetadata = ops.current();
+
+      client.delete(toTableReference(identifier));
+
+      if (purge && lastMetadata != null) {
+        CatalogUtil.dropTableData(ops.io(), lastMetadata);
+      }
+    } catch (NoSuchTableException e) {
+      return false;
+    }
+
+    return true;
+  }
+
+  @Override
+  public void renameTable(TableIdentifier from, TableIdentifier to) {
+    validateNamespace(from.namespace());
+    validateNamespace(to.namespace());
+
+    if (!namespaceExists(to.namespace())) {
+      throw new NoSuchNamespaceException("Namespace does not exist: %s", to.namespace());
+    }
+
+    if (!namespaceExists(from.namespace())) {
+      throw new NoSuchNamespaceException("Namespace does not exist: %s", from.namespace());
+    }
+
+    // Check if the source table exists
+    if (!tableExists(from)) {
+      throw new NoSuchTableException("Table does not exist: %s", from.name());
+    }
+
+    if (!from.namespace().equals(to.namespace())) {
+      throw new ValidationException("New table name must be in the same namespace");

Review Comment:
   ```suggestion
         throw new ValidationException("Cannot rename %s to %s. New table name must be in the same namespace", from, to);
   ```



-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org

For queries about this service, please contact Infrastructure at:
users@infra.apache.org


---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For additional commands, e-mail: issues-help@iceberg.apache.org


Re: [PR] Catalog: Add BigQuery Metastore Catalog Support [iceberg]

Posted by "nastra (via GitHub)" <gi...@apache.org>.
nastra commented on code in PR #12808:
URL: https://github.com/apache/iceberg/pull/12808#discussion_r2066080376


##########
bigquery/src/test/java/org/apache/iceberg/gcp/bigquery/BigQueryTableOperationsTest.java:
##########
@@ -0,0 +1,274 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.iceberg.gcp.bigquery;
+
+import static org.apache.iceberg.gcp.bigquery.BigQueryMetastoreCatalog.PROJECT_ID;
+import static org.apache.iceberg.types.Types.NestedField.required;
+import static org.assertj.core.api.Assertions.assertThat;
+import static org.assertj.core.api.Assertions.assertThatThrownBy;
+import static org.mockito.Mockito.any;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.reset;
+import static org.mockito.Mockito.times;
+import static org.mockito.Mockito.verify;
+import static org.mockito.Mockito.when;
+
+import com.google.api.services.bigquery.model.Dataset;
+import com.google.api.services.bigquery.model.DatasetReference;
+import com.google.api.services.bigquery.model.ExternalCatalogDatasetOptions;
+import com.google.api.services.bigquery.model.ExternalCatalogTableOptions;
+import com.google.api.services.bigquery.model.StorageDescriptor;
+import com.google.api.services.bigquery.model.Table;
+import com.google.api.services.bigquery.model.TableReference;
+import java.io.File;
+import java.io.IOException;
+import java.util.Collections;
+import java.util.Optional;
+import org.apache.commons.io.FileUtils;
+import org.apache.commons.io.filefilter.TrueFileFilter;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.iceberg.CatalogProperties;
+import org.apache.iceberg.PartitionSpec;
+import org.apache.iceberg.Schema;
+import org.apache.iceberg.catalog.TableIdentifier;
+import org.apache.iceberg.exceptions.CommitFailedException;
+import org.apache.iceberg.exceptions.NoSuchTableException;
+import org.apache.iceberg.exceptions.ValidationException;
+import org.apache.iceberg.relocated.com.google.common.collect.ImmutableMap;
+import org.apache.iceberg.types.Types;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.io.TempDir;
+import org.mockito.ArgumentCaptor;
+
+public class BigQueryTableOperationsTest {
+
+  @TempDir private File tempFolder;
+  public static final String METADATA_LOCATION_PROP = "metadata_location";
+  private static final String GCP_PROJECT = "my-project";
+  private static final String GCP_REGION = "us";
+  private static final String DATASET_ID = "db";
+  private static final String TABLE_ID = "tbl";
+  private static final TableIdentifier SPARK_TABLE_ID = TableIdentifier.of(DATASET_ID, TABLE_ID);
+
+  private static final TableReference TABLE_REFERENCE =
+      new TableReference().setProjectId(GCP_PROJECT).setDatasetId(DATASET_ID).setTableId(TABLE_ID);
+
+  private final BigQueryMetastoreClient bigQueryMetaStoreClient =
+      mock(BigQueryMetastoreClient.class);
+
+  private BigQueryMetastoreCatalog bigQueryMetastoreCatalog;

Review Comment:
   small nit, but might make the code a little bit more readable



##########
bigquery/src/test/java/org/apache/iceberg/gcp/bigquery/BigQueryTableOperationsTest.java:
##########
@@ -0,0 +1,274 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.iceberg.gcp.bigquery;
+
+import static org.apache.iceberg.gcp.bigquery.BigQueryMetastoreCatalog.PROJECT_ID;
+import static org.apache.iceberg.types.Types.NestedField.required;
+import static org.assertj.core.api.Assertions.assertThat;
+import static org.assertj.core.api.Assertions.assertThatThrownBy;
+import static org.mockito.Mockito.any;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.reset;
+import static org.mockito.Mockito.times;
+import static org.mockito.Mockito.verify;
+import static org.mockito.Mockito.when;
+
+import com.google.api.services.bigquery.model.Dataset;
+import com.google.api.services.bigquery.model.DatasetReference;
+import com.google.api.services.bigquery.model.ExternalCatalogDatasetOptions;
+import com.google.api.services.bigquery.model.ExternalCatalogTableOptions;
+import com.google.api.services.bigquery.model.StorageDescriptor;
+import com.google.api.services.bigquery.model.Table;
+import com.google.api.services.bigquery.model.TableReference;
+import java.io.File;
+import java.io.IOException;
+import java.util.Collections;
+import java.util.Optional;
+import org.apache.commons.io.FileUtils;
+import org.apache.commons.io.filefilter.TrueFileFilter;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.iceberg.CatalogProperties;
+import org.apache.iceberg.PartitionSpec;
+import org.apache.iceberg.Schema;
+import org.apache.iceberg.catalog.TableIdentifier;
+import org.apache.iceberg.exceptions.CommitFailedException;
+import org.apache.iceberg.exceptions.NoSuchTableException;
+import org.apache.iceberg.exceptions.ValidationException;
+import org.apache.iceberg.relocated.com.google.common.collect.ImmutableMap;
+import org.apache.iceberg.types.Types;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.io.TempDir;
+import org.mockito.ArgumentCaptor;
+
+public class BigQueryTableOperationsTest {
+
+  @TempDir private File tempFolder;
+  public static final String METADATA_LOCATION_PROP = "metadata_location";
+  private static final String GCP_PROJECT = "my-project";
+  private static final String GCP_REGION = "us";
+  private static final String DATASET_ID = "db";
+  private static final String TABLE_ID = "tbl";
+  private static final TableIdentifier SPARK_TABLE_ID = TableIdentifier.of(DATASET_ID, TABLE_ID);
+
+  private static final TableReference TABLE_REFERENCE =
+      new TableReference().setProjectId(GCP_PROJECT).setDatasetId(DATASET_ID).setTableId(TABLE_ID);
+
+  private final BigQueryMetastoreClient bigQueryMetaStoreClient =

Review Comment:
   ```suggestion
     private final BigQueryMetastoreClient client =
   ```



-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org

For queries about this service, please contact Infrastructure at:
users@infra.apache.org


---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For additional commands, e-mail: issues-help@iceberg.apache.org


Re: [PR] Catalog: Add BigQuery Metastore Catalog Support [iceberg]

Posted by "nastra (via GitHub)" <gi...@apache.org>.
nastra commented on code in PR #12808:
URL: https://github.com/apache/iceberg/pull/12808#discussion_r2054371218


##########
bigquery/src/main/java/org/apache/iceberg/gcp/bigquery/BigQueryMetastoreCatalog.java:
##########
@@ -0,0 +1,402 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.iceberg.gcp.bigquery;
+
+import com.google.api.services.bigquery.model.Dataset;
+import com.google.api.services.bigquery.model.DatasetList.Datasets;
+import com.google.api.services.bigquery.model.DatasetReference;
+import com.google.api.services.bigquery.model.ExternalCatalogDatasetOptions;
+import com.google.api.services.bigquery.model.TableReference;
+import com.google.cloud.ServiceOptions;
+import com.google.cloud.bigquery.BigQueryOptions;
+import java.io.IOException;
+import java.security.GeneralSecurityException;
+import java.util.List;
+import java.util.Locale;
+import java.util.Map;
+import java.util.Objects;
+import java.util.Set;
+import org.apache.hadoop.conf.Configurable;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.iceberg.BaseMetastoreCatalog;
+import org.apache.iceberg.CatalogProperties;
+import org.apache.iceberg.CatalogUtil;
+import org.apache.iceberg.TableMetadata;
+import org.apache.iceberg.TableOperations;
+import org.apache.iceberg.catalog.Namespace;
+import org.apache.iceberg.catalog.SupportsNamespaces;
+import org.apache.iceberg.catalog.TableIdentifier;
+import org.apache.iceberg.exceptions.AlreadyExistsException;
+import org.apache.iceberg.exceptions.NoSuchNamespaceException;
+import org.apache.iceberg.exceptions.NoSuchTableException;
+import org.apache.iceberg.exceptions.ServiceFailureException;
+import org.apache.iceberg.exceptions.ValidationException;
+import org.apache.iceberg.io.FileIO;
+import org.apache.iceberg.relocated.com.google.common.annotations.VisibleForTesting;
+import org.apache.iceberg.relocated.com.google.common.base.Preconditions;
+import org.apache.iceberg.relocated.com.google.common.base.Strings;
+import org.apache.iceberg.relocated.com.google.common.collect.ImmutableList;
+import org.apache.iceberg.relocated.com.google.common.collect.ImmutableMap;
+import org.apache.iceberg.relocated.com.google.common.collect.Maps;
+import org.apache.iceberg.util.LocationUtil;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/** Iceberg Bigquery Metastore Catalog implementation. */
+public final class BigQueryMetastoreCatalog extends BaseMetastoreCatalog
+    implements SupportsNamespaces, Configurable {
+
+  // User provided properties.
+  public static final String PROPERTIES_KEY_GCP_PROJECT = "gcp-project";
+  public static final String PROPERTIES_KEY_GCP_LOCATION = "gcp-location";
+  public static final String PROPERTIES_KEY_FILTER_UNSUPPORTED_TABLES = "filter-unsupported-tables";
+  public static final String PROPERTIES_KEY_TESTING_ENABLED = "testing-enabled";

Review Comment:
   does it make sense to add `testing-enabled` and `filter-unsupported-tables` to the Iceberg codebase?



-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org

For queries about this service, please contact Infrastructure at:
users@infra.apache.org


---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For additional commands, e-mail: issues-help@iceberg.apache.org


Re: [PR] Catalog: Add BigQuery Metastore Catalog Support [iceberg]

Posted by "nastra (via GitHub)" <gi...@apache.org>.
nastra commented on code in PR #12808:
URL: https://github.com/apache/iceberg/pull/12808#discussion_r2055556342


##########
bigquery/src/main/java/org/apache/iceberg/gcp/bigquery/BigQueryMetastoreClient.java:
##########
@@ -0,0 +1,121 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.iceberg.gcp.bigquery;
+
+import com.google.api.services.bigquery.model.Dataset;
+import com.google.api.services.bigquery.model.DatasetList.Datasets;
+import com.google.api.services.bigquery.model.DatasetReference;
+import com.google.api.services.bigquery.model.Table;
+import com.google.api.services.bigquery.model.TableList.Tables;
+import com.google.api.services.bigquery.model.TableReference;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+
+/**
+ * A client of Google BigQuery Metastore functions over the BigQuery service. Uses the Google
+ * BigQuery API.
+ */
+public interface BigQueryMetastoreClient {
+
+  /**
+   * Creates and returns a new dataset.
+   *
+   * @param dataset the dataset to create
+   */
+  Dataset createDataset(Dataset dataset);

Review Comment:
   maybe just `Dataset create(Dataset dataset)` as it's obvious from the parameter that this creates a Dataset?



-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org

For queries about this service, please contact Infrastructure at:
users@infra.apache.org


---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For additional commands, e-mail: issues-help@iceberg.apache.org


Re: [PR] Catalog: Add BigQuery Metastore Catalog Support [iceberg]

Posted by "nastra (via GitHub)" <gi...@apache.org>.
nastra commented on code in PR #12808:
URL: https://github.com/apache/iceberg/pull/12808#discussion_r2055516525


##########
build.gradle:
##########
@@ -657,6 +657,47 @@ project(':iceberg-delta-lake') {
   }
 }
 
+project(':iceberg-bigquery') {
+  test {
+    useJUnitPlatform()
+  }
+
+  dependencies {
+    implementation project(path: ':iceberg-bundled-guava', configuration: 'shadow')
+    implementation project(':iceberg-gcp')
+
+    implementation platform(libs.google.libraries.bom)
+    compileOnly "com.google.cloud:google-cloud-storage"
+    implementation "com.google.cloud:google-cloud-bigquery"
+    implementation "com.google.cloud:google-cloud-core"
+
+    api project(':iceberg-api')
+    implementation project(':iceberg-common')
+    implementation project(':iceberg-core')
+
+    compileOnly libs.hadoop3.common
+    compileOnly("org.apache.hive:hive-metastore:4.0.1") {

Review Comment:
   this should be defined in `libs.versions.toml` as `hive4-metastore` and a separate `hive4` that points to that version



-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org

For queries about this service, please contact Infrastructure at:
users@infra.apache.org


---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For additional commands, e-mail: issues-help@iceberg.apache.org


Re: [PR] Catalog: Add BigQuery Metastore Catalog Support [iceberg]

Posted by "nastra (via GitHub)" <gi...@apache.org>.
nastra commented on code in PR #12808:
URL: https://github.com/apache/iceberg/pull/12808#discussion_r2055647655


##########
bigquery/src/main/java/org/apache/iceberg/gcp/bigquery/BigQueryMetastoreCatalog.java:
##########
@@ -0,0 +1,400 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.iceberg.gcp.bigquery;
+
+import com.google.api.services.bigquery.model.Dataset;
+import com.google.api.services.bigquery.model.DatasetList.Datasets;
+import com.google.api.services.bigquery.model.DatasetReference;
+import com.google.api.services.bigquery.model.ExternalCatalogDatasetOptions;
+import com.google.api.services.bigquery.model.TableReference;
+import com.google.cloud.ServiceOptions;
+import com.google.cloud.bigquery.BigQueryOptions;
+import java.io.IOException;
+import java.security.GeneralSecurityException;
+import java.util.List;
+import java.util.Locale;
+import java.util.Map;
+import java.util.Objects;
+import java.util.Set;
+import org.apache.hadoop.conf.Configurable;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.iceberg.BaseMetastoreCatalog;
+import org.apache.iceberg.CatalogProperties;
+import org.apache.iceberg.CatalogUtil;
+import org.apache.iceberg.TableMetadata;
+import org.apache.iceberg.TableOperations;
+import org.apache.iceberg.catalog.Namespace;
+import org.apache.iceberg.catalog.SupportsNamespaces;
+import org.apache.iceberg.catalog.TableIdentifier;
+import org.apache.iceberg.exceptions.AlreadyExistsException;
+import org.apache.iceberg.exceptions.NoSuchNamespaceException;
+import org.apache.iceberg.exceptions.NoSuchTableException;
+import org.apache.iceberg.exceptions.ServiceFailureException;
+import org.apache.iceberg.exceptions.ValidationException;
+import org.apache.iceberg.io.FileIO;
+import org.apache.iceberg.relocated.com.google.common.annotations.VisibleForTesting;
+import org.apache.iceberg.relocated.com.google.common.base.Preconditions;
+import org.apache.iceberg.relocated.com.google.common.base.Strings;
+import org.apache.iceberg.relocated.com.google.common.collect.ImmutableList;
+import org.apache.iceberg.relocated.com.google.common.collect.ImmutableMap;
+import org.apache.iceberg.relocated.com.google.common.collect.Maps;
+import org.apache.iceberg.util.LocationUtil;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/** Iceberg Bigquery Metastore Catalog implementation. */
+public final class BigQueryMetastoreCatalog extends BaseMetastoreCatalog
+    implements SupportsNamespaces, Configurable {
+
+  // User provided properties.
+  public static final String PROJECT_ID = "gcp-project";
+  public static final String GCP_LOCATION = "gcp-location";
+  public static final String FILTER_UNSUPPORTED_TABLES = "filter-unsupported-tables";
+  public static final String TESTING_ENABLED = "testing-enabled";
+
+  public static final String HIVE_METASTORE_WAREHOUSE_DIR = "hive.metastore.warehouse.dir";
+
+  private static final Logger LOG = LoggerFactory.getLogger(BigQueryMetastoreCatalog.class);
+
+  private static final String DEFAULT_GCP_LOCATION = "us";
+
+  private String catalogName;
+  private Map<String, String> catalogProperties;
+  private FileIO fileIO;
+  private Configuration conf;
+  private String projectId;
+  private String location;
+  private BigQueryMetastoreClient client;
+  private boolean filterUnsupportedTables;
+
+  // Must have a no-arg constructor to be dynamically loaded
+  // initialize(String name, Map<String, String> properties) will be called to complete
+  // initialization
+  public BigQueryMetastoreCatalog() {}
+
+  @Override
+  public void initialize(String inputName, Map<String, String> properties) {
+    Preconditions.checkArgument(
+        properties.containsKey(PROJECT_ID), "GCP project must be specified");
+
+    projectId = properties.get(PROJECT_ID);
+    location = properties.getOrDefault(GCP_LOCATION, DEFAULT_GCP_LOCATION);
+    boolean testingEnabled =
+        Boolean.parseBoolean(properties.getOrDefault(TESTING_ENABLED, "false"));
+
+    BigQueryOptions options =
+        BigQueryOptions.newBuilder()
+            .setProjectId(projectId)
+            .setLocation(location)
+            .setRetrySettings(ServiceOptions.getDefaultRetrySettings())
+            .build();
+
+    try {
+      if (testingEnabled) {
+        client = new FakeBigQueryMetastoreClient(options);
+
+      } else {
+        client = new BigQueryMetastoreClientImpl(options);
+      }
+    } catch (IOException e) {
+      throw new ServiceFailureException(e, "Creating BigQuery client failed");
+    } catch (GeneralSecurityException e) {
+      throw new ValidationException(e, "Creating BigQuery client failed due to a security issue");
+    }
+    initialize(inputName, properties, projectId, location, client);
+  }
+
+  @VisibleForTesting
+  void initialize(
+      String inputName,
+      Map<String, String> properties,
+      String initialProjectId,
+      String initialLocation,
+      BigQueryMetastoreClient bigQueryMetaStoreClient) {
+    Preconditions.checkArgument(bigQueryMetaStoreClient != null, "BigQuery client can not be null");
+    this.catalogName = inputName;
+    this.catalogProperties = ImmutableMap.copyOf(properties);
+    this.projectId = initialProjectId;
+    this.location = initialLocation;
+    this.client = bigQueryMetaStoreClient;
+
+    if (this.conf == null) {
+      LOG.warn("No configuration was set, using the default environment Configuration");
+      this.conf = new Configuration();
+    }
+
+    LOG.info("Using BigQuery Metastore Iceberg Catalog: {}", inputName);
+
+    if (properties.containsKey(CatalogProperties.WAREHOUSE_LOCATION)) {
+      // Iceberg always removes trailing slash to avoid paths like "<folder>//data" in file systems
+      // like s3.
+      this.conf.set(
+          HIVE_METASTORE_WAREHOUSE_DIR,
+          LocationUtil.stripTrailingSlash(properties.get(CatalogProperties.WAREHOUSE_LOCATION)));
+    }
+
+    String fileIoImpl =
+        properties.getOrDefault(
+            CatalogProperties.FILE_IO_IMPL, "org.apache.iceberg.hadoop.HadoopFileIO");
+    this.fileIO = CatalogUtil.loadFileIO(fileIoImpl, properties, conf);
+
+    this.filterUnsupportedTables =
+        Boolean.parseBoolean(properties.getOrDefault(FILTER_UNSUPPORTED_TABLES, "false"));
+  }
+
+  @Override
+  protected TableOperations newTableOps(TableIdentifier identifier) {
+    return new BigQueryTableOperations(
+        client,
+        fileIO,
+        projectId,
+        // Sometimes extensions have the namespace contain the table name too, so we are forced to
+        // allow invalid namespace and just take the first part here like other catalog
+        // implementations do.
+        identifier.namespace().level(0),
+        identifier.name(),
+        conf);
+  }
+
+  @Override
+  protected String defaultWarehouseLocation(TableIdentifier identifier) {
+    String locationUri = null;
+    DatasetReference datasetReference = toDatasetReference(identifier.namespace());
+    Dataset dataset = client.getDataset(datasetReference);
+    if (dataset != null && dataset.getExternalCatalogDatasetOptions() != null) {
+      locationUri = dataset.getExternalCatalogDatasetOptions().getDefaultStorageLocationUri();
+    }
+    return String.format(
+        "%s/%s",
+        Strings.isNullOrEmpty(locationUri)
+            ? datasetReference.getDatasetId()
+            : LocationUtil.stripTrailingSlash(locationUri),
+        identifier.name());
+  }
+
+  @Override
+  public List<TableIdentifier> listTables(Namespace namespace) {
+    validateNamespace(namespace);
+
+    return client.listTables(toDatasetReference(namespace), filterUnsupportedTables).stream()
+        .map(
+            table -> TableIdentifier.of(namespace.level(0), table.getTableReference().getTableId()))
+        .collect(ImmutableList.toImmutableList());
+  }
+
+  @Override
+  public boolean dropTable(TableIdentifier identifier, boolean purge) {
+    try {
+      TableOperations ops = newTableOps(identifier);
+      TableMetadata lastMetadata = ops.current();
+
+      client.deleteTable(toTableReference(identifier));
+
+      if (purge && lastMetadata != null) {
+        CatalogUtil.dropTableData(ops.io(), lastMetadata);
+      }
+    } catch (NoSuchTableException e) { // Not catching a NoSuchIcebergTableException on purpose

Review Comment:
   both comments seem rather unnecessary to have



-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org

For queries about this service, please contact Infrastructure at:
users@infra.apache.org


---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For additional commands, e-mail: issues-help@iceberg.apache.org


Re: [PR] Catalog: Add BigQuery Metastore Catalog Support [iceberg]

Posted by "nastra (via GitHub)" <gi...@apache.org>.
nastra commented on code in PR #12808:
URL: https://github.com/apache/iceberg/pull/12808#discussion_r2055660486


##########
bigquery/src/main/java/org/apache/iceberg/gcp/bigquery/BigQueryMetastoreCatalog.java:
##########
@@ -0,0 +1,400 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.iceberg.gcp.bigquery;
+
+import com.google.api.services.bigquery.model.Dataset;
+import com.google.api.services.bigquery.model.DatasetList.Datasets;
+import com.google.api.services.bigquery.model.DatasetReference;
+import com.google.api.services.bigquery.model.ExternalCatalogDatasetOptions;
+import com.google.api.services.bigquery.model.TableReference;
+import com.google.cloud.ServiceOptions;
+import com.google.cloud.bigquery.BigQueryOptions;
+import java.io.IOException;
+import java.security.GeneralSecurityException;
+import java.util.List;
+import java.util.Locale;
+import java.util.Map;
+import java.util.Objects;
+import java.util.Set;
+import org.apache.hadoop.conf.Configurable;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.iceberg.BaseMetastoreCatalog;
+import org.apache.iceberg.CatalogProperties;
+import org.apache.iceberg.CatalogUtil;
+import org.apache.iceberg.TableMetadata;
+import org.apache.iceberg.TableOperations;
+import org.apache.iceberg.catalog.Namespace;
+import org.apache.iceberg.catalog.SupportsNamespaces;
+import org.apache.iceberg.catalog.TableIdentifier;
+import org.apache.iceberg.exceptions.AlreadyExistsException;
+import org.apache.iceberg.exceptions.NoSuchNamespaceException;
+import org.apache.iceberg.exceptions.NoSuchTableException;
+import org.apache.iceberg.exceptions.ServiceFailureException;
+import org.apache.iceberg.exceptions.ValidationException;
+import org.apache.iceberg.io.FileIO;
+import org.apache.iceberg.relocated.com.google.common.annotations.VisibleForTesting;
+import org.apache.iceberg.relocated.com.google.common.base.Preconditions;
+import org.apache.iceberg.relocated.com.google.common.base.Strings;
+import org.apache.iceberg.relocated.com.google.common.collect.ImmutableList;
+import org.apache.iceberg.relocated.com.google.common.collect.ImmutableMap;
+import org.apache.iceberg.relocated.com.google.common.collect.Maps;
+import org.apache.iceberg.util.LocationUtil;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/** Iceberg Bigquery Metastore Catalog implementation. */
+public final class BigQueryMetastoreCatalog extends BaseMetastoreCatalog

Review Comment:
   ```suggestion
   public class BigQueryCatalog extends BaseMetastoreCatalog
   ```



-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org

For queries about this service, please contact Infrastructure at:
users@infra.apache.org


---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For additional commands, e-mail: issues-help@iceberg.apache.org


Re: [PR] Catalog: Add BigQuery Metastore Catalog Support [iceberg]

Posted by "talatuyarer (via GitHub)" <gi...@apache.org>.
talatuyarer commented on code in PR #12808:
URL: https://github.com/apache/iceberg/pull/12808#discussion_r2059828155


##########
bigquery/src/main/java/org/apache/iceberg/gcp/bigquery/BigQueryTableOperations.java:
##########
@@ -0,0 +1,297 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.iceberg.gcp.bigquery;
+
+import com.google.api.services.bigquery.model.ExternalCatalogTableOptions;
+import com.google.api.services.bigquery.model.Table;
+import com.google.api.services.bigquery.model.TableReference;
+import java.util.Locale;
+import java.util.Map;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hive.common.StatsSetupConst;
+import org.apache.iceberg.BaseMetastoreOperations;
+import org.apache.iceberg.BaseMetastoreTableOperations;
+import org.apache.iceberg.SnapshotSummary;
+import org.apache.iceberg.TableMetadata;
+import org.apache.iceberg.TableProperties;
+import org.apache.iceberg.exceptions.AlreadyExistsException;
+import org.apache.iceberg.exceptions.CommitFailedException;
+import org.apache.iceberg.exceptions.CommitStateUnknownException;
+import org.apache.iceberg.exceptions.NoSuchTableException;
+import org.apache.iceberg.exceptions.ValidationException;
+import org.apache.iceberg.hadoop.ConfigProperties;
+import org.apache.iceberg.io.FileIO;
+import org.apache.iceberg.relocated.com.google.common.collect.Maps;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/** Handles BigQuery metastore table operations. */
+public final class BigQueryTableOperations extends BaseMetastoreTableOperations {
+
+  private static final Logger LOG = LoggerFactory.getLogger(BigQueryTableOperations.class);
+
+  public static final String TABLE_PROPERTIES_BQ_CONNECTION = "bq_connection";
+
+  private final BigQueryMetastoreClient client;
+  private final FileIO fileIO;
+  private final TableReference tableReference;
+  private final Configuration conf;
+
+  BigQueryTableOperations(
+      BigQueryMetastoreClient client,
+      FileIO fileIO,
+      String project,
+      String dataset,
+      String table,
+      Configuration conf) {
+    this.client = client;
+    this.fileIO = fileIO;
+    this.tableReference =

Review Comment:
   Thanks for suggestion. I changed this constructor according your suggestion. 



-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org

For queries about this service, please contact Infrastructure at:
users@infra.apache.org


---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For additional commands, e-mail: issues-help@iceberg.apache.org


Re: [PR] Catalog: Add BigQuery Metastore Catalog Support [iceberg]

Posted by "nastra (via GitHub)" <gi...@apache.org>.
nastra commented on code in PR #12808:
URL: https://github.com/apache/iceberg/pull/12808#discussion_r2059750697


##########
bigquery/src/main/java/org/apache/iceberg/gcp/bigquery/BigQueryMetastoreCatalog.java:
##########
@@ -0,0 +1,402 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.iceberg.gcp.bigquery;
+
+import com.google.api.services.bigquery.model.Dataset;
+import com.google.api.services.bigquery.model.DatasetList.Datasets;
+import com.google.api.services.bigquery.model.DatasetReference;
+import com.google.api.services.bigquery.model.ExternalCatalogDatasetOptions;
+import com.google.api.services.bigquery.model.TableReference;
+import com.google.cloud.ServiceOptions;
+import com.google.cloud.bigquery.BigQueryOptions;
+import java.io.IOException;
+import java.security.GeneralSecurityException;
+import java.util.List;
+import java.util.Locale;
+import java.util.Map;
+import java.util.Objects;
+import java.util.Set;
+import org.apache.hadoop.conf.Configurable;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.iceberg.BaseMetastoreCatalog;
+import org.apache.iceberg.CatalogProperties;
+import org.apache.iceberg.CatalogUtil;
+import org.apache.iceberg.TableMetadata;
+import org.apache.iceberg.TableOperations;
+import org.apache.iceberg.catalog.Namespace;
+import org.apache.iceberg.catalog.SupportsNamespaces;
+import org.apache.iceberg.catalog.TableIdentifier;
+import org.apache.iceberg.exceptions.AlreadyExistsException;
+import org.apache.iceberg.exceptions.NoSuchNamespaceException;
+import org.apache.iceberg.exceptions.NoSuchTableException;
+import org.apache.iceberg.exceptions.ServiceFailureException;
+import org.apache.iceberg.exceptions.ValidationException;
+import org.apache.iceberg.io.FileIO;
+import org.apache.iceberg.relocated.com.google.common.annotations.VisibleForTesting;
+import org.apache.iceberg.relocated.com.google.common.base.Preconditions;
+import org.apache.iceberg.relocated.com.google.common.base.Strings;
+import org.apache.iceberg.relocated.com.google.common.collect.ImmutableList;
+import org.apache.iceberg.relocated.com.google.common.collect.ImmutableMap;
+import org.apache.iceberg.relocated.com.google.common.collect.Maps;
+import org.apache.iceberg.util.LocationUtil;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/** Iceberg Bigquery Metastore Catalog implementation. */
+public final class BigQueryMetastoreCatalog extends BaseMetastoreCatalog
+    implements SupportsNamespaces, Configurable {
+
+  // User provided properties.
+  public static final String PROPERTIES_KEY_GCP_PROJECT = "gcp-project";
+  public static final String PROPERTIES_KEY_GCP_LOCATION = "gcp-location";
+  public static final String PROPERTIES_KEY_FILTER_UNSUPPORTED_TABLES = "filter-unsupported-tables";

Review Comment:
   @amogh-jahagirdar I don't think they should be in `GCPProperties` because these properties here are related to the actual catalog and not the FileIO implementation



-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org

For queries about this service, please contact Infrastructure at:
users@infra.apache.org


---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For additional commands, e-mail: issues-help@iceberg.apache.org


Re: [PR] Catalog: Add BigQuery Metastore Catalog Support [iceberg]

Posted by "nastra (via GitHub)" <gi...@apache.org>.
nastra commented on code in PR #12808:
URL: https://github.com/apache/iceberg/pull/12808#discussion_r2059727438


##########
bigquery/src/main/java/org/apache/iceberg/gcp/bigquery/BigQueryMetastoreClient.java:
##########
@@ -0,0 +1,121 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.iceberg.gcp.bigquery;
+
+import com.google.api.services.bigquery.model.Dataset;
+import com.google.api.services.bigquery.model.DatasetList.Datasets;
+import com.google.api.services.bigquery.model.DatasetReference;
+import com.google.api.services.bigquery.model.Table;
+import com.google.api.services.bigquery.model.TableList.Tables;
+import com.google.api.services.bigquery.model.TableReference;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+
+/**
+ * A client of Google BigQuery Metastore functions over the BigQuery service. Uses the Google
+ * BigQuery API.
+ */
+public interface BigQueryMetastoreClient {
+
+  /**
+   * Creates and returns a new dataset.
+   *
+   * @param dataset the dataset to create
+   */
+  Dataset create(Dataset dataset);
+
+  /**
+   * Returns a dataset.
+   *
+   * @param datasetReference full dataset reference
+   */
+  Dataset load(DatasetReference datasetReference);
+
+  /**
+   * Deletes a dataset.
+   *
+   * @param datasetReference full dataset reference
+   */
+  void delete(DatasetReference datasetReference);
+
+  /**
+   * Updates parameters of a dataset or adds them if did not exist, leaving already-existing ones
+   * intact.
+   *
+   * @param datasetReference full dataset reference
+   * @param parameters metadata parameters to add
+   * @return dataset after patch
+   */
+  Dataset setDatasetParameters(DatasetReference datasetReference, Map<String, String> parameters);

Review Comment:
   I would probably also update the naming here and for the remove and omit the `Dataset` from the method name



-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org

For queries about this service, please contact Infrastructure at:
users@infra.apache.org


---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For additional commands, e-mail: issues-help@iceberg.apache.org


Re: [PR] Catalog: Add BigQuery Metastore Catalog Support [iceberg]

Posted by "talatuyarer (via GitHub)" <gi...@apache.org>.
talatuyarer commented on code in PR #12808:
URL: https://github.com/apache/iceberg/pull/12808#discussion_r2067931567


##########
bigquery/src/test/java/org/apache/iceberg/gcp/bigquery/BigQueryTableOperationsTest.java:
##########
@@ -0,0 +1,274 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.iceberg.gcp.bigquery;
+
+import static org.apache.iceberg.gcp.bigquery.BigQueryMetastoreCatalog.PROJECT_ID;
+import static org.apache.iceberg.types.Types.NestedField.required;
+import static org.assertj.core.api.Assertions.assertThat;
+import static org.assertj.core.api.Assertions.assertThatThrownBy;
+import static org.mockito.Mockito.any;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.reset;
+import static org.mockito.Mockito.times;
+import static org.mockito.Mockito.verify;
+import static org.mockito.Mockito.when;
+
+import com.google.api.services.bigquery.model.Dataset;
+import com.google.api.services.bigquery.model.DatasetReference;
+import com.google.api.services.bigquery.model.ExternalCatalogDatasetOptions;
+import com.google.api.services.bigquery.model.ExternalCatalogTableOptions;
+import com.google.api.services.bigquery.model.StorageDescriptor;
+import com.google.api.services.bigquery.model.Table;
+import com.google.api.services.bigquery.model.TableReference;
+import java.io.File;
+import java.io.IOException;
+import java.util.Collections;
+import java.util.Optional;
+import org.apache.commons.io.FileUtils;
+import org.apache.commons.io.filefilter.TrueFileFilter;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.iceberg.CatalogProperties;
+import org.apache.iceberg.PartitionSpec;
+import org.apache.iceberg.Schema;
+import org.apache.iceberg.catalog.TableIdentifier;
+import org.apache.iceberg.exceptions.CommitFailedException;
+import org.apache.iceberg.exceptions.NoSuchTableException;
+import org.apache.iceberg.exceptions.ValidationException;
+import org.apache.iceberg.relocated.com.google.common.collect.ImmutableMap;
+import org.apache.iceberg.types.Types;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.io.TempDir;
+import org.mockito.ArgumentCaptor;
+
+public class BigQueryTableOperationsTest {
+
+  @TempDir private File tempFolder;
+  public static final String METADATA_LOCATION_PROP = "metadata_location";

Review Comment:
   Done



-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org

For queries about this service, please contact Infrastructure at:
users@infra.apache.org


---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For additional commands, e-mail: issues-help@iceberg.apache.org


Re: [PR] Catalog: Add BigQuery Metastore Catalog Support [iceberg]

Posted by "talatuyarer (via GitHub)" <gi...@apache.org>.
talatuyarer commented on code in PR #12808:
URL: https://github.com/apache/iceberg/pull/12808#discussion_r2067695000


##########
bigquery/src/main/java/org/apache/iceberg/gcp/bigquery/BigQueryMetastoreCatalog.java:
##########
@@ -0,0 +1,405 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.iceberg.gcp.bigquery;
+
+import com.google.api.services.bigquery.model.Dataset;
+import com.google.api.services.bigquery.model.DatasetList.Datasets;
+import com.google.api.services.bigquery.model.DatasetReference;
+import com.google.api.services.bigquery.model.ExternalCatalogDatasetOptions;
+import com.google.api.services.bigquery.model.TableReference;
+import com.google.cloud.ServiceOptions;
+import com.google.cloud.bigquery.BigQueryOptions;
+import java.io.IOException;
+import java.security.GeneralSecurityException;
+import java.util.List;
+import java.util.Locale;
+import java.util.Map;
+import java.util.Objects;
+import java.util.Set;
+import org.apache.hadoop.conf.Configurable;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.iceberg.BaseMetastoreCatalog;
+import org.apache.iceberg.CatalogProperties;
+import org.apache.iceberg.CatalogUtil;
+import org.apache.iceberg.TableMetadata;
+import org.apache.iceberg.TableOperations;
+import org.apache.iceberg.catalog.Namespace;
+import org.apache.iceberg.catalog.SupportsNamespaces;
+import org.apache.iceberg.catalog.TableIdentifier;
+import org.apache.iceberg.exceptions.AlreadyExistsException;
+import org.apache.iceberg.exceptions.NoSuchNamespaceException;
+import org.apache.iceberg.exceptions.NoSuchTableException;
+import org.apache.iceberg.exceptions.ServiceFailureException;
+import org.apache.iceberg.exceptions.ValidationException;
+import org.apache.iceberg.io.FileIO;
+import org.apache.iceberg.relocated.com.google.common.annotations.VisibleForTesting;
+import org.apache.iceberg.relocated.com.google.common.base.Preconditions;
+import org.apache.iceberg.relocated.com.google.common.base.Strings;
+import org.apache.iceberg.relocated.com.google.common.collect.ImmutableList;
+import org.apache.iceberg.relocated.com.google.common.collect.ImmutableMap;
+import org.apache.iceberg.relocated.com.google.common.collect.Maps;
+import org.apache.iceberg.util.LocationUtil;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/** Iceberg Bigquery Metastore Catalog implementation. */
+public class BigQueryMetastoreCatalog extends BaseMetastoreCatalog
+    implements SupportsNamespaces, Configurable {
+
+  // User provided properties.
+  public static final String PROJECT_ID = "gcp-project";
+  public static final String GCP_LOCATION = "gcp-location";
+  public static final String LIST_ALL_TABLES = "list-all-tables";
+
+  public static final String HIVE_METASTORE_WAREHOUSE_DIR = "hive.metastore.warehouse.dir";
+
+  private static final Logger LOG = LoggerFactory.getLogger(BigQueryMetastoreCatalog.class);
+
+  private static final String DEFAULT_GCP_LOCATION = "us";
+
+  private String catalogName;
+  private Map<String, String> catalogProperties;
+  private FileIO fileIO;
+  private Configuration conf;
+  private String projectId;
+  private String location;
+  private BigQueryMetastoreClient client;
+  private boolean listAllTables;
+
+  // Must have a no-arg constructor to be dynamically loaded
+  // initialize(String name, Map<String, String> properties) will be called to complete
+  // initialization
+  public BigQueryMetastoreCatalog() {}
+
+  @Override
+  public void initialize(String inputName, Map<String, String> properties) {
+    Preconditions.checkArgument(
+        properties.containsKey(PROJECT_ID),
+        "Invalid GCP project: %s must be specified",
+        PROJECT_ID);
+
+    projectId = properties.get(PROJECT_ID);
+    location = properties.getOrDefault(GCP_LOCATION, DEFAULT_GCP_LOCATION);
+
+    BigQueryOptions options =
+        BigQueryOptions.newBuilder()
+            .setProjectId(projectId)
+            .setLocation(location)
+            .setRetrySettings(ServiceOptions.getDefaultRetrySettings())
+            .build();
+
+    try {
+      client = new BigQueryMetastoreClientImpl(options);
+    } catch (IOException e) {
+      throw new ServiceFailureException(e, "Creating BigQuery client failed");
+    } catch (GeneralSecurityException e) {
+      throw new ValidationException(e, "Creating BigQuery client failed due to a security issue");
+    }
+
+    initialize(inputName, properties, projectId, location, client);
+  }
+
+  @VisibleForTesting
+  void initialize(
+      String inputName,
+      Map<String, String> properties,
+      String initialProjectId,
+      String initialLocation,
+      BigQueryMetastoreClient bigQueryMetaStoreClient) {
+    Preconditions.checkArgument(bigQueryMetaStoreClient != null, "Invalid BigQuery client: null");
+    this.catalogName = inputName;
+    this.catalogProperties = ImmutableMap.copyOf(properties);
+    this.projectId = initialProjectId;
+    this.location = initialLocation;
+    this.client = bigQueryMetaStoreClient;
+
+    if (this.conf == null) {
+      LOG.warn("No configuration was set, using the default environment Configuration");
+      this.conf = new Configuration();
+    }
+
+    LOG.info("Using BigQuery Metastore Iceberg Catalog: {}", inputName);
+
+    if (properties.containsKey(CatalogProperties.WAREHOUSE_LOCATION)) {
+      this.conf.set(
+          HIVE_METASTORE_WAREHOUSE_DIR,
+          LocationUtil.stripTrailingSlash(properties.get(CatalogProperties.WAREHOUSE_LOCATION)));
+    }
+
+    String fileIoImpl =
+        properties.getOrDefault(
+            CatalogProperties.FILE_IO_IMPL, "org.apache.iceberg.hadoop.HadoopFileIO");
+    this.fileIO = CatalogUtil.loadFileIO(fileIoImpl, properties, conf);
+
+    this.listAllTables = Boolean.parseBoolean(properties.getOrDefault(LIST_ALL_TABLES, "true"));
+  }
+
+  @Override
+  protected TableOperations newTableOps(TableIdentifier identifier) {
+    return new BigQueryTableOperations(client, fileIO, toTableReference(identifier), conf);
+  }
+
+  @Override
+  protected String defaultWarehouseLocation(TableIdentifier identifier) {
+    String locationUri = null;
+    DatasetReference datasetReference = toDatasetReference(identifier.namespace());
+    Dataset dataset = client.load(datasetReference);
+    if (dataset != null && dataset.getExternalCatalogDatasetOptions() != null) {
+      locationUri = dataset.getExternalCatalogDatasetOptions().getDefaultStorageLocationUri();
+    }
+
+    return String.format(
+        "%s/%s",
+        Strings.isNullOrEmpty(locationUri)
+            ? createDefaultStorageLocationUri(datasetReference.getDatasetId())
+            : LocationUtil.stripTrailingSlash(locationUri),
+        identifier.name());
+  }
+
+  @Override
+  public List<TableIdentifier> listTables(Namespace namespace) {
+    validateNamespace(namespace);
+
+    return client.list(toDatasetReference(namespace), listAllTables).stream()
+        .map(
+            table -> TableIdentifier.of(namespace.level(0), table.getTableReference().getTableId()))
+        .collect(ImmutableList.toImmutableList());
+  }
+
+  @Override
+  public boolean dropTable(TableIdentifier identifier, boolean purge) {
+    try {
+      TableOperations ops = newTableOps(identifier);
+      TableMetadata lastMetadata = ops.current();
+
+      client.delete(toTableReference(identifier));
+
+      if (purge && lastMetadata != null) {
+        CatalogUtil.dropTableData(ops.io(), lastMetadata);
+      }
+    } catch (NoSuchTableException e) {
+      return false;
+    }
+
+    return true;
+  }
+
+  @Override
+  public void renameTable(TableIdentifier from, TableIdentifier to) {
+    validateNamespace(from.namespace());
+    validateNamespace(to.namespace());
+
+    if (!namespaceExists(to.namespace())) {
+      throw new NoSuchNamespaceException(
+          "Cannot rename %s to %s. Namespace does not exist: %s",
+          from.name(), to.name(), to.namespace());
+    }
+
+    if (!namespaceExists(from.namespace())) {
+      throw new NoSuchNamespaceException(
+          "Cannot rename %s to %s. Namespace does not exist: %s",
+          from.name(), to.name(), from.namespace());
+    }
+
+    if (!tableExists(from)) {
+      throw new NoSuchTableException(
+          "Cannot rename %s to %s. Table does not exist: %s", from.name(), to.name(), from.name());
+    }
+
+    if (!from.namespace().equals(to.namespace())) {
+      throw new ValidationException(
+          "Cannot rename %s to %s. New table name must be in the same namespace",
+          from.name(), to.name());
+    }
+
+    if (tableExists(to)) {
+      throw new AlreadyExistsException(
+          "Cannot rename %s to %s. Table already exists: %s", from.name(), to.name(), to.name());
+    }
+
+    // TODO: Enable once supported by BigQuery API.
+    throw new UnsupportedOperationException("Table rename operation is unsupported.");
+  }
+
+  @Override
+  public void createNamespace(Namespace namespace, Map<String, String> metadata) {
+    Dataset builder = new Dataset();
+    DatasetReference datasetReference = toDatasetReference(namespace);
+    builder.setLocation(this.location);
+    builder.setDatasetReference(datasetReference);
+    builder.setExternalCatalogDatasetOptions(
+        BigQueryMetastoreUtils.createExternalCatalogDatasetOptions(
+            createDefaultStorageLocationUri(datasetReference.getDatasetId()), metadata));
+
+    client.create(builder);
+  }
+
+  @Override
+  public List<Namespace> listNamespaces() {
+    try {
+      return listNamespaces(Namespace.empty());
+    } catch (NoSuchNamespaceException e) {
+      return ImmutableList.of();
+    }
+  }
+
+  /**
+   * Since this catalog only supports one-level namespaces, it always returns an empty list unless
+   * passed an empty namespace to list all namespaces within the catalog.
+   */
+  @Override
+  public List<Namespace> listNamespaces(Namespace namespace) {
+    List<Datasets> allDatasets = client.list(projectId);
+
+    ImmutableList<Namespace> namespaces =
+        allDatasets.stream()
+            .map(BigQueryMetastoreCatalog::toNamespace)
+            .collect(ImmutableList.toImmutableList());
+    if (namespaces.isEmpty()) {
+      throw new NoSuchNamespaceException("Namespace does not exist: %s", namespace);
+    }
+
+    return namespaces;
+  }
+
+  @Override
+  public boolean dropNamespace(Namespace namespace) {
+    try {
+      client.delete(toDatasetReference(namespace));
+      // We don't delete the data folder for safety, which aligns with Hive Metastore's default
+      // behavior.
+      // We can support database or catalog level config controlling file deletion in the future.
+      return true;
+    } catch (NoSuchNamespaceException e) {
+      return false;
+    }
+  }
+
+  @Override
+  public boolean setProperties(Namespace namespace, Map<String, String> properties) {
+    Dataset dataset = client.load(toDatasetReference(namespace));
+
+    ExternalCatalogDatasetOptions existingOptions = dataset.getExternalCatalogDatasetOptions();
+    Map<String, String> existingParameters =
+        existingOptions != null ? existingOptions.getParameters() : null;
+
+    Map<String, String> newParameters = Maps.newHashMap();
+    if (existingParameters != null) {
+      newParameters.putAll(existingParameters);
+    }
+
+    newParameters.putAll(properties);
+
+    if (Objects.equals(existingParameters, newParameters)) {
+      // No change in parameters detected
+      return false;
+    }
+
+    client.setParameters(toDatasetReference(namespace), properties);
+    return true;
+  }
+
+  @Override
+  public boolean removeProperties(Namespace namespace, Set<String> properties) {
+    client.removeParameters(toDatasetReference(namespace), properties);
+    return true;
+  }
+
+  @SuppressWarnings("FormatStringAnnotation")
+  @Override
+  public Map<String, String> loadNamespaceMetadata(Namespace namespace) {
+    try {
+      return toMetadata(client.load(toDatasetReference(namespace)));
+    } catch (IllegalArgumentException e) {
+      throw new NoSuchNamespaceException(e.getMessage());
+    }
+  }
+
+  @Override
+  public String name() {
+    return catalogName;
+  }
+
+  @Override
+  protected Map<String, String> properties() {
+    return catalogProperties == null ? ImmutableMap.of() : catalogProperties;
+  }
+
+  @Override
+  public void setConf(Configuration conf) {
+    this.conf = new Configuration(conf);
+  }
+
+  @Override
+  public Configuration getConf() {
+    return this.conf;
+  }
+
+  private String createDefaultStorageLocationUri(String dbId) {

Review Comment:
   AFAIK There is no limitation for this. 



-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org

For queries about this service, please contact Infrastructure at:
users@infra.apache.org


---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For additional commands, e-mail: issues-help@iceberg.apache.org


Re: [PR] Catalog: Add BigQuery Metastore Catalog Support [iceberg]

Posted by "danielcweeks (via GitHub)" <gi...@apache.org>.
danielcweeks commented on code in PR #12808:
URL: https://github.com/apache/iceberg/pull/12808#discussion_r2072089160


##########
bigquery/src/main/java/org/apache/iceberg/gcp/bigquery/BigQueryTableOperations.java:
##########
@@ -0,0 +1,296 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.iceberg.gcp.bigquery;
+
+import com.google.api.services.bigquery.model.ExternalCatalogTableOptions;
+import com.google.api.services.bigquery.model.Table;
+import com.google.api.services.bigquery.model.TableReference;
+import java.util.Locale;
+import java.util.Map;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hive.common.StatsSetupConst;
+import org.apache.iceberg.BaseMetastoreOperations;
+import org.apache.iceberg.BaseMetastoreTableOperations;
+import org.apache.iceberg.SnapshotSummary;
+import org.apache.iceberg.TableMetadata;
+import org.apache.iceberg.TableProperties;
+import org.apache.iceberg.exceptions.AlreadyExistsException;
+import org.apache.iceberg.exceptions.CommitFailedException;
+import org.apache.iceberg.exceptions.CommitStateUnknownException;
+import org.apache.iceberg.exceptions.NoSuchTableException;
+import org.apache.iceberg.exceptions.ValidationException;
+import org.apache.iceberg.hadoop.ConfigProperties;
+import org.apache.iceberg.io.FileIO;
+import org.apache.iceberg.relocated.com.google.common.collect.Maps;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/** Handles BigQuery metastore table operations. */
+public final class BigQueryTableOperations extends BaseMetastoreTableOperations {
+
+  private static final Logger LOG = LoggerFactory.getLogger(BigQueryTableOperations.class);
+
+  private static final String TABLE_PROPERTIES_BQ_CONNECTION = "bq_connection";
+
+  private final BigQueryMetastoreClient client;
+  private final FileIO fileIO;
+  private final TableReference tableReference;
+  private final Configuration conf;
+
+  BigQueryTableOperations(
+      BigQueryMetastoreClient client,
+      FileIO fileIO,
+      TableReference tableReference,
+      Configuration conf) {
+    this.client = client;
+    this.fileIO = fileIO;
+    this.tableReference = tableReference;
+    this.conf = conf;
+  }
+
+  // The doRefresh method should provide implementation on how to get the metadata location.
+  @Override
+  public void doRefresh() {
+    // Must default to null.
+    String metadataLocation = null;
+    try {
+      metadataLocation =
+          loadMetadataLocationOrThrow(
+              client.load(this.tableReference).getExternalCatalogTableOptions());
+    } catch (NoSuchTableException e) {
+      if (currentMetadataLocation() != null) {
+        // Re-throws the exception because the table must exist in this case.
+        throw e;
+      }
+    }
+    refreshFromMetadataLocation(metadataLocation);
+  }
+
+  // The doCommit method should provide implementation on how to update with metadata location
+  // atomically
+  @Override
+  public void doCommit(TableMetadata base, TableMetadata metadata) {
+    String newMetadataLocation =
+        base == null && metadata.metadataFileLocation() != null
+            ? metadata.metadataFileLocation()
+            : writeNewMetadata(metadata, currentVersion() + 1);
+    BaseMetastoreOperations.CommitStatus commitStatus =
+        BaseMetastoreOperations.CommitStatus.FAILURE;
+    try {
+      if (base == null) {
+        createTable(newMetadataLocation, metadata);
+      } else {
+        updateTable(base.metadataFileLocation(), newMetadataLocation, metadata);
+      }
+      commitStatus = BaseMetastoreOperations.CommitStatus.SUCCESS;
+    } catch (CommitFailedException | CommitStateUnknownException e) {
+      throw e;
+    } catch (Throwable e) {
+      LOG.error("Exception thrown on commit: ", e);
+      if (e instanceof AlreadyExistsException) {
+        throw e;
+      }
+      commitStatus =
+          BaseMetastoreOperations.CommitStatus.valueOf(
+              checkCommitStatus(newMetadataLocation, metadata).name());
+      if (commitStatus == BaseMetastoreOperations.CommitStatus.FAILURE) {
+        throw new CommitFailedException(e, "Failed to commit");
+      }
+      if (commitStatus == BaseMetastoreOperations.CommitStatus.UNKNOWN) {
+        throw new CommitStateUnknownException(e);
+      }
+    } finally {
+      try {
+        if (commitStatus == BaseMetastoreOperations.CommitStatus.FAILURE) {
+          LOG.warn("Failed to commit updates to table {}", tableName());
+          io().deleteFile(newMetadataLocation);
+        }
+      } catch (RuntimeException e) {
+        LOG.error(
+            "Failed to cleanup metadata file at {} for table {}",
+            newMetadataLocation,
+            tableName(),
+            e);
+      }
+    }
+  }
+
+  @Override
+  public String tableName() {
+    return String.format("%s.%s", tableReference.getDatasetId(), tableReference.getTableId());
+  }
+
+  @Override
+  public FileIO io() {
+    return fileIO;
+  }
+
+  private void createTable(String newMetadataLocation, TableMetadata metadata) {
+    LOG.debug("Creating a new Iceberg table: {}", tableName());
+    Table tableBuilder = makeNewTable(metadata, newMetadataLocation);
+    tableBuilder.setTableReference(this.tableReference);
+    addConnectionIfProvided(tableBuilder, metadata.properties());
+
+    client.create(tableBuilder);
+  }
+
+  private void addConnectionIfProvided(Table tableBuilder, Map<String, String> metadataProperties) {
+    if (metadataProperties.containsKey(TABLE_PROPERTIES_BQ_CONNECTION)) {
+      tableBuilder
+          .getExternalCatalogTableOptions()
+          .setConnectionId(metadataProperties.get(TABLE_PROPERTIES_BQ_CONNECTION));
+    }
+  }
+
+  /** Update table properties with concurrent update detection using etag. */
+  private void updateTable(
+      String oldMetadataLocation, String newMetadataLocation, TableMetadata metadata) {
+    Table table = client.load(this.tableReference);
+    if (table.getEtag().isEmpty()) {
+      throw new ValidationException(
+          "Etag of legacy table %s is empty, manually update the table via the BigQuery API or"
+              + " recreate and retry",
+          tableName());
+    }
+    ExternalCatalogTableOptions options = table.getExternalCatalogTableOptions();
+    addConnectionIfProvided(table, metadata.properties());
+
+    // If `metadataLocationFromMetastore` is different from metadata location of base, it means
+    // someone has updated metadata location in metastore, which is a conflict update.
+    String metadataLocationFromMetastore =
+        options.getParameters().getOrDefault(METADATA_LOCATION_PROP, "");
+    if (!metadataLocationFromMetastore.isEmpty()
+        && !metadataLocationFromMetastore.equals(oldMetadataLocation)) {
+      throw new CommitFailedException(
+          "Cannot commit base metadata location '%s' is not same as the current table metadata location '%s' for"
+              + " %s.%s",
+          oldMetadataLocation,
+          metadataLocationFromMetastore,
+          tableReference.getDatasetId(),
+          tableReference.getTableId());
+    }
+
+    options.setParameters(buildTableParameters(newMetadataLocation, metadata));
+    try {
+      client.update(tableReference, table);
+    } catch (ValidationException e) {
+      if (e.getMessage().toLowerCase(Locale.ENGLISH).contains("etag mismatch")) {
+        throw new CommitFailedException(
+            "Updating table failed due to conflict updates (etag mismatch). Retry the update");
+      }
+
+      throw e;
+    }
+  }
+
+  // To make the table queryable from Hive, the user would likely be setting the HIVE_ENGINE_ENABLED
+  // parameter.
+  //
+  // TODO: We need to make a decision on how to make the table queryable from Hive.
+  // (could be a server side change or a client side change - that's TBD).
+  private Table makeNewTable(TableMetadata metadata, String metadataFileLocation) {
+    return new Table()
+        .setExternalCatalogTableOptions(
+            BigQueryMetastoreUtils.createExternalCatalogTableOptions(
+                metadata.location(),
+                buildTableParameters(metadataFileLocation, metadata),
+                hiveEngineEnabled(metadata)));

Review Comment:
   Do we actually need to reference the `HIVE_ENGINE_ENABLED`?  It looks like this is an unnecessary hold-over from hive.  If we get rid of this, I think you can also get rid of references to the Hadoop Configuration (I think this is the only thing we're referencing from it).



-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org

For queries about this service, please contact Infrastructure at:
users@infra.apache.org


---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For additional commands, e-mail: issues-help@iceberg.apache.org


Re: [PR] Catalog: Add BigQuery Metastore Catalog Support [iceberg]

Posted by "talatuyarer (via GitHub)" <gi...@apache.org>.
talatuyarer commented on code in PR #12808:
URL: https://github.com/apache/iceberg/pull/12808#discussion_r2071955755


##########
bigquery/src/main/java/org/apache/iceberg/gcp/bigquery/BigQueryMetastoreClientImpl.java:
##########
@@ -0,0 +1,581 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.iceberg.gcp.bigquery;
+
+import com.google.api.client.googleapis.javanet.GoogleNetHttpTransport;
+import com.google.api.client.http.HttpHeaders;
+import com.google.api.client.http.HttpResponse;
+import com.google.api.client.http.HttpResponseException;
+import com.google.api.client.http.HttpStatusCodes;
+import com.google.api.client.json.gson.GsonFactory;
+import com.google.api.client.util.Data;
+import com.google.api.services.bigquery.Bigquery;
+import com.google.api.services.bigquery.BigqueryScopes;
+import com.google.api.services.bigquery.model.Dataset;
+import com.google.api.services.bigquery.model.DatasetList;
+import com.google.api.services.bigquery.model.DatasetList.Datasets;
+import com.google.api.services.bigquery.model.DatasetReference;
+import com.google.api.services.bigquery.model.ExternalCatalogDatasetOptions;
+import com.google.api.services.bigquery.model.ExternalCatalogTableOptions;
+import com.google.api.services.bigquery.model.Table;
+import com.google.api.services.bigquery.model.TableList;
+import com.google.api.services.bigquery.model.TableList.Tables;
+import com.google.api.services.bigquery.model.TableReference;
+import com.google.api.services.bigquery.model.TableSchema;
+import com.google.auth.http.HttpCredentialsAdapter;
+import com.google.auth.oauth2.GoogleCredentials;
+import com.google.cloud.BaseServiceException;
+import com.google.cloud.ExceptionHandler;
+import com.google.cloud.bigquery.BigQueryErrorMessages;
+import com.google.cloud.bigquery.BigQueryOptions;
+import com.google.cloud.bigquery.BigQueryRetryConfig;
+import com.google.cloud.bigquery.BigQueryRetryHelper;
+import java.io.IOException;
+import java.nio.charset.StandardCharsets;
+import java.security.GeneralSecurityException;
+import java.util.List;
+import java.util.Locale;
+import java.util.Map;
+import java.util.Set;
+import java.util.stream.Collectors;
+import java.util.stream.Stream;
+import org.apache.iceberg.BaseMetastoreTableOperations;
+import org.apache.iceberg.exceptions.AlreadyExistsException;
+import org.apache.iceberg.exceptions.BadRequestException;
+import org.apache.iceberg.exceptions.ForbiddenException;
+import org.apache.iceberg.exceptions.NoSuchIcebergTableException;
+import org.apache.iceberg.exceptions.NoSuchNamespaceException;
+import org.apache.iceberg.exceptions.NoSuchTableException;
+import org.apache.iceberg.exceptions.NotAuthorizedException;
+import org.apache.iceberg.exceptions.RuntimeIOException;
+import org.apache.iceberg.exceptions.ServiceFailureException;
+import org.apache.iceberg.exceptions.ServiceUnavailableException;
+import org.apache.iceberg.exceptions.ValidationException;
+import org.apache.iceberg.relocated.com.google.common.base.Preconditions;
+import org.apache.iceberg.relocated.com.google.common.collect.Lists;
+import org.apache.iceberg.relocated.com.google.common.collect.Maps;
+
+/** A client of Google Bigquery Metastore functions over the BigQuery service. */
+public final class BigQueryMetastoreClientImpl implements BigQueryMetastoreClient {
+
+  private final Bigquery client;
+  private final BigQueryOptions bigqueryOptions;
+
+  public static final ExceptionHandler.Interceptor EXCEPTION_HANDLER_INTERCEPTOR =
+      new ExceptionHandler.Interceptor() {
+
+        private static final long serialVersionUID = -8429573486870467828L;

Review Comment:
   I put it implements Serializable. Yes there is no serialization for this class. I am going to remove this



-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org

For queries about this service, please contact Infrastructure at:
users@infra.apache.org


---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For additional commands, e-mail: issues-help@iceberg.apache.org


Re: [PR] Catalog: Add BigQuery Metastore Catalog Support [iceberg]

Posted by "danielcweeks (via GitHub)" <gi...@apache.org>.
danielcweeks commented on code in PR #12808:
URL: https://github.com/apache/iceberg/pull/12808#discussion_r2072198227


##########
bigquery/src/main/java/org/apache/iceberg/gcp/bigquery/BigQueryTableOperations.java:
##########
@@ -0,0 +1,296 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.iceberg.gcp.bigquery;
+
+import com.google.api.services.bigquery.model.ExternalCatalogTableOptions;
+import com.google.api.services.bigquery.model.Table;
+import com.google.api.services.bigquery.model.TableReference;
+import java.util.Locale;
+import java.util.Map;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hive.common.StatsSetupConst;
+import org.apache.iceberg.BaseMetastoreOperations;
+import org.apache.iceberg.BaseMetastoreTableOperations;
+import org.apache.iceberg.SnapshotSummary;
+import org.apache.iceberg.TableMetadata;
+import org.apache.iceberg.TableProperties;
+import org.apache.iceberg.exceptions.AlreadyExistsException;
+import org.apache.iceberg.exceptions.CommitFailedException;
+import org.apache.iceberg.exceptions.CommitStateUnknownException;
+import org.apache.iceberg.exceptions.NoSuchTableException;
+import org.apache.iceberg.exceptions.ValidationException;
+import org.apache.iceberg.hadoop.ConfigProperties;
+import org.apache.iceberg.io.FileIO;
+import org.apache.iceberg.relocated.com.google.common.collect.Maps;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/** Handles BigQuery metastore table operations. */
+public final class BigQueryTableOperations extends BaseMetastoreTableOperations {
+
+  private static final Logger LOG = LoggerFactory.getLogger(BigQueryTableOperations.class);
+
+  private static final String TABLE_PROPERTIES_BQ_CONNECTION = "bq_connection";
+
+  private final BigQueryMetastoreClient client;
+  private final FileIO fileIO;
+  private final TableReference tableReference;
+  private final Configuration conf;
+
+  BigQueryTableOperations(
+      BigQueryMetastoreClient client,
+      FileIO fileIO,
+      TableReference tableReference,
+      Configuration conf) {
+    this.client = client;
+    this.fileIO = fileIO;
+    this.tableReference = tableReference;
+    this.conf = conf;
+  }
+
+  // The doRefresh method should provide implementation on how to get the metadata location.
+  @Override
+  public void doRefresh() {
+    // Must default to null.
+    String metadataLocation = null;
+    try {
+      metadataLocation =
+          loadMetadataLocationOrThrow(
+              client.load(this.tableReference).getExternalCatalogTableOptions());
+    } catch (NoSuchTableException e) {
+      if (currentMetadataLocation() != null) {
+        // Re-throws the exception because the table must exist in this case.
+        throw e;
+      }
+    }
+    refreshFromMetadataLocation(metadataLocation);
+  }
+
+  // The doCommit method should provide implementation on how to update with metadata location
+  // atomically
+  @Override
+  public void doCommit(TableMetadata base, TableMetadata metadata) {
+    String newMetadataLocation =
+        base == null && metadata.metadataFileLocation() != null
+            ? metadata.metadataFileLocation()
+            : writeNewMetadata(metadata, currentVersion() + 1);
+    BaseMetastoreOperations.CommitStatus commitStatus =
+        BaseMetastoreOperations.CommitStatus.FAILURE;
+    try {
+      if (base == null) {
+        createTable(newMetadataLocation, metadata);
+      } else {
+        updateTable(base.metadataFileLocation(), newMetadataLocation, metadata);
+      }
+      commitStatus = BaseMetastoreOperations.CommitStatus.SUCCESS;
+    } catch (CommitFailedException | CommitStateUnknownException e) {
+      throw e;
+    } catch (Throwable e) {
+      LOG.error("Exception thrown on commit: ", e);
+      if (e instanceof AlreadyExistsException) {
+        throw e;
+      }
+      commitStatus =
+          BaseMetastoreOperations.CommitStatus.valueOf(
+              checkCommitStatus(newMetadataLocation, metadata).name());
+      if (commitStatus == BaseMetastoreOperations.CommitStatus.FAILURE) {
+        throw new CommitFailedException(e, "Failed to commit");
+      }
+      if (commitStatus == BaseMetastoreOperations.CommitStatus.UNKNOWN) {
+        throw new CommitStateUnknownException(e);
+      }
+    } finally {
+      try {
+        if (commitStatus == BaseMetastoreOperations.CommitStatus.FAILURE) {
+          LOG.warn("Failed to commit updates to table {}", tableName());
+          io().deleteFile(newMetadataLocation);
+        }
+      } catch (RuntimeException e) {
+        LOG.error(
+            "Failed to cleanup metadata file at {} for table {}",
+            newMetadataLocation,
+            tableName(),
+            e);
+      }
+    }
+  }
+
+  @Override
+  public String tableName() {
+    return String.format("%s.%s", tableReference.getDatasetId(), tableReference.getTableId());
+  }
+
+  @Override
+  public FileIO io() {
+    return fileIO;
+  }
+
+  private void createTable(String newMetadataLocation, TableMetadata metadata) {
+    LOG.debug("Creating a new Iceberg table: {}", tableName());
+    Table tableBuilder = makeNewTable(metadata, newMetadataLocation);
+    tableBuilder.setTableReference(this.tableReference);

Review Comment:
   remote `this` reference



##########
bigquery/src/main/java/org/apache/iceberg/gcp/bigquery/BigQueryTableOperations.java:
##########
@@ -0,0 +1,296 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.iceberg.gcp.bigquery;
+
+import com.google.api.services.bigquery.model.ExternalCatalogTableOptions;
+import com.google.api.services.bigquery.model.Table;
+import com.google.api.services.bigquery.model.TableReference;
+import java.util.Locale;
+import java.util.Map;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hive.common.StatsSetupConst;
+import org.apache.iceberg.BaseMetastoreOperations;
+import org.apache.iceberg.BaseMetastoreTableOperations;
+import org.apache.iceberg.SnapshotSummary;
+import org.apache.iceberg.TableMetadata;
+import org.apache.iceberg.TableProperties;
+import org.apache.iceberg.exceptions.AlreadyExistsException;
+import org.apache.iceberg.exceptions.CommitFailedException;
+import org.apache.iceberg.exceptions.CommitStateUnknownException;
+import org.apache.iceberg.exceptions.NoSuchTableException;
+import org.apache.iceberg.exceptions.ValidationException;
+import org.apache.iceberg.hadoop.ConfigProperties;
+import org.apache.iceberg.io.FileIO;
+import org.apache.iceberg.relocated.com.google.common.collect.Maps;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/** Handles BigQuery metastore table operations. */
+public final class BigQueryTableOperations extends BaseMetastoreTableOperations {
+
+  private static final Logger LOG = LoggerFactory.getLogger(BigQueryTableOperations.class);
+
+  private static final String TABLE_PROPERTIES_BQ_CONNECTION = "bq_connection";
+
+  private final BigQueryMetastoreClient client;
+  private final FileIO fileIO;
+  private final TableReference tableReference;
+  private final Configuration conf;
+
+  BigQueryTableOperations(
+      BigQueryMetastoreClient client,
+      FileIO fileIO,
+      TableReference tableReference,
+      Configuration conf) {
+    this.client = client;
+    this.fileIO = fileIO;
+    this.tableReference = tableReference;
+    this.conf = conf;
+  }
+
+  // The doRefresh method should provide implementation on how to get the metadata location.
+  @Override
+  public void doRefresh() {
+    // Must default to null.
+    String metadataLocation = null;
+    try {
+      metadataLocation =
+          loadMetadataLocationOrThrow(
+              client.load(this.tableReference).getExternalCatalogTableOptions());
+    } catch (NoSuchTableException e) {
+      if (currentMetadataLocation() != null) {
+        // Re-throws the exception because the table must exist in this case.
+        throw e;
+      }
+    }
+    refreshFromMetadataLocation(metadataLocation);
+  }
+
+  // The doCommit method should provide implementation on how to update with metadata location
+  // atomically
+  @Override
+  public void doCommit(TableMetadata base, TableMetadata metadata) {
+    String newMetadataLocation =
+        base == null && metadata.metadataFileLocation() != null
+            ? metadata.metadataFileLocation()
+            : writeNewMetadata(metadata, currentVersion() + 1);
+    BaseMetastoreOperations.CommitStatus commitStatus =
+        BaseMetastoreOperations.CommitStatus.FAILURE;
+    try {
+      if (base == null) {
+        createTable(newMetadataLocation, metadata);
+      } else {
+        updateTable(base.metadataFileLocation(), newMetadataLocation, metadata);
+      }
+      commitStatus = BaseMetastoreOperations.CommitStatus.SUCCESS;
+    } catch (CommitFailedException | CommitStateUnknownException e) {
+      throw e;
+    } catch (Throwable e) {
+      LOG.error("Exception thrown on commit: ", e);
+      if (e instanceof AlreadyExistsException) {
+        throw e;
+      }
+      commitStatus =
+          BaseMetastoreOperations.CommitStatus.valueOf(
+              checkCommitStatus(newMetadataLocation, metadata).name());
+      if (commitStatus == BaseMetastoreOperations.CommitStatus.FAILURE) {
+        throw new CommitFailedException(e, "Failed to commit");
+      }
+      if (commitStatus == BaseMetastoreOperations.CommitStatus.UNKNOWN) {
+        throw new CommitStateUnknownException(e);
+      }
+    } finally {
+      try {
+        if (commitStatus == BaseMetastoreOperations.CommitStatus.FAILURE) {
+          LOG.warn("Failed to commit updates to table {}", tableName());
+          io().deleteFile(newMetadataLocation);
+        }
+      } catch (RuntimeException e) {
+        LOG.error(
+            "Failed to cleanup metadata file at {} for table {}",
+            newMetadataLocation,
+            tableName(),
+            e);
+      }
+    }
+  }
+
+  @Override
+  public String tableName() {
+    return String.format("%s.%s", tableReference.getDatasetId(), tableReference.getTableId());
+  }
+
+  @Override
+  public FileIO io() {
+    return fileIO;
+  }
+
+  private void createTable(String newMetadataLocation, TableMetadata metadata) {
+    LOG.debug("Creating a new Iceberg table: {}", tableName());
+    Table tableBuilder = makeNewTable(metadata, newMetadataLocation);
+    tableBuilder.setTableReference(this.tableReference);
+    addConnectionIfProvided(tableBuilder, metadata.properties());
+
+    client.create(tableBuilder);
+  }
+
+  private void addConnectionIfProvided(Table tableBuilder, Map<String, String> metadataProperties) {
+    if (metadataProperties.containsKey(TABLE_PROPERTIES_BQ_CONNECTION)) {
+      tableBuilder
+          .getExternalCatalogTableOptions()
+          .setConnectionId(metadataProperties.get(TABLE_PROPERTIES_BQ_CONNECTION));
+    }
+  }
+
+  /** Update table properties with concurrent update detection using etag. */
+  private void updateTable(
+      String oldMetadataLocation, String newMetadataLocation, TableMetadata metadata) {
+    Table table = client.load(this.tableReference);

Review Comment:
   remote `this` reference



-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org

For queries about this service, please contact Infrastructure at:
users@infra.apache.org


---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For additional commands, e-mail: issues-help@iceberg.apache.org


Re: [PR] Catalog: Add BigQuery Metastore Catalog Support [iceberg]

Posted by "ebyhr (via GitHub)" <gi...@apache.org>.
ebyhr commented on code in PR #12808:
URL: https://github.com/apache/iceberg/pull/12808#discussion_r2074868327


##########
bigquery/src/main/java/org/apache/iceberg/gcp/bigquery/BigQueryMetastoreClientImpl.java:
##########
@@ -0,0 +1,579 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.iceberg.gcp.bigquery;
+
+import com.google.api.client.googleapis.javanet.GoogleNetHttpTransport;
+import com.google.api.client.http.HttpHeaders;
+import com.google.api.client.http.HttpResponse;
+import com.google.api.client.http.HttpResponseException;
+import com.google.api.client.http.HttpStatusCodes;
+import com.google.api.client.json.gson.GsonFactory;
+import com.google.api.client.util.Data;
+import com.google.api.services.bigquery.Bigquery;
+import com.google.api.services.bigquery.BigqueryScopes;
+import com.google.api.services.bigquery.model.Dataset;
+import com.google.api.services.bigquery.model.DatasetList;
+import com.google.api.services.bigquery.model.DatasetList.Datasets;
+import com.google.api.services.bigquery.model.DatasetReference;
+import com.google.api.services.bigquery.model.ExternalCatalogDatasetOptions;
+import com.google.api.services.bigquery.model.ExternalCatalogTableOptions;
+import com.google.api.services.bigquery.model.Table;
+import com.google.api.services.bigquery.model.TableList;
+import com.google.api.services.bigquery.model.TableList.Tables;
+import com.google.api.services.bigquery.model.TableReference;
+import com.google.api.services.bigquery.model.TableSchema;
+import com.google.auth.http.HttpCredentialsAdapter;
+import com.google.auth.oauth2.GoogleCredentials;
+import com.google.cloud.BaseServiceException;
+import com.google.cloud.ExceptionHandler;
+import com.google.cloud.bigquery.BigQueryErrorMessages;
+import com.google.cloud.bigquery.BigQueryOptions;
+import com.google.cloud.bigquery.BigQueryRetryConfig;
+import com.google.cloud.bigquery.BigQueryRetryHelper;
+import java.io.IOException;
+import java.nio.charset.StandardCharsets;
+import java.security.GeneralSecurityException;
+import java.util.List;
+import java.util.Locale;
+import java.util.Map;
+import java.util.Set;
+import java.util.stream.Collectors;
+import java.util.stream.Stream;
+import org.apache.iceberg.BaseMetastoreTableOperations;
+import org.apache.iceberg.exceptions.AlreadyExistsException;
+import org.apache.iceberg.exceptions.BadRequestException;
+import org.apache.iceberg.exceptions.ForbiddenException;
+import org.apache.iceberg.exceptions.NoSuchIcebergTableException;
+import org.apache.iceberg.exceptions.NoSuchNamespaceException;
+import org.apache.iceberg.exceptions.NoSuchTableException;
+import org.apache.iceberg.exceptions.NotAuthorizedException;
+import org.apache.iceberg.exceptions.RuntimeIOException;
+import org.apache.iceberg.exceptions.ServiceFailureException;
+import org.apache.iceberg.exceptions.ServiceUnavailableException;
+import org.apache.iceberg.exceptions.ValidationException;
+import org.apache.iceberg.relocated.com.google.common.base.Preconditions;
+import org.apache.iceberg.relocated.com.google.common.collect.Lists;
+import org.apache.iceberg.relocated.com.google.common.collect.Maps;
+
+/** A client of Google Bigquery Metastore functions over the BigQuery service. */
+public final class BigQueryMetastoreClientImpl implements BigQueryMetastoreClient {
+
+  private final Bigquery client;
+  private final BigQueryOptions bigqueryOptions;
+
+  public static final ExceptionHandler.Interceptor EXCEPTION_HANDLER_INTERCEPTOR =
+      new ExceptionHandler.Interceptor() {
+
+        @Override
+        public RetryResult afterEval(Exception exception, RetryResult retryResult) {
+          return ExceptionHandler.Interceptor.RetryResult.CONTINUE_EVALUATION;
+        }
+
+        @Override
+        public RetryResult beforeEval(Exception exception) {
+          if (exception instanceof BaseServiceException) {
+            boolean retriable = ((BaseServiceException) exception).isRetryable();
+            return retriable
+                ? ExceptionHandler.Interceptor.RetryResult.RETRY
+                : ExceptionHandler.Interceptor.RetryResult.CONTINUE_EVALUATION;
+          }
+
+          return ExceptionHandler.Interceptor.RetryResult.CONTINUE_EVALUATION;
+        }
+      };
+
+  // Retry config with error messages and regex for rate limit exceeded errors.
+  private static final BigQueryRetryConfig DEFAULT_RETRY_CONFIG =
+      BigQueryRetryConfig.newBuilder()
+          .retryOnMessage(BigQueryErrorMessages.RATE_LIMIT_EXCEEDED_MSG)
+          .retryOnMessage(BigQueryErrorMessages.JOB_RATE_LIMIT_EXCEEDED_MSG)
+          .retryOnRegEx(BigQueryErrorMessages.RetryRegExPatterns.RATE_LIMIT_EXCEEDED_REGEX)
+          .build();
+
+  public static final ExceptionHandler BIGQUERY_EXCEPTION_HANDLER =
+      ExceptionHandler.newBuilder()
+          .abortOn(RuntimeException.class)
+          // Retry on connection failures due to transient network issues.
+          .retryOn(java.net.ConnectException.class)
+          // Retry to recover from temporary DNS resolution failures.
+          .retryOn(java.net.UnknownHostException.class)
+          .addInterceptors(EXCEPTION_HANDLER_INTERCEPTOR)
+          .build();
+
+  /** Constructs a client of the Google BigQuery service. */
+  public BigQueryMetastoreClientImpl(BigQueryOptions options)
+      throws IOException, GeneralSecurityException {
+    // Initialize client that will be used to send requests. This client only needs to be created
+    // once, and can be reused for multiple requests
+    HttpCredentialsAdapter httpCredentialsAdapter =
+        new HttpCredentialsAdapter(
+            GoogleCredentials.getApplicationDefault().createScoped(BigqueryScopes.all()));

Review Comment:
   This credential relies on environment variables by default, right? How can we connect to multiple BigQuery projects  from a single machine? How about adding a dedicated property with `GoogleCredentials#fromStream` or something?  



-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org

For queries about this service, please contact Infrastructure at:
users@infra.apache.org


---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For additional commands, e-mail: issues-help@iceberg.apache.org


Re: [PR] Catalog: Add BigQuery Metastore Catalog Support [iceberg]

Posted by "talatuyarer (via GitHub)" <gi...@apache.org>.
talatuyarer commented on code in PR #12808:
URL: https://github.com/apache/iceberg/pull/12808#discussion_r2081049091


##########
bigquery/src/main/java/org/apache/iceberg/gcp/bigquery/BigQueryTableOperations.java:
##########
@@ -0,0 +1,296 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.iceberg.gcp.bigquery;
+
+import com.google.api.services.bigquery.model.ExternalCatalogTableOptions;
+import com.google.api.services.bigquery.model.Table;
+import com.google.api.services.bigquery.model.TableReference;
+import java.util.Locale;
+import java.util.Map;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hive.common.StatsSetupConst;
+import org.apache.iceberg.BaseMetastoreOperations;
+import org.apache.iceberg.BaseMetastoreTableOperations;
+import org.apache.iceberg.SnapshotSummary;
+import org.apache.iceberg.TableMetadata;
+import org.apache.iceberg.TableProperties;
+import org.apache.iceberg.exceptions.AlreadyExistsException;
+import org.apache.iceberg.exceptions.CommitFailedException;
+import org.apache.iceberg.exceptions.CommitStateUnknownException;
+import org.apache.iceberg.exceptions.NoSuchTableException;
+import org.apache.iceberg.exceptions.ValidationException;
+import org.apache.iceberg.hadoop.ConfigProperties;
+import org.apache.iceberg.io.FileIO;
+import org.apache.iceberg.relocated.com.google.common.collect.Maps;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/** Handles BigQuery metastore table operations. */
+public final class BigQueryTableOperations extends BaseMetastoreTableOperations {
+
+  private static final Logger LOG = LoggerFactory.getLogger(BigQueryTableOperations.class);
+
+  private static final String TABLE_PROPERTIES_BQ_CONNECTION = "bq_connection";
+
+  private final BigQueryMetastoreClient client;
+  private final FileIO fileIO;
+  private final TableReference tableReference;
+  private final Configuration conf;
+
+  BigQueryTableOperations(
+      BigQueryMetastoreClient client,
+      FileIO fileIO,
+      TableReference tableReference,
+      Configuration conf) {
+    this.client = client;
+    this.fileIO = fileIO;
+    this.tableReference = tableReference;
+    this.conf = conf;
+  }
+
+  // The doRefresh method should provide implementation on how to get the metadata location.
+  @Override
+  public void doRefresh() {
+    // Must default to null.
+    String metadataLocation = null;
+    try {
+      metadataLocation =
+          loadMetadataLocationOrThrow(
+              client.load(this.tableReference).getExternalCatalogTableOptions());
+    } catch (NoSuchTableException e) {
+      if (currentMetadataLocation() != null) {
+        // Re-throws the exception because the table must exist in this case.
+        throw e;
+      }
+    }
+    refreshFromMetadataLocation(metadataLocation);
+  }
+
+  // The doCommit method should provide implementation on how to update with metadata location
+  // atomically
+  @Override
+  public void doCommit(TableMetadata base, TableMetadata metadata) {
+    String newMetadataLocation =
+        base == null && metadata.metadataFileLocation() != null
+            ? metadata.metadataFileLocation()
+            : writeNewMetadata(metadata, currentVersion() + 1);
+    BaseMetastoreOperations.CommitStatus commitStatus =
+        BaseMetastoreOperations.CommitStatus.FAILURE;
+    try {
+      if (base == null) {
+        createTable(newMetadataLocation, metadata);
+      } else {
+        updateTable(base.metadataFileLocation(), newMetadataLocation, metadata);
+      }
+      commitStatus = BaseMetastoreOperations.CommitStatus.SUCCESS;
+    } catch (CommitFailedException | CommitStateUnknownException e) {
+      throw e;
+    } catch (Throwable e) {
+      LOG.error("Exception thrown on commit: ", e);
+      if (e instanceof AlreadyExistsException) {
+        throw e;
+      }
+      commitStatus =
+          BaseMetastoreOperations.CommitStatus.valueOf(
+              checkCommitStatus(newMetadataLocation, metadata).name());
+      if (commitStatus == BaseMetastoreOperations.CommitStatus.FAILURE) {
+        throw new CommitFailedException(e, "Failed to commit");
+      }
+      if (commitStatus == BaseMetastoreOperations.CommitStatus.UNKNOWN) {
+        throw new CommitStateUnknownException(e);
+      }
+    } finally {
+      try {
+        if (commitStatus == BaseMetastoreOperations.CommitStatus.FAILURE) {
+          LOG.warn("Failed to commit updates to table {}", tableName());
+          io().deleteFile(newMetadataLocation);
+        }
+      } catch (RuntimeException e) {
+        LOG.error(
+            "Failed to cleanup metadata file at {} for table {}",
+            newMetadataLocation,
+            tableName(),
+            e);
+      }
+    }
+  }
+
+  @Override
+  public String tableName() {
+    return String.format("%s.%s", tableReference.getDatasetId(), tableReference.getTableId());
+  }
+
+  @Override
+  public FileIO io() {
+    return fileIO;
+  }
+
+  private void createTable(String newMetadataLocation, TableMetadata metadata) {
+    LOG.debug("Creating a new Iceberg table: {}", tableName());
+    Table tableBuilder = makeNewTable(metadata, newMetadataLocation);
+    tableBuilder.setTableReference(this.tableReference);
+    addConnectionIfProvided(tableBuilder, metadata.properties());
+
+    client.create(tableBuilder);
+  }
+
+  private void addConnectionIfProvided(Table tableBuilder, Map<String, String> metadataProperties) {
+    if (metadataProperties.containsKey(TABLE_PROPERTIES_BQ_CONNECTION)) {
+      tableBuilder
+          .getExternalCatalogTableOptions()
+          .setConnectionId(metadataProperties.get(TABLE_PROPERTIES_BQ_CONNECTION));
+    }
+  }
+
+  /** Update table properties with concurrent update detection using etag. */
+  private void updateTable(
+      String oldMetadataLocation, String newMetadataLocation, TableMetadata metadata) {
+    Table table = client.load(this.tableReference);
+    if (table.getEtag().isEmpty()) {
+      throw new ValidationException(
+          "Etag of legacy table %s is empty, manually update the table via the BigQuery API or"
+              + " recreate and retry",
+          tableName());
+    }
+    ExternalCatalogTableOptions options = table.getExternalCatalogTableOptions();
+    addConnectionIfProvided(table, metadata.properties());
+
+    // If `metadataLocationFromMetastore` is different from metadata location of base, it means
+    // someone has updated metadata location in metastore, which is a conflict update.
+    String metadataLocationFromMetastore =
+        options.getParameters().getOrDefault(METADATA_LOCATION_PROP, "");
+    if (!metadataLocationFromMetastore.isEmpty()
+        && !metadataLocationFromMetastore.equals(oldMetadataLocation)) {
+      throw new CommitFailedException(
+          "Cannot commit base metadata location '%s' is not same as the current table metadata location '%s' for"
+              + " %s.%s",
+          oldMetadataLocation,
+          metadataLocationFromMetastore,
+          tableReference.getDatasetId(),
+          tableReference.getTableId());
+    }
+
+    options.setParameters(buildTableParameters(newMetadataLocation, metadata));
+    try {
+      client.update(tableReference, table);
+    } catch (ValidationException e) {
+      if (e.getMessage().toLowerCase(Locale.ENGLISH).contains("etag mismatch")) {
+        throw new CommitFailedException(
+            "Updating table failed due to conflict updates (etag mismatch). Retry the update");
+      }
+
+      throw e;
+    }
+  }
+
+  // To make the table queryable from Hive, the user would likely be setting the HIVE_ENGINE_ENABLED
+  // parameter.
+  //
+  // TODO: We need to make a decision on how to make the table queryable from Hive.
+  // (could be a server side change or a client side change - that's TBD).
+  private Table makeNewTable(TableMetadata metadata, String metadataFileLocation) {
+    return new Table()
+        .setExternalCatalogTableOptions(
+            BigQueryMetastoreUtils.createExternalCatalogTableOptions(
+                metadata.location(),
+                buildTableParameters(metadataFileLocation, metadata),
+                hiveEngineEnabled(metadata)));

Review Comment:
   Ok removed hadoop and hive dependencies 



-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org

For queries about this service, please contact Infrastructure at:
users@infra.apache.org


---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For additional commands, e-mail: issues-help@iceberg.apache.org


Re: [PR] Catalog: Add BigQuery Metastore Catalog Support [iceberg]

Posted by "nastra (via GitHub)" <gi...@apache.org>.
nastra commented on code in PR #12808:
URL: https://github.com/apache/iceberg/pull/12808#discussion_r2081139987


##########
bigquery/src/main/java/org/apache/iceberg/gcp/bigquery/BigQueryMetastoreCatalog.java:
##########
@@ -0,0 +1,374 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.iceberg.gcp.bigquery;
+
+import com.google.api.services.bigquery.model.Dataset;
+import com.google.api.services.bigquery.model.DatasetList.Datasets;
+import com.google.api.services.bigquery.model.DatasetReference;
+import com.google.api.services.bigquery.model.ExternalCatalogDatasetOptions;
+import com.google.api.services.bigquery.model.TableReference;
+import com.google.cloud.ServiceOptions;
+import com.google.cloud.bigquery.BigQueryOptions;
+import java.io.IOException;
+import java.io.UncheckedIOException;
+import java.security.GeneralSecurityException;
+import java.util.List;
+import java.util.Locale;
+import java.util.Map;
+import java.util.Objects;
+import java.util.Set;
+import org.apache.iceberg.BaseMetastoreCatalog;
+import org.apache.iceberg.CatalogProperties;
+import org.apache.iceberg.CatalogUtil;
+import org.apache.iceberg.TableMetadata;
+import org.apache.iceberg.TableOperations;
+import org.apache.iceberg.catalog.Namespace;
+import org.apache.iceberg.catalog.SupportsNamespaces;
+import org.apache.iceberg.catalog.TableIdentifier;
+import org.apache.iceberg.exceptions.NoSuchNamespaceException;
+import org.apache.iceberg.exceptions.NoSuchTableException;
+import org.apache.iceberg.hadoop.Configurable;
+import org.apache.iceberg.io.FileIO;
+import org.apache.iceberg.relocated.com.google.common.annotations.VisibleForTesting;
+import org.apache.iceberg.relocated.com.google.common.base.Preconditions;
+import org.apache.iceberg.relocated.com.google.common.base.Strings;
+import org.apache.iceberg.relocated.com.google.common.collect.ImmutableList;
+import org.apache.iceberg.relocated.com.google.common.collect.ImmutableMap;
+import org.apache.iceberg.relocated.com.google.common.collect.Maps;
+import org.apache.iceberg.util.LocationUtil;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/** Iceberg Bigquery Metastore Catalog implementation. */
+public class BigQueryMetastoreCatalog extends BaseMetastoreCatalog
+    implements SupportsNamespaces, Configurable<Object> {
+
+  // User provided properties.
+  public static final String PROJECT_ID = "gcp.bigquery.project-id";
+  public static final String GCP_LOCATION = "gcp.bigquery.location";
+  public static final String LIST_ALL_TABLES = "gcp.bigquery.list-all-tables";
+
+  private static final Logger LOG = LoggerFactory.getLogger(BigQueryMetastoreCatalog.class);
+
+  private static final String DEFAULT_GCP_LOCATION = "us";
+
+  private String catalogName;
+  private Map<String, String> catalogProperties;
+  private FileIO fileIO;
+  private Object conf;
+  private String projectId;
+  private String projectLocation;
+  private BigQueryMetastoreClient client;
+  private boolean listAllTables;
+  private String warehouseLocation;
+
+  public BigQueryMetastoreCatalog() {}
+
+  @Override
+  public void initialize(String name, Map<String, String> properties) {
+    Preconditions.checkArgument(
+        properties.containsKey(PROJECT_ID),
+        "Invalid GCP project: %s must be specified",
+        PROJECT_ID);
+
+    this.projectId = properties.get(PROJECT_ID);
+    this.projectLocation = properties.getOrDefault(GCP_LOCATION, DEFAULT_GCP_LOCATION);
+
+    BigQueryOptions options =
+        BigQueryOptions.newBuilder()
+            .setProjectId(projectId)
+            .setLocation(projectLocation)
+            .setRetrySettings(ServiceOptions.getDefaultRetrySettings())
+            .build();
+
+    try {
+      client = new BigQueryMetastoreClientImpl(options);
+    } catch (IOException e) {
+      throw new UncheckedIOException("Creating BigQuery client failed", e);
+    } catch (GeneralSecurityException e) {
+      throw new RuntimeException("Creating BigQuery client failed due to a security issue", e);
+    }
+
+    initialize(name, properties, projectId, projectLocation, client);
+  }
+
+  @VisibleForTesting
+  void initialize(
+      String name,
+      Map<String, String> properties,
+      String initialProjectId,
+      String initialLocation,
+      BigQueryMetastoreClient bigQueryMetaStoreClient) {
+    Preconditions.checkArgument(bigQueryMetaStoreClient != null, "Invalid BigQuery client: null");
+    this.catalogName = name;
+    this.catalogProperties = ImmutableMap.copyOf(properties);
+    this.projectId = initialProjectId;
+    this.projectLocation = initialLocation;
+    this.client = bigQueryMetaStoreClient;
+
+    LOG.info("Using BigQuery Metastore Iceberg Catalog: {}", name);
+
+    if (properties.containsKey(CatalogProperties.WAREHOUSE_LOCATION)) {
+      this.warehouseLocation =
+          LocationUtil.stripTrailingSlash(properties.get(CatalogProperties.WAREHOUSE_LOCATION));
+    }
+
+    this.fileIO =
+        CatalogUtil.loadFileIO(
+            properties.getOrDefault(
+                CatalogProperties.FILE_IO_IMPL, "org.apache.iceberg.io.ResolvingFileIO"),
+            properties,
+            conf);
+
+    this.listAllTables = Boolean.parseBoolean(properties.getOrDefault(LIST_ALL_TABLES, "true"));
+  }
+
+  @Override
+  protected TableOperations newTableOps(TableIdentifier identifier) {
+    return new BigQueryTableOperations(client, fileIO, toTableReference(identifier));
+  }
+
+  @Override
+  protected String defaultWarehouseLocation(TableIdentifier identifier) {
+    String locationUri = null;
+    DatasetReference datasetReference = toDatasetReference(identifier.namespace());
+    Dataset dataset = client.load(datasetReference);
+    if (dataset != null && dataset.getExternalCatalogDatasetOptions() != null) {
+      locationUri = dataset.getExternalCatalogDatasetOptions().getDefaultStorageLocationUri();
+    }
+
+    return String.format(
+        "%s/%s",
+        Strings.isNullOrEmpty(locationUri)
+            ? createDefaultStorageLocationUri(datasetReference.getDatasetId())
+            : LocationUtil.stripTrailingSlash(locationUri),
+        identifier.name());
+  }
+
+  @Override
+  public List<TableIdentifier> listTables(Namespace namespace) {
+    validateNamespace(namespace);
+
+    return client.list(toDatasetReference(namespace), listAllTables).stream()
+        .map(
+            table -> TableIdentifier.of(namespace.level(0), table.getTableReference().getTableId()))
+        .collect(ImmutableList.toImmutableList());
+  }
+
+  @Override
+  public boolean dropTable(TableIdentifier identifier, boolean purge) {
+    try {
+      TableOperations ops = newTableOps(identifier);
+      TableMetadata lastMetadata = ops.current();
+
+      client.delete(toTableReference(identifier));
+
+      if (purge && lastMetadata != null) {
+        CatalogUtil.dropTableData(ops.io(), lastMetadata);
+      }
+    } catch (NoSuchTableException e) {
+      return false;
+    }
+
+    return true;
+  }
+
+  @Override
+  public void renameTable(TableIdentifier from, TableIdentifier to) {
+    // TODO: Enable once supported by BigQuery API.
+    throw new UnsupportedOperationException("Table rename operation is unsupported.");
+  }
+
+  @Override
+  public void createNamespace(Namespace namespace, Map<String, String> metadata) {
+    Dataset builder = new Dataset();
+    DatasetReference datasetReference = toDatasetReference(namespace);
+    builder.setLocation(this.projectLocation);
+    builder.setDatasetReference(datasetReference);
+    builder.setExternalCatalogDatasetOptions(
+        BigQueryMetastoreUtils.createExternalCatalogDatasetOptions(
+            createDefaultStorageLocationUri(datasetReference.getDatasetId()), metadata));
+
+    client.create(builder);
+  }
+
+  @Override
+  public List<Namespace> listNamespaces() {
+    try {
+      return listNamespaces(Namespace.empty());
+    } catch (NoSuchNamespaceException e) {
+      return ImmutableList.of();
+    }
+  }
+
+  /**
+   * Since this catalog only supports one-level namespaces, it always returns an empty list unless
+   * passed an empty namespace to list all namespaces within the catalog.
+   */
+  @Override
+  public List<Namespace> listNamespaces(Namespace namespace) {
+    if (!namespace.isEmpty()) {
+      throw new NoSuchNamespaceException("Namespace does not exist: %s", namespace);
+    }
+    if (!namespace.isEmpty()) {

Review Comment:
   seems like the same condition as in the lines before?



-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org

For queries about this service, please contact Infrastructure at:
users@infra.apache.org


---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For additional commands, e-mail: issues-help@iceberg.apache.org


Re: [PR] Catalog: Add BigQuery Metastore Catalog Support [iceberg]

Posted by "deepaknettem (via GitHub)" <gi...@apache.org>.
deepaknettem commented on code in PR #12808:
URL: https://github.com/apache/iceberg/pull/12808#discussion_r2047473865


##########
bigquery/src/main/java/org/apache/iceberg/gcp/bigquery/BigQueryMetaStoreClientImpl.java:
##########
@@ -0,0 +1,620 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.iceberg.gcp.bigquery;
+
+import com.google.api.client.googleapis.javanet.GoogleNetHttpTransport;
+import com.google.api.client.http.HttpHeaders;
+import com.google.api.client.http.HttpResponse;
+import com.google.api.client.http.HttpResponseException;
+import com.google.api.client.http.HttpStatusCodes;
+import com.google.api.client.json.gson.GsonFactory;
+import com.google.api.client.util.Data;
+import com.google.api.services.bigquery.Bigquery;
+import com.google.api.services.bigquery.BigqueryScopes;
+import com.google.api.services.bigquery.model.Dataset;
+import com.google.api.services.bigquery.model.DatasetList;
+import com.google.api.services.bigquery.model.DatasetList.Datasets;
+import com.google.api.services.bigquery.model.DatasetReference;
+import com.google.api.services.bigquery.model.ExternalCatalogDatasetOptions;
+import com.google.api.services.bigquery.model.ExternalCatalogTableOptions;
+import com.google.api.services.bigquery.model.Table;
+import com.google.api.services.bigquery.model.TableList;
+import com.google.api.services.bigquery.model.TableList.Tables;
+import com.google.api.services.bigquery.model.TableReference;
+import com.google.api.services.bigquery.model.TableSchema;
+import com.google.auth.http.HttpCredentialsAdapter;
+import com.google.auth.oauth2.GoogleCredentials;
+import com.google.cloud.BaseServiceException;
+import com.google.cloud.ExceptionHandler;
+import com.google.cloud.bigquery.BigQueryErrorMessages;
+import com.google.cloud.bigquery.BigQueryOptions;
+import com.google.cloud.bigquery.BigQueryRetryConfig;
+import com.google.cloud.bigquery.BigQueryRetryHelper;
+import java.io.IOException;
+import java.nio.charset.StandardCharsets;
+import java.security.GeneralSecurityException;
+import java.util.List;
+import java.util.Locale;
+import java.util.Map;
+import java.util.Set;
+import java.util.concurrent.Callable;
+import java.util.stream.Collectors;
+import java.util.stream.Stream;
+import org.apache.iceberg.BaseMetastoreTableOperations;
+import org.apache.iceberg.exceptions.AlreadyExistsException;
+import org.apache.iceberg.exceptions.BadRequestException;
+import org.apache.iceberg.exceptions.ForbiddenException;
+import org.apache.iceberg.exceptions.NoSuchIcebergTableException;
+import org.apache.iceberg.exceptions.NoSuchNamespaceException;
+import org.apache.iceberg.exceptions.NoSuchTableException;
+import org.apache.iceberg.exceptions.NotAuthorizedException;
+import org.apache.iceberg.exceptions.RuntimeIOException;
+import org.apache.iceberg.exceptions.ServiceFailureException;
+import org.apache.iceberg.exceptions.ServiceUnavailableException;
+import org.apache.iceberg.exceptions.ValidationException;
+import org.apache.iceberg.relocated.com.google.common.base.Preconditions;
+import org.apache.iceberg.relocated.com.google.common.collect.Lists;
+import org.apache.iceberg.relocated.com.google.common.collect.Maps;
+
+/** A client of Google Bigquery Metastore functions over the BigQuery service. */
+public final class BigQueryMetaStoreClientImpl implements BigQueryMetaStoreClient {
+
+  public static final ExceptionHandler.Interceptor EXCEPTION_HANDLER_INTERCEPTOR =
+      new ExceptionHandler.Interceptor() {
+
+        private static final long serialVersionUID = -8429573486870467828L;
+
+        @Override
+        public RetryResult afterEval(Exception exception, RetryResult retryResult) {
+          return ExceptionHandler.Interceptor.RetryResult.CONTINUE_EVALUATION;
+        }
+
+        @Override
+        public RetryResult beforeEval(Exception exception) {
+          if (exception instanceof BaseServiceException) {
+            boolean retriable = ((BaseServiceException) exception).isRetryable();
+            return retriable
+                ? ExceptionHandler.Interceptor.RetryResult.RETRY
+                : ExceptionHandler.Interceptor.RetryResult.CONTINUE_EVALUATION;
+          }
+          return ExceptionHandler.Interceptor.RetryResult.CONTINUE_EVALUATION;
+        }
+      };
+
+  private final Bigquery client;
+  private final BigQueryOptions bigqueryOptions;
+
+  // Retry config with error messages and regex for rate limit exceeded errors.
+  private static final BigQueryRetryConfig DEFAULT_RETRY_CONFIG =
+      BigQueryRetryConfig.newBuilder()
+          .retryOnMessage(BigQueryErrorMessages.RATE_LIMIT_EXCEEDED_MSG)
+          .retryOnMessage(BigQueryErrorMessages.JOB_RATE_LIMIT_EXCEEDED_MSG)
+          .retryOnRegEx(BigQueryErrorMessages.RetryRegExPatterns.RATE_LIMIT_EXCEEDED_REGEX)
+          .build();
+
+  public static final ExceptionHandler BIGQUERY_EXCEPTION_HANDLER =
+      ExceptionHandler.newBuilder()
+          .abortOn(RuntimeException.class)
+          .retryOn(java.net.ConnectException.class) // retry on Connection Exception
+          .retryOn(java.net.UnknownHostException.class) // retry on UnknownHostException
+          .addInterceptors(EXCEPTION_HANDLER_INTERCEPTOR)
+          .build();
+
+  /** Constructs a client of the Google BigQuery service. */
+  public BigQueryMetaStoreClientImpl(BigQueryOptions options)
+      throws IOException, GeneralSecurityException {
+    // Initialize client that will be used to send requests. This client only needs to be created
+    // once, and can be reused for multiple requests
+    HttpCredentialsAdapter httpCredentialsAdapter =
+        new HttpCredentialsAdapter(
+            GoogleCredentials.getApplicationDefault().createScoped(BigqueryScopes.all()));
+    this.client =
+        new Bigquery.Builder(
+                GoogleNetHttpTransport.newTrustedTransport(),
+                GsonFactory.getDefaultInstance(),
+                httpRequest -> {
+                  httpCredentialsAdapter.initialize(httpRequest);
+                  // Instead of throwing exceptions, analyze the HttpResponse object and inspect its
+                  // status code. This will allow BigQuery API errors to be converted into Iceberg
+                  // exceptions.
+                  httpRequest.setThrowExceptionOnExecuteError(false);
+                })
+            .setApplicationName("BigQuery Iceberg Catalog Plugin")
+            .build();
+    this.bigqueryOptions = options;
+  }
+
+  @Override
+  public Dataset createDataset(Dataset dataset) {
+    // TODO (b/399885863): Ensure Dataset creation is idempotent when handling retries.
+    Dataset response = null;
+    try {
+      response =
+          BigQueryRetryHelper.runWithRetries(
+              new Callable<Dataset>() {
+                @Override
+                public Dataset call() {
+                  return create(dataset);
+                }
+              },
+              bigqueryOptions.getRetrySettings(),
+              BIGQUERY_EXCEPTION_HANDLER,
+              bigqueryOptions.getClock(),
+              DEFAULT_RETRY_CONFIG);
+    } catch (BigQueryRetryHelper.BigQueryRetryHelperException e) {
+      handleBigQueryRetryException(e);
+    }
+    return response;
+  }
+
+  @SuppressWarnings("FormatStringAnnotation")
+  private Dataset create(Dataset dataset) {
+    try {
+      HttpResponse response =
+          client
+              .datasets()
+              .insert(dataset.getDatasetReference().getProjectId(), dataset)
+              .executeUnparsed();
+      return convertExceptionIfUnsuccessful(response).parseAs(Dataset.class);
+    } catch (IOException e) {
+      throw new RuntimeIOException(e);
+    } catch (AlreadyExistsException e) {
+      throw new AlreadyExistsException("Namespace already exists: " + dataset.getId());
+    }
+  }
+
+  @Override
+  @SuppressWarnings("FormatStringAnnotation")
+  public Dataset getDataset(DatasetReference datasetReference) {
+    try {
+      HttpResponse response =
+          client
+              .datasets()
+              .get(datasetReference.getProjectId(), datasetReference.getDatasetId())
+              .executeUnparsed();
+      if (response.getStatusCode() == HttpStatusCodes.STATUS_CODE_NOT_FOUND) {
+        throw new NoSuchNamespaceException(
+            "Namespace does not exist: " + datasetReference.getDatasetId());
+      }
+      return convertExceptionIfUnsuccessful(response).parseAs(Dataset.class);
+    } catch (IOException e) {
+      throw new RuntimeIOException(e);
+    }
+  }
+
+  @Override
+  @SuppressWarnings("FormatStringAnnotation")
+  public void deleteDataset(DatasetReference datasetReference) {
+    try {
+      HttpResponse response =
+          client
+              .datasets()
+              .delete(datasetReference.getProjectId(), datasetReference.getDatasetId())
+              .executeUnparsed();
+      if (response.getStatusCode() == HttpStatusCodes.STATUS_CODE_NOT_FOUND) {
+        throw new NoSuchNamespaceException(
+            "Namespace does not exist: " + datasetReference.getDatasetId());
+      }
+      convertExceptionIfUnsuccessful(response);
+    } catch (IOException e) {
+      throw new RuntimeIOException(e);
+    }
+  }
+
+  @Override
+  public Dataset setDatasetParameters(
+      DatasetReference datasetReference, Map<String, String> parameters) {
+    Dataset dataset = getDataset(datasetReference);
+    ExternalCatalogDatasetOptions externalCatalogDatasetOptions =
+        dataset.getExternalCatalogDatasetOptions() == null
+            ? new ExternalCatalogDatasetOptions()
+            : dataset.getExternalCatalogDatasetOptions();
+    Map<String, String> finalParameters =
+        externalCatalogDatasetOptions.getParameters() == null
+            ? Maps.newHashMap()
+            : externalCatalogDatasetOptions.getParameters();
+    finalParameters.putAll(parameters);
+
+    dataset.setExternalCatalogDatasetOptions(
+        externalCatalogDatasetOptions.setParameters(finalParameters));
+
+    return updateDataset(dataset);
+  }
+
+  @Override
+  public Dataset removeDatasetParameters(
+      DatasetReference datasetReference, Set<String> parameters) {
+    Dataset dataset = getDataset(datasetReference);
+    ExternalCatalogDatasetOptions externalCatalogDatasetOptions =
+        dataset.getExternalCatalogDatasetOptions() == null
+            ? new ExternalCatalogDatasetOptions()
+            : dataset.getExternalCatalogDatasetOptions();
+    Map<String, String> finalParameters =
+        externalCatalogDatasetOptions.getParameters() == null
+            ? Maps.newHashMap()
+            : externalCatalogDatasetOptions.getParameters();
+    parameters.forEach(finalParameters::remove);
+
+    dataset.setExternalCatalogDatasetOptions(
+        externalCatalogDatasetOptions.setParameters(finalParameters));
+
+    return updateDataset(dataset);
+  }
+
+  @Override
+  public List<Datasets> listDatasets(String projectId) {
+    try {
+      String nextPageToken = null;
+      List<Datasets> datasets = Lists.newArrayList();
+      do {
+        HttpResponse pageResponse =
+            client.datasets().list(projectId).setPageToken(nextPageToken).executeUnparsed();
+        DatasetList result =
+            convertExceptionIfUnsuccessful(pageResponse).parseAs(DatasetList.class);
+        nextPageToken = result.getNextPageToken();
+        if (result.getDatasets() != null) {
+          datasets.addAll(result.getDatasets());
+        }
+      } while (nextPageToken != null && !nextPageToken.isEmpty());
+      return datasets;
+    } catch (IOException e) {
+      throw new RuntimeIOException(e);
+    }
+  }
+
+  @Override
+  public Table createTable(Table table) {
+    // Ensure it is an Iceberg table supported by the BQ metastore catalog.

Review Comment:
   nit: s/BQ/BigQuery here and other places



-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org

For queries about this service, please contact Infrastructure at:
users@infra.apache.org


---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For additional commands, e-mail: issues-help@iceberg.apache.org


Re: [PR] Catalog: Add BigQuery Metastore Catalog Support [iceberg]

Posted by "danielcweeks (via GitHub)" <gi...@apache.org>.
danielcweeks merged PR #12808:
URL: https://github.com/apache/iceberg/pull/12808


-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org

For queries about this service, please contact Infrastructure at:
users@infra.apache.org


---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For additional commands, e-mail: issues-help@iceberg.apache.org


Re: [PR] Catalog: Add BigQuery Metastore Catalog Support [iceberg]

Posted by "talatuyarer (via GitHub)" <gi...@apache.org>.
talatuyarer commented on code in PR #12808:
URL: https://github.com/apache/iceberg/pull/12808#discussion_r2051081131


##########
bigquery/src/main/java/org/apache/iceberg/gcp/bigquery/BigQueryMetaStoreClientImpl.java:
##########
@@ -0,0 +1,620 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.iceberg.gcp.bigquery;
+
+import com.google.api.client.googleapis.javanet.GoogleNetHttpTransport;
+import com.google.api.client.http.HttpHeaders;
+import com.google.api.client.http.HttpResponse;
+import com.google.api.client.http.HttpResponseException;
+import com.google.api.client.http.HttpStatusCodes;
+import com.google.api.client.json.gson.GsonFactory;
+import com.google.api.client.util.Data;
+import com.google.api.services.bigquery.Bigquery;
+import com.google.api.services.bigquery.BigqueryScopes;
+import com.google.api.services.bigquery.model.Dataset;
+import com.google.api.services.bigquery.model.DatasetList;
+import com.google.api.services.bigquery.model.DatasetList.Datasets;
+import com.google.api.services.bigquery.model.DatasetReference;
+import com.google.api.services.bigquery.model.ExternalCatalogDatasetOptions;
+import com.google.api.services.bigquery.model.ExternalCatalogTableOptions;
+import com.google.api.services.bigquery.model.Table;
+import com.google.api.services.bigquery.model.TableList;
+import com.google.api.services.bigquery.model.TableList.Tables;
+import com.google.api.services.bigquery.model.TableReference;
+import com.google.api.services.bigquery.model.TableSchema;
+import com.google.auth.http.HttpCredentialsAdapter;
+import com.google.auth.oauth2.GoogleCredentials;
+import com.google.cloud.BaseServiceException;
+import com.google.cloud.ExceptionHandler;
+import com.google.cloud.bigquery.BigQueryErrorMessages;
+import com.google.cloud.bigquery.BigQueryOptions;
+import com.google.cloud.bigquery.BigQueryRetryConfig;
+import com.google.cloud.bigquery.BigQueryRetryHelper;
+import java.io.IOException;
+import java.nio.charset.StandardCharsets;
+import java.security.GeneralSecurityException;
+import java.util.List;
+import java.util.Locale;
+import java.util.Map;
+import java.util.Set;
+import java.util.concurrent.Callable;
+import java.util.stream.Collectors;
+import java.util.stream.Stream;
+import org.apache.iceberg.BaseMetastoreTableOperations;
+import org.apache.iceberg.exceptions.AlreadyExistsException;
+import org.apache.iceberg.exceptions.BadRequestException;
+import org.apache.iceberg.exceptions.ForbiddenException;
+import org.apache.iceberg.exceptions.NoSuchIcebergTableException;
+import org.apache.iceberg.exceptions.NoSuchNamespaceException;
+import org.apache.iceberg.exceptions.NoSuchTableException;
+import org.apache.iceberg.exceptions.NotAuthorizedException;
+import org.apache.iceberg.exceptions.RuntimeIOException;
+import org.apache.iceberg.exceptions.ServiceFailureException;
+import org.apache.iceberg.exceptions.ServiceUnavailableException;
+import org.apache.iceberg.exceptions.ValidationException;
+import org.apache.iceberg.relocated.com.google.common.base.Preconditions;
+import org.apache.iceberg.relocated.com.google.common.collect.Lists;
+import org.apache.iceberg.relocated.com.google.common.collect.Maps;
+
+/** A client of Google Bigquery Metastore functions over the BigQuery service. */
+public final class BigQueryMetaStoreClientImpl implements BigQueryMetaStoreClient {
+
+  public static final ExceptionHandler.Interceptor EXCEPTION_HANDLER_INTERCEPTOR =
+      new ExceptionHandler.Interceptor() {
+
+        private static final long serialVersionUID = -8429573486870467828L;
+
+        @Override
+        public RetryResult afterEval(Exception exception, RetryResult retryResult) {
+          return ExceptionHandler.Interceptor.RetryResult.CONTINUE_EVALUATION;
+        }
+
+        @Override
+        public RetryResult beforeEval(Exception exception) {
+          if (exception instanceof BaseServiceException) {
+            boolean retriable = ((BaseServiceException) exception).isRetryable();
+            return retriable
+                ? ExceptionHandler.Interceptor.RetryResult.RETRY
+                : ExceptionHandler.Interceptor.RetryResult.CONTINUE_EVALUATION;
+          }
+          return ExceptionHandler.Interceptor.RetryResult.CONTINUE_EVALUATION;
+        }
+      };
+
+  private final Bigquery client;
+  private final BigQueryOptions bigqueryOptions;
+
+  // Retry config with error messages and regex for rate limit exceeded errors.
+  private static final BigQueryRetryConfig DEFAULT_RETRY_CONFIG =
+      BigQueryRetryConfig.newBuilder()
+          .retryOnMessage(BigQueryErrorMessages.RATE_LIMIT_EXCEEDED_MSG)
+          .retryOnMessage(BigQueryErrorMessages.JOB_RATE_LIMIT_EXCEEDED_MSG)
+          .retryOnRegEx(BigQueryErrorMessages.RetryRegExPatterns.RATE_LIMIT_EXCEEDED_REGEX)
+          .build();
+
+  public static final ExceptionHandler BIGQUERY_EXCEPTION_HANDLER =
+      ExceptionHandler.newBuilder()
+          .abortOn(RuntimeException.class)
+          .retryOn(java.net.ConnectException.class) // retry on Connection Exception
+          .retryOn(java.net.UnknownHostException.class) // retry on UnknownHostException
+          .addInterceptors(EXCEPTION_HANDLER_INTERCEPTOR)
+          .build();
+
+  /** Constructs a client of the Google BigQuery service. */
+  public BigQueryMetaStoreClientImpl(BigQueryOptions options)
+      throws IOException, GeneralSecurityException {
+    // Initialize client that will be used to send requests. This client only needs to be created
+    // once, and can be reused for multiple requests
+    HttpCredentialsAdapter httpCredentialsAdapter =
+        new HttpCredentialsAdapter(
+            GoogleCredentials.getApplicationDefault().createScoped(BigqueryScopes.all()));
+    this.client =
+        new Bigquery.Builder(
+                GoogleNetHttpTransport.newTrustedTransport(),
+                GsonFactory.getDefaultInstance(),
+                httpRequest -> {
+                  httpCredentialsAdapter.initialize(httpRequest);
+                  // Instead of throwing exceptions, analyze the HttpResponse object and inspect its
+                  // status code. This will allow BigQuery API errors to be converted into Iceberg
+                  // exceptions.
+                  httpRequest.setThrowExceptionOnExecuteError(false);
+                })
+            .setApplicationName("BigQuery Metastore Iceberg Catalog Plugin")
+            .build();
+    this.bigqueryOptions = options;
+  }
+
+  @Override
+  public Dataset createDataset(Dataset dataset) {
+    // TODO (b/399885863): Ensure Dataset creation is idempotent when handling retries.
+    Dataset response = null;
+    try {
+      response =
+          BigQueryRetryHelper.runWithRetries(
+              new Callable<Dataset>() {
+                @Override
+                public Dataset call() {
+                  return create(dataset);
+                }
+              },
+              bigqueryOptions.getRetrySettings(),
+              BIGQUERY_EXCEPTION_HANDLER,
+              bigqueryOptions.getClock(),
+              DEFAULT_RETRY_CONFIG);
+    } catch (BigQueryRetryHelper.BigQueryRetryHelperException e) {
+      handleBigQueryRetryException(e);
+    }
+    return response;
+  }
+
+  @SuppressWarnings("FormatStringAnnotation")
+  private Dataset create(Dataset dataset) {
+    try {
+      HttpResponse response =
+          client
+              .datasets()
+              .insert(dataset.getDatasetReference().getProjectId(), dataset)
+              .executeUnparsed();
+      return convertExceptionIfUnsuccessful(response).parseAs(Dataset.class);
+    } catch (IOException e) {
+      throw new RuntimeIOException(e);
+    } catch (AlreadyExistsException e) {
+      throw new AlreadyExistsException("Namespace already exists: " + dataset.getId());
+    }
+  }
+
+  @Override
+  @SuppressWarnings("FormatStringAnnotation")
+  public Dataset getDataset(DatasetReference datasetReference) {
+    try {
+      HttpResponse response =
+          client
+              .datasets()
+              .get(datasetReference.getProjectId(), datasetReference.getDatasetId())
+              .executeUnparsed();
+      if (response.getStatusCode() == HttpStatusCodes.STATUS_CODE_NOT_FOUND) {
+        throw new NoSuchNamespaceException(
+            "Namespace does not exist: " + datasetReference.getDatasetId());
+      }
+      return convertExceptionIfUnsuccessful(response).parseAs(Dataset.class);
+    } catch (IOException e) {
+      throw new RuntimeIOException(e);
+    }
+  }
+
+  @Override
+  @SuppressWarnings("FormatStringAnnotation")
+  public void deleteDataset(DatasetReference datasetReference) {
+    try {
+      HttpResponse response =
+          client
+              .datasets()
+              .delete(datasetReference.getProjectId(), datasetReference.getDatasetId())
+              .executeUnparsed();
+      if (response.getStatusCode() == HttpStatusCodes.STATUS_CODE_NOT_FOUND) {
+        throw new NoSuchNamespaceException(
+            "Namespace does not exist: " + datasetReference.getDatasetId());
+      }
+      convertExceptionIfUnsuccessful(response);
+    } catch (IOException e) {
+      throw new RuntimeIOException(e);
+    }
+  }
+
+  @Override
+  public Dataset setDatasetParameters(
+      DatasetReference datasetReference, Map<String, String> parameters) {
+    Dataset dataset = getDataset(datasetReference);
+    ExternalCatalogDatasetOptions externalCatalogDatasetOptions =
+        dataset.getExternalCatalogDatasetOptions() == null
+            ? new ExternalCatalogDatasetOptions()
+            : dataset.getExternalCatalogDatasetOptions();
+    Map<String, String> finalParameters =
+        externalCatalogDatasetOptions.getParameters() == null
+            ? Maps.newHashMap()
+            : externalCatalogDatasetOptions.getParameters();
+    finalParameters.putAll(parameters);
+
+    dataset.setExternalCatalogDatasetOptions(
+        externalCatalogDatasetOptions.setParameters(finalParameters));
+
+    return updateDataset(dataset);

Review Comment:
   BigQuery uses optimistic locking aka ETag. If it is changed update will be failed. Do you see any issue for that ? @ebyhr 



-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org

For queries about this service, please contact Infrastructure at:
users@infra.apache.org


---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For additional commands, e-mail: issues-help@iceberg.apache.org


Re: [PR] Catalog: Add BigQuery Metastore Catalog Support [iceberg]

Posted by "talatuyarer (via GitHub)" <gi...@apache.org>.
talatuyarer commented on code in PR #12808:
URL: https://github.com/apache/iceberg/pull/12808#discussion_r2051079641


##########
bigquery/src/main/java/org/apache/iceberg/gcp/bigquery/BigQueryMetaStoreClient.java:
##########
@@ -0,0 +1,129 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.iceberg.gcp.bigquery;
+
+import com.google.api.services.bigquery.model.Dataset;
+import com.google.api.services.bigquery.model.DatasetList.Datasets;
+import com.google.api.services.bigquery.model.DatasetReference;
+import com.google.api.services.bigquery.model.Table;
+import com.google.api.services.bigquery.model.TableList.Tables;
+import com.google.api.services.bigquery.model.TableReference;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+
+/**
+ * A client of Google BigQuery Metastore functions over the BigQuery service. Uses the Google
+ * BigQuery API.
+ */
+public interface BigQueryMetaStoreClient {
+
+  /**
+   * Creates and returns a new dataset.
+   *
+   * @param dataset the dataset to create
+   */
+  Dataset createDataset(Dataset dataset);
+
+  /**
+   * Returns a dataset.
+   *
+   * @param datasetReference full dataset reference
+   */
+  Dataset getDataset(DatasetReference datasetReference);
+
+  /**
+   * Deletes a dataset.
+   *
+   * @param datasetReference full dataset reference
+   */
+  void deleteDataset(DatasetReference datasetReference);
+
+  /**
+   * Updates parameters of a dataset or adds them if did not exist, leaving already-existing ones
+   * intact.
+   *
+   * @param datasetReference full dataset reference
+   * @param parameters metadata parameters to add
+   * @return dataset after patch
+   */
+  Dataset setDatasetParameters(DatasetReference datasetReference, Map<String, String> parameters);
+
+  /**
+   * Removes given set of keys of parameters of a dataset. Ignores keys that do not exist already.
+   *
+   * @param datasetReference full dataset reference
+   * @param parameters metadata parameter keys to remove
+   * @return dataset after patch
+   */
+  Dataset removeDatasetParameters(DatasetReference datasetReference, Set<String> parameters);
+
+  /**
+   * Lists datasets under a given project
+   *
+   * @param projectId the identifier of the project to list datasets under
+   */
+  List<Datasets> listDatasets(String projectId);
+
+  /**
+   * Creates and returns a new table.
+   *
+   * @param table body of the table to create
+   */
+  Table createTable(Table table);
+
+  /**
+   * Returns a table.
+   *
+   * @param tableReference full table reference
+   */
+  Table getTable(TableReference tableReference);
+
+  /**
+   * Updates the catalog table options of an Iceberg table and returns the updated table.
+   *
+   * @param tableReference full table reference
+   * @param table to patch
+   */
+  Table patchTable(TableReference tableReference, Table table);
+
+  /**
+   * Renames a table.
+   *
+   * @param tableToRename full table reference
+   * @param newTableId new table identifier
+   */
+  Table renameTable(TableReference tableToRename, String newTableId);

Review Comment:
   Yeah. I thought we can have this feature but looks like not. I removed it.



-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org

For queries about this service, please contact Infrastructure at:
users@infra.apache.org


---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For additional commands, e-mail: issues-help@iceberg.apache.org


Re: [PR] Catalog: Add BigQuery Metastore Catalog Support [iceberg]

Posted by "danielcweeks (via GitHub)" <gi...@apache.org>.
danielcweeks commented on code in PR #12808:
URL: https://github.com/apache/iceberg/pull/12808#discussion_r2072166426


##########
bigquery/src/main/java/org/apache/iceberg/gcp/bigquery/BigQueryMetastoreUtils.java:
##########
@@ -0,0 +1,95 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.iceberg.gcp.bigquery;
+
+import com.google.api.services.bigquery.model.ExternalCatalogDatasetOptions;
+import com.google.api.services.bigquery.model.ExternalCatalogTableOptions;
+import com.google.api.services.bigquery.model.SerDeInfo;
+import com.google.api.services.bigquery.model.StorageDescriptor;
+import java.util.Map;
+
+/** Shared utilities for BigQuery Metastore specific functions and constants. */
+public final class BigQueryMetastoreUtils {
+
+  private BigQueryMetastoreUtils() {}
+
+  private static final String HIVE_SERIALIZATION_LIBRARY =
+      "org.apache.iceberg.mr.hive.HiveIcebergSerDe";
+  private static final String HIVE_FILE_INPUT_FORMAT =
+      "org.apache.iceberg.mr.hive.HiveIcebergInputFormat";
+  private static final String HIVE_FILE_OUTPUT_FORMAT =
+      "org.apache.iceberg.mr.hive.HiveIcebergOutputFormat";
+
+  private static final String SERIALIZATION_LIBRARY =
+      "org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe";
+  private static final String FILE_INPUT_FORMAT = "org.apache.hadoop.mapred.FileInputFormat";
+  private static final String FILE_OUTPUT_FORMAT = "org.apache.hadoop.mapred.FileOutputFormat";
+
+  /**
+   * Creates a new ExternalCatalogTableOptions object populated with the supported library constants
+   * and parameters given.
+   *
+   * @param locationUri storage location uri
+   * @param parameters table metadata parameters
+   */
+  public static ExternalCatalogTableOptions createExternalCatalogTableOptions(
+      String locationUri, Map<String, String> parameters, boolean hiveEngineEnabled) {
+    return new ExternalCatalogTableOptions()
+        .setStorageDescriptor(createStorageDescriptor(locationUri, hiveEngineEnabled))
+        .setParameters(parameters);
+  }
+
+  /**
+   * Creates a new ExternalCatalogDatasetOptions object populated with the supported library
+   * constants and parameters given.
+   *
+   * @param defaultStorageLocationUri dataset's default location uri
+   * @param metadataParameters metadata parameters for the dataset
+   */
+  public static ExternalCatalogDatasetOptions createExternalCatalogDatasetOptions(
+      String defaultStorageLocationUri, Map<String, String> metadataParameters) {
+    return new ExternalCatalogDatasetOptions()
+        .setDefaultStorageLocationUri(defaultStorageLocationUri)
+        .setParameters(metadataParameters);
+  }
+
+  private static StorageDescriptor createStorageDescriptor(
+      String locationUri, boolean hiveEngineEnabled) {
+    String inputFormat;
+    String outputFormat;
+    String serializationLibrary;
+    if (hiveEngineEnabled) {
+      serializationLibrary = HIVE_SERIALIZATION_LIBRARY;
+      inputFormat = HIVE_FILE_INPUT_FORMAT;
+      outputFormat = HIVE_FILE_OUTPUT_FORMAT;
+    } else {
+      serializationLibrary = SERIALIZATION_LIBRARY;

Review Comment:
   Why is this even an option?  We can't support anything other Iceberg so why are we supporting a way to configure something that cannot be used?



-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org

For queries about this service, please contact Infrastructure at:
users@infra.apache.org


---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For additional commands, e-mail: issues-help@iceberg.apache.org


Re: [PR] Catalog: Add BigQuery Metastore Catalog Support [iceberg]

Posted by "danielcweeks (via GitHub)" <gi...@apache.org>.
danielcweeks commented on code in PR #12808:
URL: https://github.com/apache/iceberg/pull/12808#discussion_r2071895166


##########
bigquery/src/main/java/org/apache/iceberg/gcp/bigquery/BigQueryMetastoreClient.java:
##########
@@ -0,0 +1,121 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.iceberg.gcp.bigquery;
+
+import com.google.api.services.bigquery.model.Dataset;
+import com.google.api.services.bigquery.model.DatasetList.Datasets;
+import com.google.api.services.bigquery.model.DatasetReference;
+import com.google.api.services.bigquery.model.Table;
+import com.google.api.services.bigquery.model.TableList.Tables;
+import com.google.api.services.bigquery.model.TableReference;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+
+/**
+ * A client of Google BigQuery Metastore functions over the BigQuery service. Uses the Google
+ * BigQuery API.
+ */
+public interface BigQueryMetastoreClient {

Review Comment:
   I don't think this need to be public



-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org

For queries about this service, please contact Infrastructure at:
users@infra.apache.org


---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For additional commands, e-mail: issues-help@iceberg.apache.org


Re: [PR] Catalog: Add BigQuery Metastore Catalog Support [iceberg]

Posted by "danielcweeks (via GitHub)" <gi...@apache.org>.
danielcweeks commented on code in PR #12808:
URL: https://github.com/apache/iceberg/pull/12808#discussion_r2072148592


##########
bigquery/src/main/java/org/apache/iceberg/gcp/bigquery/BigQueryMetastoreCatalog.java:
##########
@@ -0,0 +1,366 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.iceberg.gcp.bigquery;
+
+import com.google.api.services.bigquery.model.Dataset;
+import com.google.api.services.bigquery.model.DatasetList.Datasets;
+import com.google.api.services.bigquery.model.DatasetReference;
+import com.google.api.services.bigquery.model.ExternalCatalogDatasetOptions;
+import com.google.api.services.bigquery.model.TableReference;
+import com.google.cloud.ServiceOptions;
+import com.google.cloud.bigquery.BigQueryOptions;
+import java.io.IOException;
+import java.io.UncheckedIOException;
+import java.security.GeneralSecurityException;
+import java.util.List;
+import java.util.Locale;
+import java.util.Map;
+import java.util.Objects;
+import java.util.Set;
+import org.apache.hadoop.conf.Configurable;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.iceberg.BaseMetastoreCatalog;
+import org.apache.iceberg.CatalogProperties;
+import org.apache.iceberg.CatalogUtil;
+import org.apache.iceberg.TableMetadata;
+import org.apache.iceberg.TableOperations;
+import org.apache.iceberg.catalog.Namespace;
+import org.apache.iceberg.catalog.SupportsNamespaces;
+import org.apache.iceberg.catalog.TableIdentifier;
+import org.apache.iceberg.exceptions.NoSuchNamespaceException;
+import org.apache.iceberg.exceptions.NoSuchTableException;
+import org.apache.iceberg.io.FileIO;
+import org.apache.iceberg.relocated.com.google.common.annotations.VisibleForTesting;
+import org.apache.iceberg.relocated.com.google.common.base.Preconditions;
+import org.apache.iceberg.relocated.com.google.common.base.Strings;
+import org.apache.iceberg.relocated.com.google.common.collect.ImmutableList;
+import org.apache.iceberg.relocated.com.google.common.collect.ImmutableMap;
+import org.apache.iceberg.relocated.com.google.common.collect.Maps;
+import org.apache.iceberg.util.LocationUtil;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/** Iceberg Bigquery Metastore Catalog implementation. */
+public class BigQueryMetastoreCatalog extends BaseMetastoreCatalog
+    implements SupportsNamespaces, Configurable {
+
+  // User provided properties.
+  public static final String PROJECT_ID = "gcp.bigquery.project-id";
+  public static final String GCP_LOCATION = "gcp.bigquery.location";
+  public static final String LIST_ALL_TABLES = "gcp.bigquery.list-all-tables";
+
+  private static final Logger LOG = LoggerFactory.getLogger(BigQueryMetastoreCatalog.class);
+
+  private static final String DEFAULT_GCP_LOCATION = "us";
+
+  private String catalogName;
+  private Map<String, String> catalogProperties;
+  private FileIO fileIO;
+  private Configuration conf;
+  private String projectId;
+  private String projectLocation;
+  private BigQueryMetastoreClient client;
+  private boolean listAllTables;
+  private String warehouseLocation;
+
+  public BigQueryMetastoreCatalog() {}
+
+  @Override
+  public void initialize(String name, Map<String, String> properties) {
+    Preconditions.checkArgument(
+        properties.containsKey(PROJECT_ID),
+        "Invalid GCP project: %s must be specified",
+        PROJECT_ID);
+
+    this.projectId = properties.get(PROJECT_ID);
+    this.projectLocation = properties.getOrDefault(GCP_LOCATION, DEFAULT_GCP_LOCATION);
+
+    BigQueryOptions options =
+        BigQueryOptions.newBuilder()
+            .setProjectId(projectId)
+            .setLocation(projectLocation)
+            .setRetrySettings(ServiceOptions.getDefaultRetrySettings())
+            .build();
+
+    try {
+      client = new BigQueryMetastoreClientImpl(options);
+    } catch (IOException e) {
+      throw new UncheckedIOException("Creating BigQuery client failed", e);
+    } catch (GeneralSecurityException e) {
+      throw new RuntimeException("Creating BigQuery client failed due to a security issue", e);
+    }
+
+    initialize(name, properties, projectId, projectLocation, client);
+  }
+
+  @VisibleForTesting
+  void initialize(
+      String name,
+      Map<String, String> properties,
+      String initialProjectId,
+      String initialLocation,
+      BigQueryMetastoreClient bigQueryMetaStoreClient) {
+    Preconditions.checkArgument(bigQueryMetaStoreClient != null, "Invalid BigQuery client: null");
+    this.catalogName = name;
+    this.catalogProperties = ImmutableMap.copyOf(properties);
+    this.projectId = initialProjectId;
+    this.projectLocation = initialLocation;
+    this.client = bigQueryMetaStoreClient;
+
+    if (this.conf == null) {
+      LOG.warn("No configuration was set, using the default environment Configuration");
+      this.conf = new Configuration();
+    }
+
+    LOG.info("Using BigQuery Metastore Iceberg Catalog: {}", name);
+
+    if (properties.containsKey(CatalogProperties.WAREHOUSE_LOCATION)) {
+      this.warehouseLocation =
+          LocationUtil.stripTrailingSlash(properties.get(CatalogProperties.WAREHOUSE_LOCATION));
+    }
+
+    this.fileIO =
+        CatalogUtil.loadFileIO(
+            properties.getOrDefault(
+                CatalogProperties.FILE_IO_IMPL, "org.apache.iceberg.io.ResolvingFileIO"),
+            properties,
+            conf);
+
+    this.listAllTables = Boolean.parseBoolean(properties.getOrDefault(LIST_ALL_TABLES, "true"));
+  }
+
+  @Override
+  protected TableOperations newTableOps(TableIdentifier identifier) {
+    return new BigQueryTableOperations(client, fileIO, toTableReference(identifier), conf);
+  }
+
+  @Override
+  protected String defaultWarehouseLocation(TableIdentifier identifier) {
+    String locationUri = null;
+    DatasetReference datasetReference = toDatasetReference(identifier.namespace());
+    Dataset dataset = client.load(datasetReference);
+    if (dataset != null && dataset.getExternalCatalogDatasetOptions() != null) {
+      locationUri = dataset.getExternalCatalogDatasetOptions().getDefaultStorageLocationUri();
+    }
+
+    return String.format(
+        "%s/%s",
+        Strings.isNullOrEmpty(locationUri)
+            ? createDefaultStorageLocationUri(datasetReference.getDatasetId())
+            : LocationUtil.stripTrailingSlash(locationUri),
+        identifier.name());
+  }
+
+  @Override
+  public List<TableIdentifier> listTables(Namespace namespace) {
+    validateNamespace(namespace);
+
+    return client.list(toDatasetReference(namespace), listAllTables).stream()
+        .map(
+            table -> TableIdentifier.of(namespace.level(0), table.getTableReference().getTableId()))
+        .collect(ImmutableList.toImmutableList());
+  }
+
+  @Override
+  public boolean dropTable(TableIdentifier identifier, boolean purge) {
+    try {
+      TableOperations ops = newTableOps(identifier);
+      TableMetadata lastMetadata = ops.current();
+
+      client.delete(toTableReference(identifier));
+
+      if (purge && lastMetadata != null) {
+        CatalogUtil.dropTableData(ops.io(), lastMetadata);
+      }
+    } catch (NoSuchTableException e) {
+      return false;
+    }
+
+    return true;
+  }
+
+  @Override
+  public void renameTable(TableIdentifier from, TableIdentifier to) {
+    // TODO: Enable once supported by BigQuery API.
+    throw new UnsupportedOperationException("Table rename operation is unsupported.");
+  }
+
+  @Override
+  public void createNamespace(Namespace namespace, Map<String, String> metadata) {
+    Dataset builder = new Dataset();
+    DatasetReference datasetReference = toDatasetReference(namespace);
+    builder.setLocation(this.projectLocation);
+    builder.setDatasetReference(datasetReference);
+    builder.setExternalCatalogDatasetOptions(
+        BigQueryMetastoreUtils.createExternalCatalogDatasetOptions(
+            createDefaultStorageLocationUri(datasetReference.getDatasetId()), metadata));
+
+    client.create(builder);
+  }
+
+  @Override
+  public List<Namespace> listNamespaces() {
+    try {
+      return listNamespaces(Namespace.empty());
+    } catch (NoSuchNamespaceException e) {
+      return ImmutableList.of();
+    }
+  }
+
+  /**
+   * Since this catalog only supports one-level namespaces, it always returns an empty list unless
+   * passed an empty namespace to list all namespaces within the catalog.
+   */
+  @Override
+  public List<Namespace> listNamespaces(Namespace namespace) {
+    List<Datasets> allDatasets = client.list(projectId);
+
+    ImmutableList<Namespace> namespaces =
+        allDatasets.stream().map(this::toNamespace).collect(ImmutableList.toImmutableList());
+    if (namespaces.isEmpty()) {
+      throw new NoSuchNamespaceException("Namespace does not exist: %s", namespace);
+    }
+
+    return namespaces;
+  }
+
+  @Override
+  public boolean dropNamespace(Namespace namespace) {
+    try {
+      client.delete(toDatasetReference(namespace));
+      // We don't delete the data folder for safety, which aligns with Hive Metastore's default
+      // behavior.
+      // We can support database or catalog level config controlling file deletion in the future.
+      return true;
+    } catch (NoSuchNamespaceException e) {
+      return false;
+    }
+  }
+
+  @Override
+  public boolean setProperties(Namespace namespace, Map<String, String> properties) {
+    Dataset dataset = client.load(toDatasetReference(namespace));
+
+    ExternalCatalogDatasetOptions existingOptions = dataset.getExternalCatalogDatasetOptions();
+    Map<String, String> existingParameters =
+        existingOptions != null ? existingOptions.getParameters() : null;
+
+    Map<String, String> newParameters = Maps.newHashMap();
+    if (existingParameters != null) {
+      newParameters.putAll(existingParameters);
+    }
+
+    newParameters.putAll(properties);
+
+    if (Objects.equals(existingParameters, newParameters)) {
+      // No change in parameters detected
+      return false;
+    }
+
+    client.setParameters(toDatasetReference(namespace), properties);
+    return true;
+  }
+
+  @Override
+  public boolean removeProperties(Namespace namespace, Set<String> properties) {
+    client.removeParameters(toDatasetReference(namespace), properties);
+    return true;
+  }
+
+  @Override
+  public Map<String, String> loadNamespaceMetadata(Namespace namespace) {
+    try {
+      return toMetadata(client.load(toDatasetReference(namespace)));
+    } catch (IllegalArgumentException e) {
+      throw new NoSuchNamespaceException("%s", e.getMessage());
+    }
+  }
+
+  @Override
+  public String name() {
+    return catalogName;
+  }
+
+  @Override
+  protected Map<String, String> properties() {
+    return catalogProperties == null ? ImmutableMap.of() : catalogProperties;
+  }
+
+  @Override
+  public void setConf(Configuration conf) {
+    this.conf = new Configuration(conf);
+  }
+
+  @Override
+  public Configuration getConf() {
+    return this.conf;
+  }
+
+  private String createDefaultStorageLocationUri(String dbId) {
+    Preconditions.checkArgument(
+        this.warehouseLocation != null,

Review Comment:
   nit: only use `this` with assignment



-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org

For queries about this service, please contact Infrastructure at:
users@infra.apache.org


---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For additional commands, e-mail: issues-help@iceberg.apache.org


Re: [PR] Catalog: Add BigQuery Metastore Catalog Support [iceberg]

Posted by "danielcweeks (via GitHub)" <gi...@apache.org>.
danielcweeks commented on code in PR #12808:
URL: https://github.com/apache/iceberg/pull/12808#discussion_r2072142459


##########
bigquery/src/main/java/org/apache/iceberg/gcp/bigquery/BigQueryMetastoreCatalog.java:
##########
@@ -0,0 +1,366 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.iceberg.gcp.bigquery;
+
+import com.google.api.services.bigquery.model.Dataset;
+import com.google.api.services.bigquery.model.DatasetList.Datasets;
+import com.google.api.services.bigquery.model.DatasetReference;
+import com.google.api.services.bigquery.model.ExternalCatalogDatasetOptions;
+import com.google.api.services.bigquery.model.TableReference;
+import com.google.cloud.ServiceOptions;
+import com.google.cloud.bigquery.BigQueryOptions;
+import java.io.IOException;
+import java.io.UncheckedIOException;
+import java.security.GeneralSecurityException;
+import java.util.List;
+import java.util.Locale;
+import java.util.Map;
+import java.util.Objects;
+import java.util.Set;
+import org.apache.hadoop.conf.Configurable;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.iceberg.BaseMetastoreCatalog;
+import org.apache.iceberg.CatalogProperties;
+import org.apache.iceberg.CatalogUtil;
+import org.apache.iceberg.TableMetadata;
+import org.apache.iceberg.TableOperations;
+import org.apache.iceberg.catalog.Namespace;
+import org.apache.iceberg.catalog.SupportsNamespaces;
+import org.apache.iceberg.catalog.TableIdentifier;
+import org.apache.iceberg.exceptions.NoSuchNamespaceException;
+import org.apache.iceberg.exceptions.NoSuchTableException;
+import org.apache.iceberg.io.FileIO;
+import org.apache.iceberg.relocated.com.google.common.annotations.VisibleForTesting;
+import org.apache.iceberg.relocated.com.google.common.base.Preconditions;
+import org.apache.iceberg.relocated.com.google.common.base.Strings;
+import org.apache.iceberg.relocated.com.google.common.collect.ImmutableList;
+import org.apache.iceberg.relocated.com.google.common.collect.ImmutableMap;
+import org.apache.iceberg.relocated.com.google.common.collect.Maps;
+import org.apache.iceberg.util.LocationUtil;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/** Iceberg Bigquery Metastore Catalog implementation. */
+public class BigQueryMetastoreCatalog extends BaseMetastoreCatalog
+    implements SupportsNamespaces, Configurable {
+
+  // User provided properties.
+  public static final String PROJECT_ID = "gcp.bigquery.project-id";
+  public static final String GCP_LOCATION = "gcp.bigquery.location";
+  public static final String LIST_ALL_TABLES = "gcp.bigquery.list-all-tables";
+
+  private static final Logger LOG = LoggerFactory.getLogger(BigQueryMetastoreCatalog.class);
+
+  private static final String DEFAULT_GCP_LOCATION = "us";
+
+  private String catalogName;
+  private Map<String, String> catalogProperties;
+  private FileIO fileIO;
+  private Configuration conf;
+  private String projectId;
+  private String projectLocation;
+  private BigQueryMetastoreClient client;
+  private boolean listAllTables;
+  private String warehouseLocation;
+
+  public BigQueryMetastoreCatalog() {}
+
+  @Override
+  public void initialize(String name, Map<String, String> properties) {
+    Preconditions.checkArgument(
+        properties.containsKey(PROJECT_ID),
+        "Invalid GCP project: %s must be specified",
+        PROJECT_ID);
+
+    this.projectId = properties.get(PROJECT_ID);
+    this.projectLocation = properties.getOrDefault(GCP_LOCATION, DEFAULT_GCP_LOCATION);
+
+    BigQueryOptions options =
+        BigQueryOptions.newBuilder()
+            .setProjectId(projectId)
+            .setLocation(projectLocation)
+            .setRetrySettings(ServiceOptions.getDefaultRetrySettings())
+            .build();
+
+    try {
+      client = new BigQueryMetastoreClientImpl(options);
+    } catch (IOException e) {
+      throw new UncheckedIOException("Creating BigQuery client failed", e);
+    } catch (GeneralSecurityException e) {
+      throw new RuntimeException("Creating BigQuery client failed due to a security issue", e);
+    }
+
+    initialize(name, properties, projectId, projectLocation, client);
+  }
+
+  @VisibleForTesting
+  void initialize(
+      String name,
+      Map<String, String> properties,
+      String initialProjectId,
+      String initialLocation,
+      BigQueryMetastoreClient bigQueryMetaStoreClient) {
+    Preconditions.checkArgument(bigQueryMetaStoreClient != null, "Invalid BigQuery client: null");
+    this.catalogName = name;
+    this.catalogProperties = ImmutableMap.copyOf(properties);
+    this.projectId = initialProjectId;
+    this.projectLocation = initialLocation;
+    this.client = bigQueryMetaStoreClient;
+
+    if (this.conf == null) {
+      LOG.warn("No configuration was set, using the default environment Configuration");
+      this.conf = new Configuration();
+    }
+
+    LOG.info("Using BigQuery Metastore Iceberg Catalog: {}", name);
+
+    if (properties.containsKey(CatalogProperties.WAREHOUSE_LOCATION)) {
+      this.warehouseLocation =
+          LocationUtil.stripTrailingSlash(properties.get(CatalogProperties.WAREHOUSE_LOCATION));
+    }
+
+    this.fileIO =
+        CatalogUtil.loadFileIO(
+            properties.getOrDefault(
+                CatalogProperties.FILE_IO_IMPL, "org.apache.iceberg.io.ResolvingFileIO"),
+            properties,
+            conf);
+
+    this.listAllTables = Boolean.parseBoolean(properties.getOrDefault(LIST_ALL_TABLES, "true"));
+  }
+
+  @Override
+  protected TableOperations newTableOps(TableIdentifier identifier) {
+    return new BigQueryTableOperations(client, fileIO, toTableReference(identifier), conf);
+  }
+
+  @Override
+  protected String defaultWarehouseLocation(TableIdentifier identifier) {
+    String locationUri = null;
+    DatasetReference datasetReference = toDatasetReference(identifier.namespace());
+    Dataset dataset = client.load(datasetReference);
+    if (dataset != null && dataset.getExternalCatalogDatasetOptions() != null) {
+      locationUri = dataset.getExternalCatalogDatasetOptions().getDefaultStorageLocationUri();
+    }
+
+    return String.format(
+        "%s/%s",
+        Strings.isNullOrEmpty(locationUri)
+            ? createDefaultStorageLocationUri(datasetReference.getDatasetId())
+            : LocationUtil.stripTrailingSlash(locationUri),
+        identifier.name());
+  }
+
+  @Override
+  public List<TableIdentifier> listTables(Namespace namespace) {
+    validateNamespace(namespace);
+
+    return client.list(toDatasetReference(namespace), listAllTables).stream()
+        .map(
+            table -> TableIdentifier.of(namespace.level(0), table.getTableReference().getTableId()))
+        .collect(ImmutableList.toImmutableList());
+  }
+
+  @Override
+  public boolean dropTable(TableIdentifier identifier, boolean purge) {
+    try {
+      TableOperations ops = newTableOps(identifier);
+      TableMetadata lastMetadata = ops.current();
+
+      client.delete(toTableReference(identifier));
+
+      if (purge && lastMetadata != null) {
+        CatalogUtil.dropTableData(ops.io(), lastMetadata);
+      }
+    } catch (NoSuchTableException e) {
+      return false;
+    }
+
+    return true;
+  }
+
+  @Override
+  public void renameTable(TableIdentifier from, TableIdentifier to) {
+    // TODO: Enable once supported by BigQuery API.
+    throw new UnsupportedOperationException("Table rename operation is unsupported.");
+  }
+
+  @Override
+  public void createNamespace(Namespace namespace, Map<String, String> metadata) {
+    Dataset builder = new Dataset();
+    DatasetReference datasetReference = toDatasetReference(namespace);
+    builder.setLocation(this.projectLocation);
+    builder.setDatasetReference(datasetReference);
+    builder.setExternalCatalogDatasetOptions(
+        BigQueryMetastoreUtils.createExternalCatalogDatasetOptions(
+            createDefaultStorageLocationUri(datasetReference.getDatasetId()), metadata));
+
+    client.create(builder);
+  }
+
+  @Override
+  public List<Namespace> listNamespaces() {
+    try {
+      return listNamespaces(Namespace.empty());
+    } catch (NoSuchNamespaceException e) {
+      return ImmutableList.of();
+    }
+  }
+
+  /**
+   * Since this catalog only supports one-level namespaces, it always returns an empty list unless
+   * passed an empty namespace to list all namespaces within the catalog.
+   */
+  @Override
+  public List<Namespace> listNamespaces(Namespace namespace) {
+    List<Datasets> allDatasets = client.list(projectId);
+
+    ImmutableList<Namespace> namespaces =
+        allDatasets.stream().map(this::toNamespace).collect(ImmutableList.toImmutableList());
+    if (namespaces.isEmpty()) {
+      throw new NoSuchNamespaceException("Namespace does not exist: %s", namespace);
+    }
+
+    return namespaces;
+  }
+
+  @Override
+  public boolean dropNamespace(Namespace namespace) {
+    try {
+      client.delete(toDatasetReference(namespace));
+      // We don't delete the data folder for safety, which aligns with Hive Metastore's default
+      // behavior.
+      // We can support database or catalog level config controlling file deletion in the future.
+      return true;
+    } catch (NoSuchNamespaceException e) {
+      return false;
+    }
+  }
+
+  @Override
+  public boolean setProperties(Namespace namespace, Map<String, String> properties) {
+    Dataset dataset = client.load(toDatasetReference(namespace));
+
+    ExternalCatalogDatasetOptions existingOptions = dataset.getExternalCatalogDatasetOptions();
+    Map<String, String> existingParameters =
+        existingOptions != null ? existingOptions.getParameters() : null;
+
+    Map<String, String> newParameters = Maps.newHashMap();
+    if (existingParameters != null) {
+      newParameters.putAll(existingParameters);
+    }
+
+    newParameters.putAll(properties);
+
+    if (Objects.equals(existingParameters, newParameters)) {
+      // No change in parameters detected
+      return false;
+    }
+
+    client.setParameters(toDatasetReference(namespace), properties);

Review Comment:
   Shouldn't this be `newProperties`?  What this appears to do is overwrite whatever the existing properties are with just what is passed in.



-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org

For queries about this service, please contact Infrastructure at:
users@infra.apache.org


---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For additional commands, e-mail: issues-help@iceberg.apache.org


Re: [PR] Catalog: Add BigQuery Metastore Catalog Support [iceberg]

Posted by "nastra (via GitHub)" <gi...@apache.org>.
nastra commented on code in PR #12808:
URL: https://github.com/apache/iceberg/pull/12808#discussion_r2066103931


##########
bigquery/src/test/java/org/apache/iceberg/gcp/bigquery/BigQueryTableOperationsTest.java:
##########
@@ -0,0 +1,274 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.iceberg.gcp.bigquery;
+
+import static org.apache.iceberg.gcp.bigquery.BigQueryMetastoreCatalog.PROJECT_ID;
+import static org.apache.iceberg.types.Types.NestedField.required;
+import static org.assertj.core.api.Assertions.assertThat;
+import static org.assertj.core.api.Assertions.assertThatThrownBy;
+import static org.mockito.Mockito.any;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.reset;
+import static org.mockito.Mockito.times;
+import static org.mockito.Mockito.verify;
+import static org.mockito.Mockito.when;
+
+import com.google.api.services.bigquery.model.Dataset;
+import com.google.api.services.bigquery.model.DatasetReference;
+import com.google.api.services.bigquery.model.ExternalCatalogDatasetOptions;
+import com.google.api.services.bigquery.model.ExternalCatalogTableOptions;
+import com.google.api.services.bigquery.model.StorageDescriptor;
+import com.google.api.services.bigquery.model.Table;
+import com.google.api.services.bigquery.model.TableReference;
+import java.io.File;
+import java.io.IOException;
+import java.util.Collections;
+import java.util.Optional;
+import org.apache.commons.io.FileUtils;
+import org.apache.commons.io.filefilter.TrueFileFilter;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.iceberg.CatalogProperties;
+import org.apache.iceberg.PartitionSpec;
+import org.apache.iceberg.Schema;
+import org.apache.iceberg.catalog.TableIdentifier;
+import org.apache.iceberg.exceptions.CommitFailedException;
+import org.apache.iceberg.exceptions.NoSuchTableException;
+import org.apache.iceberg.exceptions.ValidationException;
+import org.apache.iceberg.relocated.com.google.common.collect.ImmutableMap;
+import org.apache.iceberg.types.Types;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.io.TempDir;
+import org.mockito.ArgumentCaptor;
+
+public class BigQueryTableOperationsTest {
+
+  @TempDir private File tempFolder;
+  public static final String METADATA_LOCATION_PROP = "metadata_location";
+  private static final String GCP_PROJECT = "my-project";
+  private static final String GCP_REGION = "us";
+  private static final String DATASET_ID = "db";
+  private static final String TABLE_ID = "tbl";
+  private static final TableIdentifier SPARK_TABLE_ID = TableIdentifier.of(DATASET_ID, TABLE_ID);
+
+  private static final TableReference TABLE_REFERENCE =
+      new TableReference().setProjectId(GCP_PROJECT).setDatasetId(DATASET_ID).setTableId(TABLE_ID);
+
+  private final BigQueryMetastoreClient bigQueryMetaStoreClient =
+      mock(BigQueryMetastoreClient.class);
+
+  private BigQueryMetastoreCatalog bigQueryMetastoreCatalog;
+  private BigQueryTableOperations tableOps;
+
+  @BeforeEach
+  public void before() {
+    this.bigQueryMetastoreCatalog = new BigQueryMetastoreCatalog();
+    this.bigQueryMetastoreCatalog.setConf(new Configuration());
+    String warehouseLocation = tempFolder.toPath().resolve("hive-warehouse").toString();
+
+    bigQueryMetastoreCatalog.initialize(
+        "CATALOG_ID",
+        ImmutableMap.of(
+            PROJECT_ID,
+            GCP_PROJECT,
+            CatalogProperties.WAREHOUSE_LOCATION,
+            warehouseLocation,
+            CatalogProperties.FILE_IO_IMPL,
+            "org.apache.iceberg.hadoop.HadoopFileIO"),
+        GCP_PROJECT,
+        GCP_REGION,
+        bigQueryMetaStoreClient);
+    this.tableOps = (BigQueryTableOperations) bigQueryMetastoreCatalog.newTableOps(SPARK_TABLE_ID);
+  }
+
+  @Test
+  public void testFetchLatestMetadataFromBigQuery() throws Exception {
+    Table createdTable = createTestTable();
+    reset(bigQueryMetaStoreClient);
+    when(bigQueryMetaStoreClient.load(TABLE_REFERENCE)).thenReturn(createdTable);
+
+    tableOps.refresh();
+    assertThat(
+            createdTable
+                .getExternalCatalogTableOptions()
+                .getParameters()
+                .getOrDefault(METADATA_LOCATION_PROP, ""))
+        .isEqualTo(tableOps.currentMetadataLocation());
+
+    reset(bigQueryMetaStoreClient);
+    when(bigQueryMetaStoreClient.load(TABLE_REFERENCE))
+        .thenThrow(new NoSuchTableException("error message getTable"));
+    // Refresh fails when table is not found but metadata already presents.
+    assertThatThrownBy(() -> tableOps.refresh())
+        .isInstanceOf(NoSuchTableException.class)
+        .hasMessageContaining("error message getTable");
+  }
+
+  @Test
+  public void testFailForNonIcebergTable() {
+    when(bigQueryMetaStoreClient.load(TABLE_REFERENCE))
+        .thenReturn(new Table().setTableReference(TABLE_REFERENCE));
+
+    assertThatThrownBy(() -> tableOps.refresh())
+        .isInstanceOf(ValidationException.class)
+        .hasMessageContaining("metadata location not found");
+  }
+
+  @Test
+  public void testNoOpWhenMetadataAndTableNotFound() {
+    when(bigQueryMetaStoreClient.load(TABLE_REFERENCE))
+        .thenThrow(new NoSuchTableException("error message getTable"));
+    // Table not found won't cause errors when the metadata is null.
+    assertThat(tableOps.currentMetadataLocation()).isNull();
+    tableOps.refresh();
+  }
+
+  @Test
+  public void testTableNameAsExpected() {
+    assertThat(tableOps.tableName()).isEqualTo("db.tbl");
+  }
+
+  @Test
+  public void testUseEtagForUpdateTable() throws Exception {
+    Table tableWithEtag = createTestTable().setEtag("etag");
+    reset(bigQueryMetaStoreClient);
+    when(bigQueryMetaStoreClient.load(TABLE_REFERENCE)).thenReturn(tableWithEtag, tableWithEtag);
+
+    org.apache.iceberg.Table loadedTable = bigQueryMetastoreCatalog.loadTable(SPARK_TABLE_ID);
+
+    when(bigQueryMetaStoreClient.update(any(), any())).thenReturn(tableWithEtag);
+    loadedTable.updateSchema().addColumn("n", Types.IntegerType.get()).commit();
+
+    ArgumentCaptor<TableReference> tableReferenceArgumentCaptor =
+        ArgumentCaptor.forClass(TableReference.class);
+    ArgumentCaptor<Table> tableArgumentCaptor = ArgumentCaptor.forClass(Table.class);
+    verify(bigQueryMetaStoreClient, times(1))
+        .update(tableReferenceArgumentCaptor.capture(), tableArgumentCaptor.capture());
+    assertThat(tableReferenceArgumentCaptor.getValue()).isEqualTo(TABLE_REFERENCE);
+    assertThat(tableArgumentCaptor.getValue().getEtag()).isEqualTo("etag");
+  }
+
+  @Test
+  public void testFailWhenEtagMismatch() throws Exception {
+    Table tableWithEtag = createTestTable().setEtag("etag");
+    reset(bigQueryMetaStoreClient);
+    when(bigQueryMetaStoreClient.load(TABLE_REFERENCE)).thenReturn(tableWithEtag, tableWithEtag);
+
+    org.apache.iceberg.Table loadedTable = bigQueryMetastoreCatalog.loadTable(SPARK_TABLE_ID);
+
+    when(bigQueryMetaStoreClient.update(any(), any()))
+        .thenThrow(new ValidationException("error message etag mismatch"));
+    assertThatThrownBy(
+            () -> loadedTable.updateSchema().addColumn("n", Types.IntegerType.get()).commit())
+        .isInstanceOf(CommitFailedException.class)
+        .hasMessageContaining(
+            "Updating table failed due to conflict updates (etag mismatch). Retry the update");
+  }
+
+  @Test
+  public void testFailWhenMetadataLocationDiff() throws Exception {
+    Table tableWithEtag = createTestTable().setEtag("etag");
+    Table tableWithNewMetadata =
+        new Table()
+            .setEtag("etag")
+            .setExternalCatalogTableOptions(
+                new ExternalCatalogTableOptions()
+                    .setParameters(ImmutableMap.of(METADATA_LOCATION_PROP, "a/new/location")));
+
+    reset(bigQueryMetaStoreClient);
+    // Two invocations, for loadTable and commit.
+    when(bigQueryMetaStoreClient.load(TABLE_REFERENCE))
+        .thenReturn(tableWithEtag, tableWithNewMetadata);
+
+    org.apache.iceberg.Table loadedTable = bigQueryMetastoreCatalog.loadTable(SPARK_TABLE_ID);
+
+    when(bigQueryMetaStoreClient.update(any(), any())).thenReturn(tableWithEtag);
+    assertThatThrownBy(
+            () -> loadedTable.updateSchema().addColumn("n", Types.IntegerType.get()).commit())
+        .isInstanceOf(CommitFailedException.class)
+        .hasMessageContaining("is not same as the current table metadata location");
+  }
+
+  @Test
+  public void testCreateTableCommitSucceeds() throws Exception {
+    var testTable = createTestTable();
+    when(bigQueryMetaStoreClient.create(any(Table.class))).thenReturn(testTable);
+    when(bigQueryMetaStoreClient.load(
+            new DatasetReference().setProjectId(GCP_PROJECT).setDatasetId(DATASET_ID)))
+        .thenReturn(
+            new Dataset()
+                .setExternalCatalogDatasetOptions(
+                    new ExternalCatalogDatasetOptions()
+                        .setDefaultStorageLocationUri("build/db_folder")));
+
+    Schema schema = testSchema();
+    bigQueryMetastoreCatalog.createTable(SPARK_TABLE_ID, schema, PartitionSpec.unpartitioned());

Review Comment:
   there should be some assertions verifying that the table can be loaded or so. Just having code without an assertion looks weird inside test methods



-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org

For queries about this service, please contact Infrastructure at:
users@infra.apache.org


---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For additional commands, e-mail: issues-help@iceberg.apache.org


Re: [PR] Catalog: Add BigQuery Metastore Catalog Support [iceberg]

Posted by "nastra (via GitHub)" <gi...@apache.org>.
nastra commented on code in PR #12808:
URL: https://github.com/apache/iceberg/pull/12808#discussion_r2066094802


##########
bigquery/src/test/java/org/apache/iceberg/gcp/bigquery/BigQueryTableOperationsTest.java:
##########
@@ -0,0 +1,274 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.iceberg.gcp.bigquery;
+
+import static org.apache.iceberg.gcp.bigquery.BigQueryMetastoreCatalog.PROJECT_ID;
+import static org.apache.iceberg.types.Types.NestedField.required;
+import static org.assertj.core.api.Assertions.assertThat;
+import static org.assertj.core.api.Assertions.assertThatThrownBy;
+import static org.mockito.Mockito.any;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.reset;
+import static org.mockito.Mockito.times;
+import static org.mockito.Mockito.verify;
+import static org.mockito.Mockito.when;
+
+import com.google.api.services.bigquery.model.Dataset;
+import com.google.api.services.bigquery.model.DatasetReference;
+import com.google.api.services.bigquery.model.ExternalCatalogDatasetOptions;
+import com.google.api.services.bigquery.model.ExternalCatalogTableOptions;
+import com.google.api.services.bigquery.model.StorageDescriptor;
+import com.google.api.services.bigquery.model.Table;
+import com.google.api.services.bigquery.model.TableReference;
+import java.io.File;
+import java.io.IOException;
+import java.util.Collections;
+import java.util.Optional;
+import org.apache.commons.io.FileUtils;
+import org.apache.commons.io.filefilter.TrueFileFilter;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.iceberg.CatalogProperties;
+import org.apache.iceberg.PartitionSpec;
+import org.apache.iceberg.Schema;
+import org.apache.iceberg.catalog.TableIdentifier;
+import org.apache.iceberg.exceptions.CommitFailedException;
+import org.apache.iceberg.exceptions.NoSuchTableException;
+import org.apache.iceberg.exceptions.ValidationException;
+import org.apache.iceberg.relocated.com.google.common.collect.ImmutableMap;
+import org.apache.iceberg.types.Types;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.io.TempDir;
+import org.mockito.ArgumentCaptor;
+
+public class BigQueryTableOperationsTest {
+
+  @TempDir private File tempFolder;
+  public static final String METADATA_LOCATION_PROP = "metadata_location";
+  private static final String GCP_PROJECT = "my-project";
+  private static final String GCP_REGION = "us";
+  private static final String DATASET_ID = "db";
+  private static final String TABLE_ID = "tbl";
+  private static final TableIdentifier SPARK_TABLE_ID = TableIdentifier.of(DATASET_ID, TABLE_ID);
+
+  private static final TableReference TABLE_REFERENCE =
+      new TableReference().setProjectId(GCP_PROJECT).setDatasetId(DATASET_ID).setTableId(TABLE_ID);
+
+  private final BigQueryMetastoreClient bigQueryMetaStoreClient =
+      mock(BigQueryMetastoreClient.class);
+
+  private BigQueryMetastoreCatalog bigQueryMetastoreCatalog;
+  private BigQueryTableOperations tableOps;
+
+  @BeforeEach
+  public void before() {
+    this.bigQueryMetastoreCatalog = new BigQueryMetastoreCatalog();
+    this.bigQueryMetastoreCatalog.setConf(new Configuration());
+    String warehouseLocation = tempFolder.toPath().resolve("hive-warehouse").toString();
+
+    bigQueryMetastoreCatalog.initialize(
+        "CATALOG_ID",
+        ImmutableMap.of(
+            PROJECT_ID,
+            GCP_PROJECT,
+            CatalogProperties.WAREHOUSE_LOCATION,
+            warehouseLocation,
+            CatalogProperties.FILE_IO_IMPL,
+            "org.apache.iceberg.hadoop.HadoopFileIO"),
+        GCP_PROJECT,
+        GCP_REGION,
+        bigQueryMetaStoreClient);
+    this.tableOps = (BigQueryTableOperations) bigQueryMetastoreCatalog.newTableOps(SPARK_TABLE_ID);
+  }
+
+  @Test
+  public void testFetchLatestMetadataFromBigQuery() throws Exception {
+    Table createdTable = createTestTable();
+    reset(bigQueryMetaStoreClient);
+    when(bigQueryMetaStoreClient.load(TABLE_REFERENCE)).thenReturn(createdTable);
+
+    tableOps.refresh();
+    assertThat(
+            createdTable
+                .getExternalCatalogTableOptions()
+                .getParameters()
+                .getOrDefault(METADATA_LOCATION_PROP, ""))
+        .isEqualTo(tableOps.currentMetadataLocation());
+
+    reset(bigQueryMetaStoreClient);
+    when(bigQueryMetaStoreClient.load(TABLE_REFERENCE))
+        .thenThrow(new NoSuchTableException("error message getTable"));
+    // Refresh fails when table is not found but metadata already presents.
+    assertThatThrownBy(() -> tableOps.refresh())
+        .isInstanceOf(NoSuchTableException.class)
+        .hasMessageContaining("error message getTable");
+  }
+
+  @Test
+  public void testFailForNonIcebergTable() {

Review Comment:
   ```suggestion
     public void loadingNonIcebergTableFails() {
   ```



-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org

For queries about this service, please contact Infrastructure at:
users@infra.apache.org


---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For additional commands, e-mail: issues-help@iceberg.apache.org


Re: [PR] Catalog: Add BigQuery Metastore Catalog Support [iceberg]

Posted by "nastra (via GitHub)" <gi...@apache.org>.
nastra commented on code in PR #12808:
URL: https://github.com/apache/iceberg/pull/12808#discussion_r2066121344


##########
build.gradle:
##########
@@ -657,6 +657,47 @@ project(':iceberg-delta-lake') {
   }
 }
 
+project(':iceberg-bigquery') {
+  test {
+    useJUnitPlatform()
+  }
+
+  dependencies {
+    implementation project(path: ':iceberg-bundled-guava', configuration: 'shadow')
+    implementation project(':iceberg-gcp')

Review Comment:
   @talatuyarer I've reopened this as it's not clear to me why this dependency here is needed. I understand that BigQuery would always use GCSFileIO but I don't think this dependency here is required (as nothing in the BigQuery module references anything from the gcp module). We do have the gcp-bundle for users that actually use `GCSFileIO`
   
   



-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org

For queries about this service, please contact Infrastructure at:
users@infra.apache.org


---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For additional commands, e-mail: issues-help@iceberg.apache.org


Re: [PR] Catalog: Add BigQuery Metastore Catalog Support [iceberg]

Posted by "nastra (via GitHub)" <gi...@apache.org>.
nastra commented on code in PR #12808:
URL: https://github.com/apache/iceberg/pull/12808#discussion_r2063976420


##########
bigquery/src/main/java/org/apache/iceberg/gcp/bigquery/BigQueryMetastoreCatalog.java:
##########
@@ -0,0 +1,394 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.iceberg.gcp.bigquery;
+
+import com.google.api.services.bigquery.model.Dataset;
+import com.google.api.services.bigquery.model.DatasetList.Datasets;
+import com.google.api.services.bigquery.model.DatasetReference;
+import com.google.api.services.bigquery.model.ExternalCatalogDatasetOptions;
+import com.google.api.services.bigquery.model.TableReference;
+import com.google.cloud.ServiceOptions;
+import com.google.cloud.bigquery.BigQueryOptions;
+import java.io.IOException;
+import java.security.GeneralSecurityException;
+import java.util.List;
+import java.util.Locale;
+import java.util.Map;
+import java.util.Objects;
+import java.util.Set;
+import org.apache.hadoop.conf.Configurable;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.iceberg.BaseMetastoreCatalog;
+import org.apache.iceberg.CatalogProperties;
+import org.apache.iceberg.CatalogUtil;
+import org.apache.iceberg.TableMetadata;
+import org.apache.iceberg.TableOperations;
+import org.apache.iceberg.catalog.Namespace;
+import org.apache.iceberg.catalog.SupportsNamespaces;
+import org.apache.iceberg.catalog.TableIdentifier;
+import org.apache.iceberg.exceptions.AlreadyExistsException;
+import org.apache.iceberg.exceptions.NoSuchNamespaceException;
+import org.apache.iceberg.exceptions.NoSuchTableException;
+import org.apache.iceberg.exceptions.ServiceFailureException;
+import org.apache.iceberg.exceptions.ValidationException;
+import org.apache.iceberg.io.FileIO;
+import org.apache.iceberg.relocated.com.google.common.annotations.VisibleForTesting;
+import org.apache.iceberg.relocated.com.google.common.base.Preconditions;
+import org.apache.iceberg.relocated.com.google.common.base.Strings;
+import org.apache.iceberg.relocated.com.google.common.collect.ImmutableList;
+import org.apache.iceberg.relocated.com.google.common.collect.ImmutableMap;
+import org.apache.iceberg.relocated.com.google.common.collect.Maps;
+import org.apache.iceberg.util.LocationUtil;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/** Iceberg Bigquery Metastore Catalog implementation. */
+public class BigQueryMetastoreCatalog extends BaseMetastoreCatalog
+    implements SupportsNamespaces, Configurable {
+
+  // User provided properties.
+  public static final String PROJECT_ID = "gcp-project";
+  public static final String GCP_LOCATION = "gcp-location";
+  public static final String LIST_ALL_TABLES = "list-all-tables";
+
+  public static final String HIVE_METASTORE_WAREHOUSE_DIR = "hive.metastore.warehouse.dir";
+
+  private static final Logger LOG = LoggerFactory.getLogger(BigQueryMetastoreCatalog.class);
+
+  private static final String DEFAULT_GCP_LOCATION = "us";
+
+  private String catalogName;
+  private Map<String, String> catalogProperties;
+  private FileIO fileIO;
+  private Configuration conf;
+  private String projectId;
+  private String location;
+  private BigQueryMetastoreClient client;
+  private boolean listAllTables;
+
+  // Must have a no-arg constructor to be dynamically loaded
+  // initialize(String name, Map<String, String> properties) will be called to complete
+  // initialization
+  public BigQueryMetastoreCatalog() {}
+
+  @Override
+  public void initialize(String inputName, Map<String, String> properties) {
+    Preconditions.checkArgument(
+        properties.containsKey(PROJECT_ID), "GCP project must be specified");
+
+    projectId = properties.get(PROJECT_ID);
+    location = properties.getOrDefault(GCP_LOCATION, DEFAULT_GCP_LOCATION);
+
+    BigQueryOptions options =
+        BigQueryOptions.newBuilder()
+            .setProjectId(projectId)
+            .setLocation(location)
+            .setRetrySettings(ServiceOptions.getDefaultRetrySettings())
+            .build();
+
+    try {
+      client = new BigQueryMetastoreClientImpl(options);
+    } catch (IOException e) {
+      throw new ServiceFailureException(e, "Creating BigQuery client failed");
+    } catch (GeneralSecurityException e) {
+      throw new ValidationException(e, "Creating BigQuery client failed due to a security issue");
+    }
+
+    initialize(inputName, properties, projectId, location, client);
+  }
+
+  @VisibleForTesting
+  void initialize(
+      String inputName,
+      Map<String, String> properties,
+      String initialProjectId,
+      String initialLocation,
+      BigQueryMetastoreClient bigQueryMetaStoreClient) {
+    Preconditions.checkArgument(bigQueryMetaStoreClient != null, "BigQuery client can not be null");
+    this.catalogName = inputName;
+    this.catalogProperties = ImmutableMap.copyOf(properties);
+    this.projectId = initialProjectId;
+    this.location = initialLocation;
+    this.client = bigQueryMetaStoreClient;
+
+    if (this.conf == null) {
+      LOG.warn("No configuration was set, using the default environment Configuration");
+      this.conf = new Configuration();
+    }
+
+    LOG.info("Using BigQuery Metastore Iceberg Catalog: {}", inputName);
+
+    if (properties.containsKey(CatalogProperties.WAREHOUSE_LOCATION)) {
+      this.conf.set(
+          HIVE_METASTORE_WAREHOUSE_DIR,
+          LocationUtil.stripTrailingSlash(properties.get(CatalogProperties.WAREHOUSE_LOCATION)));
+    }
+
+    String fileIoImpl =
+        properties.getOrDefault(
+            CatalogProperties.FILE_IO_IMPL, "org.apache.iceberg.hadoop.HadoopFileIO");
+    this.fileIO = CatalogUtil.loadFileIO(fileIoImpl, properties, conf);
+
+    this.listAllTables = Boolean.parseBoolean(properties.getOrDefault(LIST_ALL_TABLES, "true"));
+  }
+
+  @Override
+  protected TableOperations newTableOps(TableIdentifier identifier) {
+    return new BigQueryTableOperations(client, fileIO, toTableReference(identifier), conf);
+  }
+
+  @Override
+  protected String defaultWarehouseLocation(TableIdentifier identifier) {
+    String locationUri = null;
+    DatasetReference datasetReference = toDatasetReference(identifier.namespace());
+    Dataset dataset = client.load(datasetReference);
+    if (dataset != null && dataset.getExternalCatalogDatasetOptions() != null) {
+      locationUri = dataset.getExternalCatalogDatasetOptions().getDefaultStorageLocationUri();
+    }
+
+    return String.format(
+        "%s/%s",
+        Strings.isNullOrEmpty(locationUri)
+            ? createDefaultStorageLocationUri(datasetReference.getDatasetId())
+            : LocationUtil.stripTrailingSlash(locationUri),
+        identifier.name());
+  }
+
+  @Override
+  public List<TableIdentifier> listTables(Namespace namespace) {
+    validateNamespace(namespace);
+
+    return client.list(toDatasetReference(namespace), listAllTables).stream()
+        .map(
+            table -> TableIdentifier.of(namespace.level(0), table.getTableReference().getTableId()))
+        .collect(ImmutableList.toImmutableList());
+  }
+
+  @Override
+  public boolean dropTable(TableIdentifier identifier, boolean purge) {
+    try {
+      TableOperations ops = newTableOps(identifier);
+      TableMetadata lastMetadata = ops.current();
+
+      client.delete(toTableReference(identifier));
+
+      if (purge && lastMetadata != null) {
+        CatalogUtil.dropTableData(ops.io(), lastMetadata);
+      }
+    } catch (NoSuchTableException e) {
+      return false;
+    }
+
+    return true;
+  }
+
+  @Override
+  public void renameTable(TableIdentifier from, TableIdentifier to) {
+    validateNamespace(from.namespace());
+    validateNamespace(to.namespace());
+
+    if (!namespaceExists(to.namespace())) {
+      throw new NoSuchNamespaceException("Namespace does not exist: %s", to.namespace());
+    }
+
+    if (!namespaceExists(from.namespace())) {
+      throw new NoSuchNamespaceException("Namespace does not exist: %s", from.namespace());
+    }
+
+    // Check if the source table exists
+    if (!tableExists(from)) {
+      throw new NoSuchTableException("Table does not exist: %s", from.name());
+    }
+
+    if (!from.namespace().equals(to.namespace())) {
+      throw new ValidationException("New table name must be in the same namespace");
+    }
+
+    // Check if the destination table already exists
+    if (tableExists(to)) {
+      throw new AlreadyExistsException("Table already exists: %s", to.name());

Review Comment:
   I think here and in the other error messages during rename it would be beneficial if the error would have `Cannot rename <from> to <to>` as a prefix



-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org

For queries about this service, please contact Infrastructure at:
users@infra.apache.org


---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For additional commands, e-mail: issues-help@iceberg.apache.org


Re: [PR] Catalog: Add BigQuery Metastore Catalog Support [iceberg]

Posted by "danielcweeks (via GitHub)" <gi...@apache.org>.
danielcweeks commented on code in PR #12808:
URL: https://github.com/apache/iceberg/pull/12808#discussion_r2067558777


##########
bigquery/src/main/java/org/apache/iceberg/gcp/bigquery/BigQueryMetastoreCatalog.java:
##########
@@ -0,0 +1,405 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.iceberg.gcp.bigquery;
+
+import com.google.api.services.bigquery.model.Dataset;
+import com.google.api.services.bigquery.model.DatasetList.Datasets;
+import com.google.api.services.bigquery.model.DatasetReference;
+import com.google.api.services.bigquery.model.ExternalCatalogDatasetOptions;
+import com.google.api.services.bigquery.model.TableReference;
+import com.google.cloud.ServiceOptions;
+import com.google.cloud.bigquery.BigQueryOptions;
+import java.io.IOException;
+import java.security.GeneralSecurityException;
+import java.util.List;
+import java.util.Locale;
+import java.util.Map;
+import java.util.Objects;
+import java.util.Set;
+import org.apache.hadoop.conf.Configurable;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.iceberg.BaseMetastoreCatalog;
+import org.apache.iceberg.CatalogProperties;
+import org.apache.iceberg.CatalogUtil;
+import org.apache.iceberg.TableMetadata;
+import org.apache.iceberg.TableOperations;
+import org.apache.iceberg.catalog.Namespace;
+import org.apache.iceberg.catalog.SupportsNamespaces;
+import org.apache.iceberg.catalog.TableIdentifier;
+import org.apache.iceberg.exceptions.AlreadyExistsException;
+import org.apache.iceberg.exceptions.NoSuchNamespaceException;
+import org.apache.iceberg.exceptions.NoSuchTableException;
+import org.apache.iceberg.exceptions.ServiceFailureException;
+import org.apache.iceberg.exceptions.ValidationException;
+import org.apache.iceberg.io.FileIO;
+import org.apache.iceberg.relocated.com.google.common.annotations.VisibleForTesting;
+import org.apache.iceberg.relocated.com.google.common.base.Preconditions;
+import org.apache.iceberg.relocated.com.google.common.base.Strings;
+import org.apache.iceberg.relocated.com.google.common.collect.ImmutableList;
+import org.apache.iceberg.relocated.com.google.common.collect.ImmutableMap;
+import org.apache.iceberg.relocated.com.google.common.collect.Maps;
+import org.apache.iceberg.util.LocationUtil;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/** Iceberg Bigquery Metastore Catalog implementation. */
+public class BigQueryMetastoreCatalog extends BaseMetastoreCatalog
+    implements SupportsNamespaces, Configurable {
+
+  // User provided properties.
+  public static final String PROJECT_ID = "gcp-project";
+  public static final String GCP_LOCATION = "gcp-location";
+  public static final String LIST_ALL_TABLES = "list-all-tables";
+
+  public static final String HIVE_METASTORE_WAREHOUSE_DIR = "hive.metastore.warehouse.dir";

Review Comment:
   Do we even need the hadoop configuration?  It feels like we're relying on the hadoop config to store the warehouse location, but it doesn't look like we actually need to do that.  Can we just capture the warehouse location and not rely on the config?



-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org

For queries about this service, please contact Infrastructure at:
users@infra.apache.org


---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For additional commands, e-mail: issues-help@iceberg.apache.org


Re: [PR] Catalog: Add BigQuery Metastore Catalog Support [iceberg]

Posted by "talatuyarer (via GitHub)" <gi...@apache.org>.
talatuyarer commented on code in PR #12808:
URL: https://github.com/apache/iceberg/pull/12808#discussion_r2074077072


##########
bigquery/src/main/java/org/apache/iceberg/gcp/bigquery/BigQueryMetastoreCatalog.java:
##########
@@ -0,0 +1,366 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.iceberg.gcp.bigquery;
+
+import com.google.api.services.bigquery.model.Dataset;
+import com.google.api.services.bigquery.model.DatasetList.Datasets;
+import com.google.api.services.bigquery.model.DatasetReference;
+import com.google.api.services.bigquery.model.ExternalCatalogDatasetOptions;
+import com.google.api.services.bigquery.model.TableReference;
+import com.google.cloud.ServiceOptions;
+import com.google.cloud.bigquery.BigQueryOptions;
+import java.io.IOException;
+import java.io.UncheckedIOException;
+import java.security.GeneralSecurityException;
+import java.util.List;
+import java.util.Locale;
+import java.util.Map;
+import java.util.Objects;
+import java.util.Set;
+import org.apache.hadoop.conf.Configurable;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.iceberg.BaseMetastoreCatalog;
+import org.apache.iceberg.CatalogProperties;
+import org.apache.iceberg.CatalogUtil;
+import org.apache.iceberg.TableMetadata;
+import org.apache.iceberg.TableOperations;
+import org.apache.iceberg.catalog.Namespace;
+import org.apache.iceberg.catalog.SupportsNamespaces;
+import org.apache.iceberg.catalog.TableIdentifier;
+import org.apache.iceberg.exceptions.NoSuchNamespaceException;
+import org.apache.iceberg.exceptions.NoSuchTableException;
+import org.apache.iceberg.io.FileIO;
+import org.apache.iceberg.relocated.com.google.common.annotations.VisibleForTesting;
+import org.apache.iceberg.relocated.com.google.common.base.Preconditions;
+import org.apache.iceberg.relocated.com.google.common.base.Strings;
+import org.apache.iceberg.relocated.com.google.common.collect.ImmutableList;
+import org.apache.iceberg.relocated.com.google.common.collect.ImmutableMap;
+import org.apache.iceberg.relocated.com.google.common.collect.Maps;
+import org.apache.iceberg.util.LocationUtil;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/** Iceberg Bigquery Metastore Catalog implementation. */
+public class BigQueryMetastoreCatalog extends BaseMetastoreCatalog
+    implements SupportsNamespaces, Configurable {
+
+  // User provided properties.
+  public static final String PROJECT_ID = "gcp.bigquery.project-id";
+  public static final String GCP_LOCATION = "gcp.bigquery.location";
+  public static final String LIST_ALL_TABLES = "gcp.bigquery.list-all-tables";
+
+  private static final Logger LOG = LoggerFactory.getLogger(BigQueryMetastoreCatalog.class);
+
+  private static final String DEFAULT_GCP_LOCATION = "us";
+
+  private String catalogName;
+  private Map<String, String> catalogProperties;
+  private FileIO fileIO;
+  private Configuration conf;
+  private String projectId;
+  private String projectLocation;
+  private BigQueryMetastoreClient client;
+  private boolean listAllTables;
+  private String warehouseLocation;
+
+  public BigQueryMetastoreCatalog() {}
+
+  @Override
+  public void initialize(String name, Map<String, String> properties) {
+    Preconditions.checkArgument(
+        properties.containsKey(PROJECT_ID),
+        "Invalid GCP project: %s must be specified",
+        PROJECT_ID);
+
+    this.projectId = properties.get(PROJECT_ID);
+    this.projectLocation = properties.getOrDefault(GCP_LOCATION, DEFAULT_GCP_LOCATION);
+
+    BigQueryOptions options =
+        BigQueryOptions.newBuilder()
+            .setProjectId(projectId)
+            .setLocation(projectLocation)
+            .setRetrySettings(ServiceOptions.getDefaultRetrySettings())
+            .build();
+
+    try {
+      client = new BigQueryMetastoreClientImpl(options);
+    } catch (IOException e) {
+      throw new UncheckedIOException("Creating BigQuery client failed", e);
+    } catch (GeneralSecurityException e) {
+      throw new RuntimeException("Creating BigQuery client failed due to a security issue", e);
+    }
+
+    initialize(name, properties, projectId, projectLocation, client);
+  }
+
+  @VisibleForTesting
+  void initialize(
+      String name,
+      Map<String, String> properties,
+      String initialProjectId,
+      String initialLocation,
+      BigQueryMetastoreClient bigQueryMetaStoreClient) {
+    Preconditions.checkArgument(bigQueryMetaStoreClient != null, "Invalid BigQuery client: null");
+    this.catalogName = name;
+    this.catalogProperties = ImmutableMap.copyOf(properties);
+    this.projectId = initialProjectId;
+    this.projectLocation = initialLocation;
+    this.client = bigQueryMetaStoreClient;
+
+    if (this.conf == null) {
+      LOG.warn("No configuration was set, using the default environment Configuration");
+      this.conf = new Configuration();
+    }
+
+    LOG.info("Using BigQuery Metastore Iceberg Catalog: {}", name);
+
+    if (properties.containsKey(CatalogProperties.WAREHOUSE_LOCATION)) {
+      this.warehouseLocation =
+          LocationUtil.stripTrailingSlash(properties.get(CatalogProperties.WAREHOUSE_LOCATION));
+    }
+
+    this.fileIO =
+        CatalogUtil.loadFileIO(
+            properties.getOrDefault(
+                CatalogProperties.FILE_IO_IMPL, "org.apache.iceberg.io.ResolvingFileIO"),
+            properties,
+            conf);
+
+    this.listAllTables = Boolean.parseBoolean(properties.getOrDefault(LIST_ALL_TABLES, "true"));
+  }
+
+  @Override
+  protected TableOperations newTableOps(TableIdentifier identifier) {
+    return new BigQueryTableOperations(client, fileIO, toTableReference(identifier), conf);
+  }
+
+  @Override
+  protected String defaultWarehouseLocation(TableIdentifier identifier) {
+    String locationUri = null;
+    DatasetReference datasetReference = toDatasetReference(identifier.namespace());
+    Dataset dataset = client.load(datasetReference);
+    if (dataset != null && dataset.getExternalCatalogDatasetOptions() != null) {
+      locationUri = dataset.getExternalCatalogDatasetOptions().getDefaultStorageLocationUri();
+    }
+
+    return String.format(
+        "%s/%s",
+        Strings.isNullOrEmpty(locationUri)
+            ? createDefaultStorageLocationUri(datasetReference.getDatasetId())
+            : LocationUtil.stripTrailingSlash(locationUri),
+        identifier.name());
+  }
+
+  @Override
+  public List<TableIdentifier> listTables(Namespace namespace) {
+    validateNamespace(namespace);
+
+    return client.list(toDatasetReference(namespace), listAllTables).stream()
+        .map(
+            table -> TableIdentifier.of(namespace.level(0), table.getTableReference().getTableId()))
+        .collect(ImmutableList.toImmutableList());
+  }
+
+  @Override
+  public boolean dropTable(TableIdentifier identifier, boolean purge) {
+    try {
+      TableOperations ops = newTableOps(identifier);
+      TableMetadata lastMetadata = ops.current();
+
+      client.delete(toTableReference(identifier));
+
+      if (purge && lastMetadata != null) {
+        CatalogUtil.dropTableData(ops.io(), lastMetadata);
+      }
+    } catch (NoSuchTableException e) {
+      return false;
+    }
+
+    return true;
+  }
+
+  @Override
+  public void renameTable(TableIdentifier from, TableIdentifier to) {
+    // TODO: Enable once supported by BigQuery API.
+    throw new UnsupportedOperationException("Table rename operation is unsupported.");
+  }
+
+  @Override
+  public void createNamespace(Namespace namespace, Map<String, String> metadata) {
+    Dataset builder = new Dataset();
+    DatasetReference datasetReference = toDatasetReference(namespace);
+    builder.setLocation(this.projectLocation);
+    builder.setDatasetReference(datasetReference);
+    builder.setExternalCatalogDatasetOptions(
+        BigQueryMetastoreUtils.createExternalCatalogDatasetOptions(
+            createDefaultStorageLocationUri(datasetReference.getDatasetId()), metadata));
+
+    client.create(builder);
+  }
+
+  @Override
+  public List<Namespace> listNamespaces() {
+    try {
+      return listNamespaces(Namespace.empty());
+    } catch (NoSuchNamespaceException e) {
+      return ImmutableList.of();
+    }
+  }
+
+  /**
+   * Since this catalog only supports one-level namespaces, it always returns an empty list unless
+   * passed an empty namespace to list all namespaces within the catalog.
+   */
+  @Override
+  public List<Namespace> listNamespaces(Namespace namespace) {
+    List<Datasets> allDatasets = client.list(projectId);
+
+    ImmutableList<Namespace> namespaces =
+        allDatasets.stream().map(this::toNamespace).collect(ImmutableList.toImmutableList());
+    if (namespaces.isEmpty()) {
+      throw new NoSuchNamespaceException("Namespace does not exist: %s", namespace);
+    }
+
+    return namespaces;
+  }
+
+  @Override
+  public boolean dropNamespace(Namespace namespace) {
+    try {
+      client.delete(toDatasetReference(namespace));
+      // We don't delete the data folder for safety, which aligns with Hive Metastore's default
+      // behavior.
+      // We can support database or catalog level config controlling file deletion in the future.
+      return true;
+    } catch (NoSuchNamespaceException e) {
+      return false;
+    }
+  }
+
+  @Override
+  public boolean setProperties(Namespace namespace, Map<String, String> properties) {
+    Dataset dataset = client.load(toDatasetReference(namespace));
+
+    ExternalCatalogDatasetOptions existingOptions = dataset.getExternalCatalogDatasetOptions();
+    Map<String, String> existingParameters =
+        existingOptions != null ? existingOptions.getParameters() : null;
+
+    Map<String, String> newParameters = Maps.newHashMap();
+    if (existingParameters != null) {
+      newParameters.putAll(existingParameters);
+    }
+
+    newParameters.putAll(properties);
+
+    if (Objects.equals(existingParameters, newParameters)) {
+      // No change in parameters detected
+      return false;
+    }
+
+    client.setParameters(toDatasetReference(namespace), properties);
+    return true;
+  }
+
+  @Override
+  public boolean removeProperties(Namespace namespace, Set<String> properties) {
+    client.removeParameters(toDatasetReference(namespace), properties);
+    return true;
+  }
+
+  @Override
+  public Map<String, String> loadNamespaceMetadata(Namespace namespace) {
+    try {
+      return toMetadata(client.load(toDatasetReference(namespace)));
+    } catch (IllegalArgumentException e) {

Review Comment:
   if namespace name is not correct we can reach IAE via validateNamespace function



##########
bigquery/src/main/java/org/apache/iceberg/gcp/bigquery/BigQueryMetastoreCatalog.java:
##########
@@ -0,0 +1,366 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.iceberg.gcp.bigquery;
+
+import com.google.api.services.bigquery.model.Dataset;
+import com.google.api.services.bigquery.model.DatasetList.Datasets;
+import com.google.api.services.bigquery.model.DatasetReference;
+import com.google.api.services.bigquery.model.ExternalCatalogDatasetOptions;
+import com.google.api.services.bigquery.model.TableReference;
+import com.google.cloud.ServiceOptions;
+import com.google.cloud.bigquery.BigQueryOptions;
+import java.io.IOException;
+import java.io.UncheckedIOException;
+import java.security.GeneralSecurityException;
+import java.util.List;
+import java.util.Locale;
+import java.util.Map;
+import java.util.Objects;
+import java.util.Set;
+import org.apache.hadoop.conf.Configurable;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.iceberg.BaseMetastoreCatalog;
+import org.apache.iceberg.CatalogProperties;
+import org.apache.iceberg.CatalogUtil;
+import org.apache.iceberg.TableMetadata;
+import org.apache.iceberg.TableOperations;
+import org.apache.iceberg.catalog.Namespace;
+import org.apache.iceberg.catalog.SupportsNamespaces;
+import org.apache.iceberg.catalog.TableIdentifier;
+import org.apache.iceberg.exceptions.NoSuchNamespaceException;
+import org.apache.iceberg.exceptions.NoSuchTableException;
+import org.apache.iceberg.io.FileIO;
+import org.apache.iceberg.relocated.com.google.common.annotations.VisibleForTesting;
+import org.apache.iceberg.relocated.com.google.common.base.Preconditions;
+import org.apache.iceberg.relocated.com.google.common.base.Strings;
+import org.apache.iceberg.relocated.com.google.common.collect.ImmutableList;
+import org.apache.iceberg.relocated.com.google.common.collect.ImmutableMap;
+import org.apache.iceberg.relocated.com.google.common.collect.Maps;
+import org.apache.iceberg.util.LocationUtil;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/** Iceberg Bigquery Metastore Catalog implementation. */
+public class BigQueryMetastoreCatalog extends BaseMetastoreCatalog
+    implements SupportsNamespaces, Configurable {
+
+  // User provided properties.
+  public static final String PROJECT_ID = "gcp.bigquery.project-id";
+  public static final String GCP_LOCATION = "gcp.bigquery.location";
+  public static final String LIST_ALL_TABLES = "gcp.bigquery.list-all-tables";
+
+  private static final Logger LOG = LoggerFactory.getLogger(BigQueryMetastoreCatalog.class);
+
+  private static final String DEFAULT_GCP_LOCATION = "us";
+
+  private String catalogName;
+  private Map<String, String> catalogProperties;
+  private FileIO fileIO;
+  private Configuration conf;
+  private String projectId;
+  private String projectLocation;
+  private BigQueryMetastoreClient client;
+  private boolean listAllTables;
+  private String warehouseLocation;
+
+  public BigQueryMetastoreCatalog() {}
+
+  @Override
+  public void initialize(String name, Map<String, String> properties) {
+    Preconditions.checkArgument(
+        properties.containsKey(PROJECT_ID),
+        "Invalid GCP project: %s must be specified",
+        PROJECT_ID);
+
+    this.projectId = properties.get(PROJECT_ID);
+    this.projectLocation = properties.getOrDefault(GCP_LOCATION, DEFAULT_GCP_LOCATION);
+
+    BigQueryOptions options =
+        BigQueryOptions.newBuilder()
+            .setProjectId(projectId)
+            .setLocation(projectLocation)
+            .setRetrySettings(ServiceOptions.getDefaultRetrySettings())
+            .build();
+
+    try {
+      client = new BigQueryMetastoreClientImpl(options);
+    } catch (IOException e) {
+      throw new UncheckedIOException("Creating BigQuery client failed", e);
+    } catch (GeneralSecurityException e) {
+      throw new RuntimeException("Creating BigQuery client failed due to a security issue", e);
+    }
+
+    initialize(name, properties, projectId, projectLocation, client);
+  }
+
+  @VisibleForTesting
+  void initialize(
+      String name,
+      Map<String, String> properties,
+      String initialProjectId,
+      String initialLocation,
+      BigQueryMetastoreClient bigQueryMetaStoreClient) {
+    Preconditions.checkArgument(bigQueryMetaStoreClient != null, "Invalid BigQuery client: null");
+    this.catalogName = name;
+    this.catalogProperties = ImmutableMap.copyOf(properties);
+    this.projectId = initialProjectId;
+    this.projectLocation = initialLocation;
+    this.client = bigQueryMetaStoreClient;
+
+    if (this.conf == null) {
+      LOG.warn("No configuration was set, using the default environment Configuration");
+      this.conf = new Configuration();
+    }
+
+    LOG.info("Using BigQuery Metastore Iceberg Catalog: {}", name);
+
+    if (properties.containsKey(CatalogProperties.WAREHOUSE_LOCATION)) {
+      this.warehouseLocation =
+          LocationUtil.stripTrailingSlash(properties.get(CatalogProperties.WAREHOUSE_LOCATION));
+    }
+
+    this.fileIO =
+        CatalogUtil.loadFileIO(
+            properties.getOrDefault(
+                CatalogProperties.FILE_IO_IMPL, "org.apache.iceberg.io.ResolvingFileIO"),
+            properties,
+            conf);
+
+    this.listAllTables = Boolean.parseBoolean(properties.getOrDefault(LIST_ALL_TABLES, "true"));
+  }
+
+  @Override
+  protected TableOperations newTableOps(TableIdentifier identifier) {
+    return new BigQueryTableOperations(client, fileIO, toTableReference(identifier), conf);
+  }
+
+  @Override
+  protected String defaultWarehouseLocation(TableIdentifier identifier) {
+    String locationUri = null;
+    DatasetReference datasetReference = toDatasetReference(identifier.namespace());
+    Dataset dataset = client.load(datasetReference);
+    if (dataset != null && dataset.getExternalCatalogDatasetOptions() != null) {
+      locationUri = dataset.getExternalCatalogDatasetOptions().getDefaultStorageLocationUri();
+    }
+
+    return String.format(
+        "%s/%s",
+        Strings.isNullOrEmpty(locationUri)
+            ? createDefaultStorageLocationUri(datasetReference.getDatasetId())
+            : LocationUtil.stripTrailingSlash(locationUri),
+        identifier.name());
+  }
+
+  @Override
+  public List<TableIdentifier> listTables(Namespace namespace) {
+    validateNamespace(namespace);
+
+    return client.list(toDatasetReference(namespace), listAllTables).stream()
+        .map(
+            table -> TableIdentifier.of(namespace.level(0), table.getTableReference().getTableId()))
+        .collect(ImmutableList.toImmutableList());
+  }
+
+  @Override
+  public boolean dropTable(TableIdentifier identifier, boolean purge) {
+    try {
+      TableOperations ops = newTableOps(identifier);
+      TableMetadata lastMetadata = ops.current();
+
+      client.delete(toTableReference(identifier));
+
+      if (purge && lastMetadata != null) {
+        CatalogUtil.dropTableData(ops.io(), lastMetadata);
+      }
+    } catch (NoSuchTableException e) {
+      return false;
+    }
+
+    return true;
+  }
+
+  @Override
+  public void renameTable(TableIdentifier from, TableIdentifier to) {
+    // TODO: Enable once supported by BigQuery API.
+    throw new UnsupportedOperationException("Table rename operation is unsupported.");
+  }
+
+  @Override
+  public void createNamespace(Namespace namespace, Map<String, String> metadata) {
+    Dataset builder = new Dataset();
+    DatasetReference datasetReference = toDatasetReference(namespace);
+    builder.setLocation(this.projectLocation);
+    builder.setDatasetReference(datasetReference);
+    builder.setExternalCatalogDatasetOptions(
+        BigQueryMetastoreUtils.createExternalCatalogDatasetOptions(
+            createDefaultStorageLocationUri(datasetReference.getDatasetId()), metadata));
+
+    client.create(builder);
+  }
+
+  @Override
+  public List<Namespace> listNamespaces() {
+    try {
+      return listNamespaces(Namespace.empty());
+    } catch (NoSuchNamespaceException e) {
+      return ImmutableList.of();
+    }
+  }
+
+  /**
+   * Since this catalog only supports one-level namespaces, it always returns an empty list unless
+   * passed an empty namespace to list all namespaces within the catalog.
+   */
+  @Override
+  public List<Namespace> listNamespaces(Namespace namespace) {
+    List<Datasets> allDatasets = client.list(projectId);
+
+    ImmutableList<Namespace> namespaces =
+        allDatasets.stream().map(this::toNamespace).collect(ImmutableList.toImmutableList());
+    if (namespaces.isEmpty()) {
+      throw new NoSuchNamespaceException("Namespace does not exist: %s", namespace);
+    }
+
+    return namespaces;
+  }
+
+  @Override
+  public boolean dropNamespace(Namespace namespace) {
+    try {
+      client.delete(toDatasetReference(namespace));
+      // We don't delete the data folder for safety, which aligns with Hive Metastore's default
+      // behavior.
+      // We can support database or catalog level config controlling file deletion in the future.
+      return true;
+    } catch (NoSuchNamespaceException e) {
+      return false;
+    }
+  }
+
+  @Override
+  public boolean setProperties(Namespace namespace, Map<String, String> properties) {
+    Dataset dataset = client.load(toDatasetReference(namespace));
+
+    ExternalCatalogDatasetOptions existingOptions = dataset.getExternalCatalogDatasetOptions();
+    Map<String, String> existingParameters =
+        existingOptions != null ? existingOptions.getParameters() : null;
+
+    Map<String, String> newParameters = Maps.newHashMap();
+    if (existingParameters != null) {
+      newParameters.putAll(existingParameters);
+    }
+
+    newParameters.putAll(properties);
+
+    if (Objects.equals(existingParameters, newParameters)) {
+      // No change in parameters detected
+      return false;
+    }
+
+    client.setParameters(toDatasetReference(namespace), properties);
+    return true;
+  }
+
+  @Override
+  public boolean removeProperties(Namespace namespace, Set<String> properties) {
+    client.removeParameters(toDatasetReference(namespace), properties);
+    return true;
+  }
+
+  @Override
+  public Map<String, String> loadNamespaceMetadata(Namespace namespace) {
+    try {
+      return toMetadata(client.load(toDatasetReference(namespace)));
+    } catch (IllegalArgumentException e) {
+      throw new NoSuchNamespaceException("%s", e.getMessage());
+    }
+  }
+
+  @Override
+  public String name() {
+    return catalogName;
+  }
+
+  @Override
+  protected Map<String, String> properties() {
+    return catalogProperties == null ? ImmutableMap.of() : catalogProperties;
+  }
+
+  @Override
+  public void setConf(Configuration conf) {
+    this.conf = new Configuration(conf);
+  }
+
+  @Override
+  public Configuration getConf() {
+    return this.conf;
+  }
+
+  private String createDefaultStorageLocationUri(String dbId) {
+    Preconditions.checkArgument(
+        this.warehouseLocation != null,

Review Comment:
   Done



##########
bigquery/src/main/java/org/apache/iceberg/gcp/bigquery/BigQueryTableOperations.java:
##########
@@ -0,0 +1,296 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.iceberg.gcp.bigquery;
+
+import com.google.api.services.bigquery.model.ExternalCatalogTableOptions;
+import com.google.api.services.bigquery.model.Table;
+import com.google.api.services.bigquery.model.TableReference;
+import java.util.Locale;
+import java.util.Map;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hive.common.StatsSetupConst;
+import org.apache.iceberg.BaseMetastoreOperations;
+import org.apache.iceberg.BaseMetastoreTableOperations;
+import org.apache.iceberg.SnapshotSummary;
+import org.apache.iceberg.TableMetadata;
+import org.apache.iceberg.TableProperties;
+import org.apache.iceberg.exceptions.AlreadyExistsException;
+import org.apache.iceberg.exceptions.CommitFailedException;
+import org.apache.iceberg.exceptions.CommitStateUnknownException;
+import org.apache.iceberg.exceptions.NoSuchTableException;
+import org.apache.iceberg.exceptions.ValidationException;
+import org.apache.iceberg.hadoop.ConfigProperties;
+import org.apache.iceberg.io.FileIO;
+import org.apache.iceberg.relocated.com.google.common.collect.Maps;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/** Handles BigQuery metastore table operations. */
+public final class BigQueryTableOperations extends BaseMetastoreTableOperations {
+
+  private static final Logger LOG = LoggerFactory.getLogger(BigQueryTableOperations.class);
+
+  private static final String TABLE_PROPERTIES_BQ_CONNECTION = "bq_connection";
+
+  private final BigQueryMetastoreClient client;
+  private final FileIO fileIO;
+  private final TableReference tableReference;
+  private final Configuration conf;
+
+  BigQueryTableOperations(
+      BigQueryMetastoreClient client,
+      FileIO fileIO,
+      TableReference tableReference,
+      Configuration conf) {
+    this.client = client;
+    this.fileIO = fileIO;
+    this.tableReference = tableReference;
+    this.conf = conf;
+  }
+
+  // The doRefresh method should provide implementation on how to get the metadata location.
+  @Override
+  public void doRefresh() {
+    // Must default to null.
+    String metadataLocation = null;
+    try {
+      metadataLocation =
+          loadMetadataLocationOrThrow(
+              client.load(this.tableReference).getExternalCatalogTableOptions());
+    } catch (NoSuchTableException e) {
+      if (currentMetadataLocation() != null) {
+        // Re-throws the exception because the table must exist in this case.
+        throw e;
+      }
+    }
+    refreshFromMetadataLocation(metadataLocation);
+  }
+
+  // The doCommit method should provide implementation on how to update with metadata location
+  // atomically
+  @Override
+  public void doCommit(TableMetadata base, TableMetadata metadata) {
+    String newMetadataLocation =
+        base == null && metadata.metadataFileLocation() != null
+            ? metadata.metadataFileLocation()
+            : writeNewMetadata(metadata, currentVersion() + 1);
+    BaseMetastoreOperations.CommitStatus commitStatus =
+        BaseMetastoreOperations.CommitStatus.FAILURE;
+    try {
+      if (base == null) {
+        createTable(newMetadataLocation, metadata);
+      } else {
+        updateTable(base.metadataFileLocation(), newMetadataLocation, metadata);
+      }
+      commitStatus = BaseMetastoreOperations.CommitStatus.SUCCESS;
+    } catch (CommitFailedException | CommitStateUnknownException e) {
+      throw e;
+    } catch (Throwable e) {
+      LOG.error("Exception thrown on commit: ", e);
+      if (e instanceof AlreadyExistsException) {
+        throw e;
+      }
+      commitStatus =
+          BaseMetastoreOperations.CommitStatus.valueOf(
+              checkCommitStatus(newMetadataLocation, metadata).name());
+      if (commitStatus == BaseMetastoreOperations.CommitStatus.FAILURE) {
+        throw new CommitFailedException(e, "Failed to commit");
+      }
+      if (commitStatus == BaseMetastoreOperations.CommitStatus.UNKNOWN) {
+        throw new CommitStateUnknownException(e);
+      }
+    } finally {
+      try {
+        if (commitStatus == BaseMetastoreOperations.CommitStatus.FAILURE) {
+          LOG.warn("Failed to commit updates to table {}", tableName());
+          io().deleteFile(newMetadataLocation);
+        }
+      } catch (RuntimeException e) {
+        LOG.error(
+            "Failed to cleanup metadata file at {} for table {}",
+            newMetadataLocation,
+            tableName(),
+            e);
+      }
+    }
+  }
+
+  @Override
+  public String tableName() {
+    return String.format("%s.%s", tableReference.getDatasetId(), tableReference.getTableId());
+  }
+
+  @Override
+  public FileIO io() {
+    return fileIO;
+  }
+
+  private void createTable(String newMetadataLocation, TableMetadata metadata) {
+    LOG.debug("Creating a new Iceberg table: {}", tableName());
+    Table tableBuilder = makeNewTable(metadata, newMetadataLocation);
+    tableBuilder.setTableReference(this.tableReference);
+    addConnectionIfProvided(tableBuilder, metadata.properties());
+
+    client.create(tableBuilder);

Review Comment:
   BigQuery client will throw exception if namespace does not exists. We dont need to check in here.



##########
bigquery/src/main/java/org/apache/iceberg/gcp/bigquery/BigQueryMetastoreCatalog.java:
##########
@@ -0,0 +1,366 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.iceberg.gcp.bigquery;
+
+import com.google.api.services.bigquery.model.Dataset;
+import com.google.api.services.bigquery.model.DatasetList.Datasets;
+import com.google.api.services.bigquery.model.DatasetReference;
+import com.google.api.services.bigquery.model.ExternalCatalogDatasetOptions;
+import com.google.api.services.bigquery.model.TableReference;
+import com.google.cloud.ServiceOptions;
+import com.google.cloud.bigquery.BigQueryOptions;
+import java.io.IOException;
+import java.io.UncheckedIOException;
+import java.security.GeneralSecurityException;
+import java.util.List;
+import java.util.Locale;
+import java.util.Map;
+import java.util.Objects;
+import java.util.Set;
+import org.apache.hadoop.conf.Configurable;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.iceberg.BaseMetastoreCatalog;
+import org.apache.iceberg.CatalogProperties;
+import org.apache.iceberg.CatalogUtil;
+import org.apache.iceberg.TableMetadata;
+import org.apache.iceberg.TableOperations;
+import org.apache.iceberg.catalog.Namespace;
+import org.apache.iceberg.catalog.SupportsNamespaces;
+import org.apache.iceberg.catalog.TableIdentifier;
+import org.apache.iceberg.exceptions.NoSuchNamespaceException;
+import org.apache.iceberg.exceptions.NoSuchTableException;
+import org.apache.iceberg.io.FileIO;
+import org.apache.iceberg.relocated.com.google.common.annotations.VisibleForTesting;
+import org.apache.iceberg.relocated.com.google.common.base.Preconditions;
+import org.apache.iceberg.relocated.com.google.common.base.Strings;
+import org.apache.iceberg.relocated.com.google.common.collect.ImmutableList;
+import org.apache.iceberg.relocated.com.google.common.collect.ImmutableMap;
+import org.apache.iceberg.relocated.com.google.common.collect.Maps;
+import org.apache.iceberg.util.LocationUtil;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/** Iceberg Bigquery Metastore Catalog implementation. */
+public class BigQueryMetastoreCatalog extends BaseMetastoreCatalog
+    implements SupportsNamespaces, Configurable {
+
+  // User provided properties.
+  public static final String PROJECT_ID = "gcp.bigquery.project-id";
+  public static final String GCP_LOCATION = "gcp.bigquery.location";
+  public static final String LIST_ALL_TABLES = "gcp.bigquery.list-all-tables";
+
+  private static final Logger LOG = LoggerFactory.getLogger(BigQueryMetastoreCatalog.class);
+
+  private static final String DEFAULT_GCP_LOCATION = "us";
+
+  private String catalogName;
+  private Map<String, String> catalogProperties;
+  private FileIO fileIO;
+  private Configuration conf;
+  private String projectId;
+  private String projectLocation;
+  private BigQueryMetastoreClient client;
+  private boolean listAllTables;
+  private String warehouseLocation;
+
+  public BigQueryMetastoreCatalog() {}
+
+  @Override
+  public void initialize(String name, Map<String, String> properties) {
+    Preconditions.checkArgument(
+        properties.containsKey(PROJECT_ID),
+        "Invalid GCP project: %s must be specified",
+        PROJECT_ID);
+
+    this.projectId = properties.get(PROJECT_ID);
+    this.projectLocation = properties.getOrDefault(GCP_LOCATION, DEFAULT_GCP_LOCATION);
+
+    BigQueryOptions options =
+        BigQueryOptions.newBuilder()
+            .setProjectId(projectId)
+            .setLocation(projectLocation)
+            .setRetrySettings(ServiceOptions.getDefaultRetrySettings())
+            .build();
+
+    try {
+      client = new BigQueryMetastoreClientImpl(options);
+    } catch (IOException e) {
+      throw new UncheckedIOException("Creating BigQuery client failed", e);
+    } catch (GeneralSecurityException e) {
+      throw new RuntimeException("Creating BigQuery client failed due to a security issue", e);
+    }
+
+    initialize(name, properties, projectId, projectLocation, client);
+  }
+
+  @VisibleForTesting
+  void initialize(
+      String name,
+      Map<String, String> properties,
+      String initialProjectId,
+      String initialLocation,
+      BigQueryMetastoreClient bigQueryMetaStoreClient) {
+    Preconditions.checkArgument(bigQueryMetaStoreClient != null, "Invalid BigQuery client: null");
+    this.catalogName = name;
+    this.catalogProperties = ImmutableMap.copyOf(properties);
+    this.projectId = initialProjectId;
+    this.projectLocation = initialLocation;
+    this.client = bigQueryMetaStoreClient;
+
+    if (this.conf == null) {
+      LOG.warn("No configuration was set, using the default environment Configuration");
+      this.conf = new Configuration();
+    }
+
+    LOG.info("Using BigQuery Metastore Iceberg Catalog: {}", name);
+
+    if (properties.containsKey(CatalogProperties.WAREHOUSE_LOCATION)) {
+      this.warehouseLocation =
+          LocationUtil.stripTrailingSlash(properties.get(CatalogProperties.WAREHOUSE_LOCATION));
+    }
+
+    this.fileIO =
+        CatalogUtil.loadFileIO(
+            properties.getOrDefault(
+                CatalogProperties.FILE_IO_IMPL, "org.apache.iceberg.io.ResolvingFileIO"),
+            properties,
+            conf);
+
+    this.listAllTables = Boolean.parseBoolean(properties.getOrDefault(LIST_ALL_TABLES, "true"));
+  }
+
+  @Override
+  protected TableOperations newTableOps(TableIdentifier identifier) {
+    return new BigQueryTableOperations(client, fileIO, toTableReference(identifier), conf);
+  }
+
+  @Override
+  protected String defaultWarehouseLocation(TableIdentifier identifier) {
+    String locationUri = null;
+    DatasetReference datasetReference = toDatasetReference(identifier.namespace());
+    Dataset dataset = client.load(datasetReference);
+    if (dataset != null && dataset.getExternalCatalogDatasetOptions() != null) {
+      locationUri = dataset.getExternalCatalogDatasetOptions().getDefaultStorageLocationUri();
+    }
+
+    return String.format(
+        "%s/%s",
+        Strings.isNullOrEmpty(locationUri)
+            ? createDefaultStorageLocationUri(datasetReference.getDatasetId())
+            : LocationUtil.stripTrailingSlash(locationUri),
+        identifier.name());
+  }
+
+  @Override
+  public List<TableIdentifier> listTables(Namespace namespace) {
+    validateNamespace(namespace);
+
+    return client.list(toDatasetReference(namespace), listAllTables).stream()
+        .map(
+            table -> TableIdentifier.of(namespace.level(0), table.getTableReference().getTableId()))
+        .collect(ImmutableList.toImmutableList());
+  }
+
+  @Override
+  public boolean dropTable(TableIdentifier identifier, boolean purge) {
+    try {
+      TableOperations ops = newTableOps(identifier);
+      TableMetadata lastMetadata = ops.current();
+
+      client.delete(toTableReference(identifier));
+
+      if (purge && lastMetadata != null) {
+        CatalogUtil.dropTableData(ops.io(), lastMetadata);
+      }
+    } catch (NoSuchTableException e) {
+      return false;
+    }
+
+    return true;
+  }
+
+  @Override
+  public void renameTable(TableIdentifier from, TableIdentifier to) {
+    // TODO: Enable once supported by BigQuery API.
+    throw new UnsupportedOperationException("Table rename operation is unsupported.");
+  }
+
+  @Override
+  public void createNamespace(Namespace namespace, Map<String, String> metadata) {
+    Dataset builder = new Dataset();
+    DatasetReference datasetReference = toDatasetReference(namespace);
+    builder.setLocation(this.projectLocation);
+    builder.setDatasetReference(datasetReference);
+    builder.setExternalCatalogDatasetOptions(
+        BigQueryMetastoreUtils.createExternalCatalogDatasetOptions(
+            createDefaultStorageLocationUri(datasetReference.getDatasetId()), metadata));
+
+    client.create(builder);
+  }
+
+  @Override
+  public List<Namespace> listNamespaces() {
+    try {
+      return listNamespaces(Namespace.empty());
+    } catch (NoSuchNamespaceException e) {
+      return ImmutableList.of();
+    }
+  }
+
+  /**
+   * Since this catalog only supports one-level namespaces, it always returns an empty list unless
+   * passed an empty namespace to list all namespaces within the catalog.
+   */
+  @Override
+  public List<Namespace> listNamespaces(Namespace namespace) {
+    List<Datasets> allDatasets = client.list(projectId);
+
+    ImmutableList<Namespace> namespaces =
+        allDatasets.stream().map(this::toNamespace).collect(ImmutableList.toImmutableList());
+    if (namespaces.isEmpty()) {
+      throw new NoSuchNamespaceException("Namespace does not exist: %s", namespace);
+    }
+
+    return namespaces;
+  }
+
+  @Override
+  public boolean dropNamespace(Namespace namespace) {
+    try {
+      client.delete(toDatasetReference(namespace));

Review Comment:
   it is passing this test: https://github.com/apache/iceberg/blob/7f3f450bbddf55bb383ff1409d6d0ca4557c9ffc/core/src/test/java/org/apache/iceberg/catalog/CatalogTests.java#L392



##########
bigquery/src/main/java/org/apache/iceberg/gcp/bigquery/BigQueryMetastoreUtils.java:
##########
@@ -0,0 +1,95 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.iceberg.gcp.bigquery;
+
+import com.google.api.services.bigquery.model.ExternalCatalogDatasetOptions;
+import com.google.api.services.bigquery.model.ExternalCatalogTableOptions;
+import com.google.api.services.bigquery.model.SerDeInfo;
+import com.google.api.services.bigquery.model.StorageDescriptor;
+import java.util.Map;
+
+/** Shared utilities for BigQuery Metastore specific functions and constants. */
+public final class BigQueryMetastoreUtils {

Review Comment:
   Done



##########
bigquery/src/main/java/org/apache/iceberg/gcp/bigquery/BigQueryTableOperations.java:
##########
@@ -0,0 +1,296 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.iceberg.gcp.bigquery;
+
+import com.google.api.services.bigquery.model.ExternalCatalogTableOptions;
+import com.google.api.services.bigquery.model.Table;
+import com.google.api.services.bigquery.model.TableReference;
+import java.util.Locale;
+import java.util.Map;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hive.common.StatsSetupConst;
+import org.apache.iceberg.BaseMetastoreOperations;
+import org.apache.iceberg.BaseMetastoreTableOperations;
+import org.apache.iceberg.SnapshotSummary;
+import org.apache.iceberg.TableMetadata;
+import org.apache.iceberg.TableProperties;
+import org.apache.iceberg.exceptions.AlreadyExistsException;
+import org.apache.iceberg.exceptions.CommitFailedException;
+import org.apache.iceberg.exceptions.CommitStateUnknownException;
+import org.apache.iceberg.exceptions.NoSuchTableException;
+import org.apache.iceberg.exceptions.ValidationException;
+import org.apache.iceberg.hadoop.ConfigProperties;
+import org.apache.iceberg.io.FileIO;
+import org.apache.iceberg.relocated.com.google.common.collect.Maps;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/** Handles BigQuery metastore table operations. */
+public final class BigQueryTableOperations extends BaseMetastoreTableOperations {
+
+  private static final Logger LOG = LoggerFactory.getLogger(BigQueryTableOperations.class);
+
+  private static final String TABLE_PROPERTIES_BQ_CONNECTION = "bq_connection";
+
+  private final BigQueryMetastoreClient client;
+  private final FileIO fileIO;
+  private final TableReference tableReference;
+  private final Configuration conf;
+
+  BigQueryTableOperations(
+      BigQueryMetastoreClient client,
+      FileIO fileIO,
+      TableReference tableReference,
+      Configuration conf) {
+    this.client = client;
+    this.fileIO = fileIO;
+    this.tableReference = tableReference;
+    this.conf = conf;
+  }
+
+  // The doRefresh method should provide implementation on how to get the metadata location.
+  @Override
+  public void doRefresh() {
+    // Must default to null.
+    String metadataLocation = null;
+    try {
+      metadataLocation =
+          loadMetadataLocationOrThrow(
+              client.load(this.tableReference).getExternalCatalogTableOptions());
+    } catch (NoSuchTableException e) {
+      if (currentMetadataLocation() != null) {
+        // Re-throws the exception because the table must exist in this case.
+        throw e;
+      }
+    }
+    refreshFromMetadataLocation(metadataLocation);
+  }
+
+  // The doCommit method should provide implementation on how to update with metadata location
+  // atomically
+  @Override
+  public void doCommit(TableMetadata base, TableMetadata metadata) {
+    String newMetadataLocation =
+        base == null && metadata.metadataFileLocation() != null
+            ? metadata.metadataFileLocation()
+            : writeNewMetadata(metadata, currentVersion() + 1);
+    BaseMetastoreOperations.CommitStatus commitStatus =
+        BaseMetastoreOperations.CommitStatus.FAILURE;
+    try {
+      if (base == null) {
+        createTable(newMetadataLocation, metadata);
+      } else {
+        updateTable(base.metadataFileLocation(), newMetadataLocation, metadata);
+      }
+      commitStatus = BaseMetastoreOperations.CommitStatus.SUCCESS;
+    } catch (CommitFailedException | CommitStateUnknownException e) {
+      throw e;
+    } catch (Throwable e) {
+      LOG.error("Exception thrown on commit: ", e);
+      if (e instanceof AlreadyExistsException) {
+        throw e;
+      }
+      commitStatus =
+          BaseMetastoreOperations.CommitStatus.valueOf(
+              checkCommitStatus(newMetadataLocation, metadata).name());
+      if (commitStatus == BaseMetastoreOperations.CommitStatus.FAILURE) {
+        throw new CommitFailedException(e, "Failed to commit");
+      }
+      if (commitStatus == BaseMetastoreOperations.CommitStatus.UNKNOWN) {
+        throw new CommitStateUnknownException(e);
+      }
+    } finally {
+      try {
+        if (commitStatus == BaseMetastoreOperations.CommitStatus.FAILURE) {
+          LOG.warn("Failed to commit updates to table {}", tableName());
+          io().deleteFile(newMetadataLocation);
+        }
+      } catch (RuntimeException e) {
+        LOG.error(
+            "Failed to cleanup metadata file at {} for table {}",
+            newMetadataLocation,
+            tableName(),
+            e);
+      }
+    }
+  }
+
+  @Override
+  public String tableName() {
+    return String.format("%s.%s", tableReference.getDatasetId(), tableReference.getTableId());
+  }
+
+  @Override
+  public FileIO io() {
+    return fileIO;
+  }
+
+  private void createTable(String newMetadataLocation, TableMetadata metadata) {
+    LOG.debug("Creating a new Iceberg table: {}", tableName());
+    Table tableBuilder = makeNewTable(metadata, newMetadataLocation);
+    tableBuilder.setTableReference(this.tableReference);
+    addConnectionIfProvided(tableBuilder, metadata.properties());
+
+    client.create(tableBuilder);
+  }
+
+  private void addConnectionIfProvided(Table tableBuilder, Map<String, String> metadataProperties) {
+    if (metadataProperties.containsKey(TABLE_PROPERTIES_BQ_CONNECTION)) {
+      tableBuilder
+          .getExternalCatalogTableOptions()
+          .setConnectionId(metadataProperties.get(TABLE_PROPERTIES_BQ_CONNECTION));
+    }
+  }
+
+  /** Update table properties with concurrent update detection using etag. */
+  private void updateTable(
+      String oldMetadataLocation, String newMetadataLocation, TableMetadata metadata) {
+    Table table = client.load(this.tableReference);

Review Comment:
   Done



##########
bigquery/src/main/java/org/apache/iceberg/gcp/bigquery/BigQueryTableOperations.java:
##########
@@ -0,0 +1,296 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.iceberg.gcp.bigquery;
+
+import com.google.api.services.bigquery.model.ExternalCatalogTableOptions;
+import com.google.api.services.bigquery.model.Table;
+import com.google.api.services.bigquery.model.TableReference;
+import java.util.Locale;
+import java.util.Map;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hive.common.StatsSetupConst;
+import org.apache.iceberg.BaseMetastoreOperations;
+import org.apache.iceberg.BaseMetastoreTableOperations;
+import org.apache.iceberg.SnapshotSummary;
+import org.apache.iceberg.TableMetadata;
+import org.apache.iceberg.TableProperties;
+import org.apache.iceberg.exceptions.AlreadyExistsException;
+import org.apache.iceberg.exceptions.CommitFailedException;
+import org.apache.iceberg.exceptions.CommitStateUnknownException;
+import org.apache.iceberg.exceptions.NoSuchTableException;
+import org.apache.iceberg.exceptions.ValidationException;
+import org.apache.iceberg.hadoop.ConfigProperties;
+import org.apache.iceberg.io.FileIO;
+import org.apache.iceberg.relocated.com.google.common.collect.Maps;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/** Handles BigQuery metastore table operations. */
+public final class BigQueryTableOperations extends BaseMetastoreTableOperations {
+
+  private static final Logger LOG = LoggerFactory.getLogger(BigQueryTableOperations.class);
+
+  private static final String TABLE_PROPERTIES_BQ_CONNECTION = "bq_connection";
+
+  private final BigQueryMetastoreClient client;
+  private final FileIO fileIO;
+  private final TableReference tableReference;
+  private final Configuration conf;
+
+  BigQueryTableOperations(
+      BigQueryMetastoreClient client,
+      FileIO fileIO,
+      TableReference tableReference,
+      Configuration conf) {
+    this.client = client;
+    this.fileIO = fileIO;
+    this.tableReference = tableReference;
+    this.conf = conf;
+  }
+
+  // The doRefresh method should provide implementation on how to get the metadata location.
+  @Override
+  public void doRefresh() {
+    // Must default to null.
+    String metadataLocation = null;
+    try {
+      metadataLocation =
+          loadMetadataLocationOrThrow(
+              client.load(this.tableReference).getExternalCatalogTableOptions());

Review Comment:
   Done



##########
bigquery/src/main/java/org/apache/iceberg/gcp/bigquery/BigQueryTableOperations.java:
##########
@@ -0,0 +1,296 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.iceberg.gcp.bigquery;
+
+import com.google.api.services.bigquery.model.ExternalCatalogTableOptions;
+import com.google.api.services.bigquery.model.Table;
+import com.google.api.services.bigquery.model.TableReference;
+import java.util.Locale;
+import java.util.Map;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hive.common.StatsSetupConst;
+import org.apache.iceberg.BaseMetastoreOperations;
+import org.apache.iceberg.BaseMetastoreTableOperations;
+import org.apache.iceberg.SnapshotSummary;
+import org.apache.iceberg.TableMetadata;
+import org.apache.iceberg.TableProperties;
+import org.apache.iceberg.exceptions.AlreadyExistsException;
+import org.apache.iceberg.exceptions.CommitFailedException;
+import org.apache.iceberg.exceptions.CommitStateUnknownException;
+import org.apache.iceberg.exceptions.NoSuchTableException;
+import org.apache.iceberg.exceptions.ValidationException;
+import org.apache.iceberg.hadoop.ConfigProperties;
+import org.apache.iceberg.io.FileIO;
+import org.apache.iceberg.relocated.com.google.common.collect.Maps;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/** Handles BigQuery metastore table operations. */
+public final class BigQueryTableOperations extends BaseMetastoreTableOperations {
+
+  private static final Logger LOG = LoggerFactory.getLogger(BigQueryTableOperations.class);
+
+  private static final String TABLE_PROPERTIES_BQ_CONNECTION = "bq_connection";
+
+  private final BigQueryMetastoreClient client;
+  private final FileIO fileIO;
+  private final TableReference tableReference;
+  private final Configuration conf;
+
+  BigQueryTableOperations(
+      BigQueryMetastoreClient client,
+      FileIO fileIO,
+      TableReference tableReference,
+      Configuration conf) {
+    this.client = client;
+    this.fileIO = fileIO;
+    this.tableReference = tableReference;
+    this.conf = conf;
+  }
+
+  // The doRefresh method should provide implementation on how to get the metadata location.
+  @Override
+  public void doRefresh() {
+    // Must default to null.
+    String metadataLocation = null;
+    try {
+      metadataLocation =
+          loadMetadataLocationOrThrow(
+              client.load(this.tableReference).getExternalCatalogTableOptions());
+    } catch (NoSuchTableException e) {
+      if (currentMetadataLocation() != null) {
+        // Re-throws the exception because the table must exist in this case.

Review Comment:
   We have a metadata location. If we receive NSTE, Someone should delete Table from BigQuery or stomething else. Table must exists if someone not modified 



##########
bigquery/src/main/java/org/apache/iceberg/gcp/bigquery/BigQueryTableOperations.java:
##########
@@ -0,0 +1,296 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.iceberg.gcp.bigquery;
+
+import com.google.api.services.bigquery.model.ExternalCatalogTableOptions;
+import com.google.api.services.bigquery.model.Table;
+import com.google.api.services.bigquery.model.TableReference;
+import java.util.Locale;
+import java.util.Map;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hive.common.StatsSetupConst;
+import org.apache.iceberg.BaseMetastoreOperations;
+import org.apache.iceberg.BaseMetastoreTableOperations;
+import org.apache.iceberg.SnapshotSummary;
+import org.apache.iceberg.TableMetadata;
+import org.apache.iceberg.TableProperties;
+import org.apache.iceberg.exceptions.AlreadyExistsException;
+import org.apache.iceberg.exceptions.CommitFailedException;
+import org.apache.iceberg.exceptions.CommitStateUnknownException;
+import org.apache.iceberg.exceptions.NoSuchTableException;
+import org.apache.iceberg.exceptions.ValidationException;
+import org.apache.iceberg.hadoop.ConfigProperties;
+import org.apache.iceberg.io.FileIO;
+import org.apache.iceberg.relocated.com.google.common.collect.Maps;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/** Handles BigQuery metastore table operations. */
+public final class BigQueryTableOperations extends BaseMetastoreTableOperations {
+
+  private static final Logger LOG = LoggerFactory.getLogger(BigQueryTableOperations.class);
+
+  private static final String TABLE_PROPERTIES_BQ_CONNECTION = "bq_connection";
+
+  private final BigQueryMetastoreClient client;
+  private final FileIO fileIO;
+  private final TableReference tableReference;
+  private final Configuration conf;
+
+  BigQueryTableOperations(
+      BigQueryMetastoreClient client,
+      FileIO fileIO,
+      TableReference tableReference,
+      Configuration conf) {
+    this.client = client;
+    this.fileIO = fileIO;
+    this.tableReference = tableReference;
+    this.conf = conf;
+  }
+
+  // The doRefresh method should provide implementation on how to get the metadata location.
+  @Override
+  public void doRefresh() {
+    // Must default to null.
+    String metadataLocation = null;
+    try {
+      metadataLocation =
+          loadMetadataLocationOrThrow(
+              client.load(this.tableReference).getExternalCatalogTableOptions());
+    } catch (NoSuchTableException e) {
+      if (currentMetadataLocation() != null) {
+        // Re-throws the exception because the table must exist in this case.
+        throw e;
+      }
+    }
+    refreshFromMetadataLocation(metadataLocation);
+  }
+
+  // The doCommit method should provide implementation on how to update with metadata location
+  // atomically
+  @Override
+  public void doCommit(TableMetadata base, TableMetadata metadata) {
+    String newMetadataLocation =
+        base == null && metadata.metadataFileLocation() != null
+            ? metadata.metadataFileLocation()
+            : writeNewMetadata(metadata, currentVersion() + 1);
+    BaseMetastoreOperations.CommitStatus commitStatus =
+        BaseMetastoreOperations.CommitStatus.FAILURE;
+    try {
+      if (base == null) {
+        createTable(newMetadataLocation, metadata);
+      } else {
+        updateTable(base.metadataFileLocation(), newMetadataLocation, metadata);
+      }
+      commitStatus = BaseMetastoreOperations.CommitStatus.SUCCESS;
+    } catch (CommitFailedException | CommitStateUnknownException e) {
+      throw e;
+    } catch (Throwable e) {
+      LOG.error("Exception thrown on commit: ", e);
+      if (e instanceof AlreadyExistsException) {
+        throw e;
+      }
+      commitStatus =
+          BaseMetastoreOperations.CommitStatus.valueOf(
+              checkCommitStatus(newMetadataLocation, metadata).name());
+      if (commitStatus == BaseMetastoreOperations.CommitStatus.FAILURE) {
+        throw new CommitFailedException(e, "Failed to commit");
+      }
+      if (commitStatus == BaseMetastoreOperations.CommitStatus.UNKNOWN) {
+        throw new CommitStateUnknownException(e);
+      }
+    } finally {
+      try {
+        if (commitStatus == BaseMetastoreOperations.CommitStatus.FAILURE) {
+          LOG.warn("Failed to commit updates to table {}", tableName());
+          io().deleteFile(newMetadataLocation);
+        }
+      } catch (RuntimeException e) {
+        LOG.error(
+            "Failed to cleanup metadata file at {} for table {}",
+            newMetadataLocation,
+            tableName(),
+            e);
+      }
+    }
+  }
+
+  @Override
+  public String tableName() {
+    return String.format("%s.%s", tableReference.getDatasetId(), tableReference.getTableId());
+  }
+
+  @Override
+  public FileIO io() {
+    return fileIO;
+  }
+
+  private void createTable(String newMetadataLocation, TableMetadata metadata) {
+    LOG.debug("Creating a new Iceberg table: {}", tableName());
+    Table tableBuilder = makeNewTable(metadata, newMetadataLocation);
+    tableBuilder.setTableReference(this.tableReference);

Review Comment:
   Done



##########
bigquery/src/main/java/org/apache/iceberg/gcp/bigquery/BigQueryMetastoreCatalog.java:
##########
@@ -0,0 +1,366 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.iceberg.gcp.bigquery;
+
+import com.google.api.services.bigquery.model.Dataset;
+import com.google.api.services.bigquery.model.DatasetList.Datasets;
+import com.google.api.services.bigquery.model.DatasetReference;
+import com.google.api.services.bigquery.model.ExternalCatalogDatasetOptions;
+import com.google.api.services.bigquery.model.TableReference;
+import com.google.cloud.ServiceOptions;
+import com.google.cloud.bigquery.BigQueryOptions;
+import java.io.IOException;
+import java.io.UncheckedIOException;
+import java.security.GeneralSecurityException;
+import java.util.List;
+import java.util.Locale;
+import java.util.Map;
+import java.util.Objects;
+import java.util.Set;
+import org.apache.hadoop.conf.Configurable;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.iceberg.BaseMetastoreCatalog;
+import org.apache.iceberg.CatalogProperties;
+import org.apache.iceberg.CatalogUtil;
+import org.apache.iceberg.TableMetadata;
+import org.apache.iceberg.TableOperations;
+import org.apache.iceberg.catalog.Namespace;
+import org.apache.iceberg.catalog.SupportsNamespaces;
+import org.apache.iceberg.catalog.TableIdentifier;
+import org.apache.iceberg.exceptions.NoSuchNamespaceException;
+import org.apache.iceberg.exceptions.NoSuchTableException;
+import org.apache.iceberg.io.FileIO;
+import org.apache.iceberg.relocated.com.google.common.annotations.VisibleForTesting;
+import org.apache.iceberg.relocated.com.google.common.base.Preconditions;
+import org.apache.iceberg.relocated.com.google.common.base.Strings;
+import org.apache.iceberg.relocated.com.google.common.collect.ImmutableList;
+import org.apache.iceberg.relocated.com.google.common.collect.ImmutableMap;
+import org.apache.iceberg.relocated.com.google.common.collect.Maps;
+import org.apache.iceberg.util.LocationUtil;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/** Iceberg Bigquery Metastore Catalog implementation. */
+public class BigQueryMetastoreCatalog extends BaseMetastoreCatalog
+    implements SupportsNamespaces, Configurable {
+
+  // User provided properties.
+  public static final String PROJECT_ID = "gcp.bigquery.project-id";
+  public static final String GCP_LOCATION = "gcp.bigquery.location";
+  public static final String LIST_ALL_TABLES = "gcp.bigquery.list-all-tables";
+
+  private static final Logger LOG = LoggerFactory.getLogger(BigQueryMetastoreCatalog.class);
+
+  private static final String DEFAULT_GCP_LOCATION = "us";
+
+  private String catalogName;
+  private Map<String, String> catalogProperties;
+  private FileIO fileIO;
+  private Configuration conf;
+  private String projectId;
+  private String projectLocation;
+  private BigQueryMetastoreClient client;
+  private boolean listAllTables;
+  private String warehouseLocation;
+
+  public BigQueryMetastoreCatalog() {}
+
+  @Override
+  public void initialize(String name, Map<String, String> properties) {
+    Preconditions.checkArgument(
+        properties.containsKey(PROJECT_ID),
+        "Invalid GCP project: %s must be specified",
+        PROJECT_ID);
+
+    this.projectId = properties.get(PROJECT_ID);
+    this.projectLocation = properties.getOrDefault(GCP_LOCATION, DEFAULT_GCP_LOCATION);
+
+    BigQueryOptions options =
+        BigQueryOptions.newBuilder()
+            .setProjectId(projectId)
+            .setLocation(projectLocation)
+            .setRetrySettings(ServiceOptions.getDefaultRetrySettings())
+            .build();
+
+    try {
+      client = new BigQueryMetastoreClientImpl(options);
+    } catch (IOException e) {
+      throw new UncheckedIOException("Creating BigQuery client failed", e);
+    } catch (GeneralSecurityException e) {
+      throw new RuntimeException("Creating BigQuery client failed due to a security issue", e);
+    }
+
+    initialize(name, properties, projectId, projectLocation, client);
+  }
+
+  @VisibleForTesting
+  void initialize(
+      String name,
+      Map<String, String> properties,
+      String initialProjectId,
+      String initialLocation,
+      BigQueryMetastoreClient bigQueryMetaStoreClient) {
+    Preconditions.checkArgument(bigQueryMetaStoreClient != null, "Invalid BigQuery client: null");
+    this.catalogName = name;
+    this.catalogProperties = ImmutableMap.copyOf(properties);
+    this.projectId = initialProjectId;
+    this.projectLocation = initialLocation;
+    this.client = bigQueryMetaStoreClient;
+
+    if (this.conf == null) {
+      LOG.warn("No configuration was set, using the default environment Configuration");
+      this.conf = new Configuration();
+    }
+
+    LOG.info("Using BigQuery Metastore Iceberg Catalog: {}", name);
+
+    if (properties.containsKey(CatalogProperties.WAREHOUSE_LOCATION)) {
+      this.warehouseLocation =
+          LocationUtil.stripTrailingSlash(properties.get(CatalogProperties.WAREHOUSE_LOCATION));
+    }
+
+    this.fileIO =
+        CatalogUtil.loadFileIO(
+            properties.getOrDefault(
+                CatalogProperties.FILE_IO_IMPL, "org.apache.iceberg.io.ResolvingFileIO"),
+            properties,
+            conf);
+
+    this.listAllTables = Boolean.parseBoolean(properties.getOrDefault(LIST_ALL_TABLES, "true"));
+  }
+
+  @Override
+  protected TableOperations newTableOps(TableIdentifier identifier) {
+    return new BigQueryTableOperations(client, fileIO, toTableReference(identifier), conf);
+  }
+
+  @Override
+  protected String defaultWarehouseLocation(TableIdentifier identifier) {
+    String locationUri = null;
+    DatasetReference datasetReference = toDatasetReference(identifier.namespace());
+    Dataset dataset = client.load(datasetReference);
+    if (dataset != null && dataset.getExternalCatalogDatasetOptions() != null) {
+      locationUri = dataset.getExternalCatalogDatasetOptions().getDefaultStorageLocationUri();
+    }
+
+    return String.format(
+        "%s/%s",
+        Strings.isNullOrEmpty(locationUri)
+            ? createDefaultStorageLocationUri(datasetReference.getDatasetId())
+            : LocationUtil.stripTrailingSlash(locationUri),
+        identifier.name());
+  }
+
+  @Override
+  public List<TableIdentifier> listTables(Namespace namespace) {
+    validateNamespace(namespace);
+
+    return client.list(toDatasetReference(namespace), listAllTables).stream()
+        .map(
+            table -> TableIdentifier.of(namespace.level(0), table.getTableReference().getTableId()))
+        .collect(ImmutableList.toImmutableList());
+  }
+
+  @Override
+  public boolean dropTable(TableIdentifier identifier, boolean purge) {
+    try {
+      TableOperations ops = newTableOps(identifier);
+      TableMetadata lastMetadata = ops.current();
+
+      client.delete(toTableReference(identifier));
+
+      if (purge && lastMetadata != null) {
+        CatalogUtil.dropTableData(ops.io(), lastMetadata);
+      }
+    } catch (NoSuchTableException e) {
+      return false;
+    }
+
+    return true;
+  }
+
+  @Override
+  public void renameTable(TableIdentifier from, TableIdentifier to) {
+    // TODO: Enable once supported by BigQuery API.
+    throw new UnsupportedOperationException("Table rename operation is unsupported.");
+  }
+
+  @Override
+  public void createNamespace(Namespace namespace, Map<String, String> metadata) {
+    Dataset builder = new Dataset();
+    DatasetReference datasetReference = toDatasetReference(namespace);
+    builder.setLocation(this.projectLocation);
+    builder.setDatasetReference(datasetReference);
+    builder.setExternalCatalogDatasetOptions(
+        BigQueryMetastoreUtils.createExternalCatalogDatasetOptions(
+            createDefaultStorageLocationUri(datasetReference.getDatasetId()), metadata));
+
+    client.create(builder);
+  }
+
+  @Override
+  public List<Namespace> listNamespaces() {
+    try {
+      return listNamespaces(Namespace.empty());
+    } catch (NoSuchNamespaceException e) {
+      return ImmutableList.of();
+    }
+  }
+
+  /**
+   * Since this catalog only supports one-level namespaces, it always returns an empty list unless
+   * passed an empty namespace to list all namespaces within the catalog.
+   */
+  @Override
+  public List<Namespace> listNamespaces(Namespace namespace) {
+    List<Datasets> allDatasets = client.list(projectId);
+
+    ImmutableList<Namespace> namespaces =
+        allDatasets.stream().map(this::toNamespace).collect(ImmutableList.toImmutableList());
+    if (namespaces.isEmpty()) {
+      throw new NoSuchNamespaceException("Namespace does not exist: %s", namespace);
+    }
+
+    return namespaces;
+  }
+
+  @Override
+  public boolean dropNamespace(Namespace namespace) {
+    try {
+      client.delete(toDatasetReference(namespace));
+      // We don't delete the data folder for safety, which aligns with Hive Metastore's default
+      // behavior.
+      // We can support database or catalog level config controlling file deletion in the future.
+      return true;
+    } catch (NoSuchNamespaceException e) {
+      return false;
+    }
+  }
+
+  @Override
+  public boolean setProperties(Namespace namespace, Map<String, String> properties) {
+    Dataset dataset = client.load(toDatasetReference(namespace));
+
+    ExternalCatalogDatasetOptions existingOptions = dataset.getExternalCatalogDatasetOptions();
+    Map<String, String> existingParameters =
+        existingOptions != null ? existingOptions.getParameters() : null;
+
+    Map<String, String> newParameters = Maps.newHashMap();
+    if (existingParameters != null) {
+      newParameters.putAll(existingParameters);
+    }
+
+    newParameters.putAll(properties);
+
+    if (Objects.equals(existingParameters, newParameters)) {
+      // No change in parameters detected
+      return false;
+    }
+
+    client.setParameters(toDatasetReference(namespace), properties);
+    return true;
+  }
+
+  @Override
+  public boolean removeProperties(Namespace namespace, Set<String> properties) {
+    client.removeParameters(toDatasetReference(namespace), properties);
+    return true;
+  }
+
+  @Override
+  public Map<String, String> loadNamespaceMetadata(Namespace namespace) {
+    try {
+      return toMetadata(client.load(toDatasetReference(namespace)));
+    } catch (IllegalArgumentException e) {
+      throw new NoSuchNamespaceException("%s", e.getMessage());
+    }
+  }
+
+  @Override
+  public String name() {
+    return catalogName;
+  }
+
+  @Override
+  protected Map<String, String> properties() {
+    return catalogProperties == null ? ImmutableMap.of() : catalogProperties;
+  }
+
+  @Override
+  public void setConf(Configuration conf) {
+    this.conf = new Configuration(conf);
+  }
+
+  @Override
+  public Configuration getConf() {
+    return this.conf;

Review Comment:
   Done



##########
bigquery/src/main/java/org/apache/iceberg/gcp/bigquery/BigQueryTableOperations.java:
##########
@@ -0,0 +1,296 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.iceberg.gcp.bigquery;
+
+import com.google.api.services.bigquery.model.ExternalCatalogTableOptions;
+import com.google.api.services.bigquery.model.Table;
+import com.google.api.services.bigquery.model.TableReference;
+import java.util.Locale;
+import java.util.Map;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hive.common.StatsSetupConst;
+import org.apache.iceberg.BaseMetastoreOperations;
+import org.apache.iceberg.BaseMetastoreTableOperations;
+import org.apache.iceberg.SnapshotSummary;
+import org.apache.iceberg.TableMetadata;
+import org.apache.iceberg.TableProperties;
+import org.apache.iceberg.exceptions.AlreadyExistsException;
+import org.apache.iceberg.exceptions.CommitFailedException;
+import org.apache.iceberg.exceptions.CommitStateUnknownException;
+import org.apache.iceberg.exceptions.NoSuchTableException;
+import org.apache.iceberg.exceptions.ValidationException;
+import org.apache.iceberg.hadoop.ConfigProperties;
+import org.apache.iceberg.io.FileIO;
+import org.apache.iceberg.relocated.com.google.common.collect.Maps;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/** Handles BigQuery metastore table operations. */
+public final class BigQueryTableOperations extends BaseMetastoreTableOperations {

Review Comment:
   Done



##########
bigquery/src/main/java/org/apache/iceberg/gcp/bigquery/BigQueryMetastoreCatalog.java:
##########
@@ -0,0 +1,366 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.iceberg.gcp.bigquery;
+
+import com.google.api.services.bigquery.model.Dataset;
+import com.google.api.services.bigquery.model.DatasetList.Datasets;
+import com.google.api.services.bigquery.model.DatasetReference;
+import com.google.api.services.bigquery.model.ExternalCatalogDatasetOptions;
+import com.google.api.services.bigquery.model.TableReference;
+import com.google.cloud.ServiceOptions;
+import com.google.cloud.bigquery.BigQueryOptions;
+import java.io.IOException;
+import java.io.UncheckedIOException;
+import java.security.GeneralSecurityException;
+import java.util.List;
+import java.util.Locale;
+import java.util.Map;
+import java.util.Objects;
+import java.util.Set;
+import org.apache.hadoop.conf.Configurable;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.iceberg.BaseMetastoreCatalog;
+import org.apache.iceberg.CatalogProperties;
+import org.apache.iceberg.CatalogUtil;
+import org.apache.iceberg.TableMetadata;
+import org.apache.iceberg.TableOperations;
+import org.apache.iceberg.catalog.Namespace;
+import org.apache.iceberg.catalog.SupportsNamespaces;
+import org.apache.iceberg.catalog.TableIdentifier;
+import org.apache.iceberg.exceptions.NoSuchNamespaceException;
+import org.apache.iceberg.exceptions.NoSuchTableException;
+import org.apache.iceberg.io.FileIO;
+import org.apache.iceberg.relocated.com.google.common.annotations.VisibleForTesting;
+import org.apache.iceberg.relocated.com.google.common.base.Preconditions;
+import org.apache.iceberg.relocated.com.google.common.base.Strings;
+import org.apache.iceberg.relocated.com.google.common.collect.ImmutableList;
+import org.apache.iceberg.relocated.com.google.common.collect.ImmutableMap;
+import org.apache.iceberg.relocated.com.google.common.collect.Maps;
+import org.apache.iceberg.util.LocationUtil;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/** Iceberg Bigquery Metastore Catalog implementation. */
+public class BigQueryMetastoreCatalog extends BaseMetastoreCatalog
+    implements SupportsNamespaces, Configurable {
+
+  // User provided properties.
+  public static final String PROJECT_ID = "gcp.bigquery.project-id";
+  public static final String GCP_LOCATION = "gcp.bigquery.location";
+  public static final String LIST_ALL_TABLES = "gcp.bigquery.list-all-tables";
+
+  private static final Logger LOG = LoggerFactory.getLogger(BigQueryMetastoreCatalog.class);
+
+  private static final String DEFAULT_GCP_LOCATION = "us";
+
+  private String catalogName;
+  private Map<String, String> catalogProperties;
+  private FileIO fileIO;
+  private Configuration conf;
+  private String projectId;
+  private String projectLocation;
+  private BigQueryMetastoreClient client;
+  private boolean listAllTables;
+  private String warehouseLocation;
+
+  public BigQueryMetastoreCatalog() {}
+
+  @Override
+  public void initialize(String name, Map<String, String> properties) {
+    Preconditions.checkArgument(
+        properties.containsKey(PROJECT_ID),
+        "Invalid GCP project: %s must be specified",
+        PROJECT_ID);
+
+    this.projectId = properties.get(PROJECT_ID);
+    this.projectLocation = properties.getOrDefault(GCP_LOCATION, DEFAULT_GCP_LOCATION);
+
+    BigQueryOptions options =
+        BigQueryOptions.newBuilder()
+            .setProjectId(projectId)
+            .setLocation(projectLocation)
+            .setRetrySettings(ServiceOptions.getDefaultRetrySettings())
+            .build();
+
+    try {
+      client = new BigQueryMetastoreClientImpl(options);
+    } catch (IOException e) {
+      throw new UncheckedIOException("Creating BigQuery client failed", e);
+    } catch (GeneralSecurityException e) {
+      throw new RuntimeException("Creating BigQuery client failed due to a security issue", e);
+    }
+
+    initialize(name, properties, projectId, projectLocation, client);
+  }
+
+  @VisibleForTesting
+  void initialize(
+      String name,
+      Map<String, String> properties,
+      String initialProjectId,
+      String initialLocation,
+      BigQueryMetastoreClient bigQueryMetaStoreClient) {
+    Preconditions.checkArgument(bigQueryMetaStoreClient != null, "Invalid BigQuery client: null");
+    this.catalogName = name;
+    this.catalogProperties = ImmutableMap.copyOf(properties);
+    this.projectId = initialProjectId;
+    this.projectLocation = initialLocation;
+    this.client = bigQueryMetaStoreClient;
+
+    if (this.conf == null) {
+      LOG.warn("No configuration was set, using the default environment Configuration");
+      this.conf = new Configuration();
+    }
+
+    LOG.info("Using BigQuery Metastore Iceberg Catalog: {}", name);
+
+    if (properties.containsKey(CatalogProperties.WAREHOUSE_LOCATION)) {
+      this.warehouseLocation =
+          LocationUtil.stripTrailingSlash(properties.get(CatalogProperties.WAREHOUSE_LOCATION));
+    }
+
+    this.fileIO =
+        CatalogUtil.loadFileIO(
+            properties.getOrDefault(
+                CatalogProperties.FILE_IO_IMPL, "org.apache.iceberg.io.ResolvingFileIO"),
+            properties,
+            conf);
+
+    this.listAllTables = Boolean.parseBoolean(properties.getOrDefault(LIST_ALL_TABLES, "true"));
+  }
+
+  @Override
+  protected TableOperations newTableOps(TableIdentifier identifier) {
+    return new BigQueryTableOperations(client, fileIO, toTableReference(identifier), conf);
+  }
+
+  @Override
+  protected String defaultWarehouseLocation(TableIdentifier identifier) {
+    String locationUri = null;
+    DatasetReference datasetReference = toDatasetReference(identifier.namespace());
+    Dataset dataset = client.load(datasetReference);
+    if (dataset != null && dataset.getExternalCatalogDatasetOptions() != null) {
+      locationUri = dataset.getExternalCatalogDatasetOptions().getDefaultStorageLocationUri();
+    }
+
+    return String.format(
+        "%s/%s",
+        Strings.isNullOrEmpty(locationUri)
+            ? createDefaultStorageLocationUri(datasetReference.getDatasetId())
+            : LocationUtil.stripTrailingSlash(locationUri),
+        identifier.name());
+  }
+
+  @Override
+  public List<TableIdentifier> listTables(Namespace namespace) {
+    validateNamespace(namespace);
+
+    return client.list(toDatasetReference(namespace), listAllTables).stream()
+        .map(
+            table -> TableIdentifier.of(namespace.level(0), table.getTableReference().getTableId()))
+        .collect(ImmutableList.toImmutableList());
+  }
+
+  @Override
+  public boolean dropTable(TableIdentifier identifier, boolean purge) {
+    try {
+      TableOperations ops = newTableOps(identifier);
+      TableMetadata lastMetadata = ops.current();
+
+      client.delete(toTableReference(identifier));
+
+      if (purge && lastMetadata != null) {
+        CatalogUtil.dropTableData(ops.io(), lastMetadata);
+      }
+    } catch (NoSuchTableException e) {
+      return false;
+    }
+
+    return true;
+  }
+
+  @Override
+  public void renameTable(TableIdentifier from, TableIdentifier to) {
+    // TODO: Enable once supported by BigQuery API.
+    throw new UnsupportedOperationException("Table rename operation is unsupported.");
+  }
+
+  @Override
+  public void createNamespace(Namespace namespace, Map<String, String> metadata) {
+    Dataset builder = new Dataset();
+    DatasetReference datasetReference = toDatasetReference(namespace);
+    builder.setLocation(this.projectLocation);
+    builder.setDatasetReference(datasetReference);
+    builder.setExternalCatalogDatasetOptions(
+        BigQueryMetastoreUtils.createExternalCatalogDatasetOptions(
+            createDefaultStorageLocationUri(datasetReference.getDatasetId()), metadata));
+
+    client.create(builder);
+  }
+
+  @Override
+  public List<Namespace> listNamespaces() {
+    try {
+      return listNamespaces(Namespace.empty());
+    } catch (NoSuchNamespaceException e) {
+      return ImmutableList.of();
+    }
+  }
+
+  /**
+   * Since this catalog only supports one-level namespaces, it always returns an empty list unless
+   * passed an empty namespace to list all namespaces within the catalog.
+   */
+  @Override
+  public List<Namespace> listNamespaces(Namespace namespace) {
+    List<Datasets> allDatasets = client.list(projectId);
+
+    ImmutableList<Namespace> namespaces =
+        allDatasets.stream().map(this::toNamespace).collect(ImmutableList.toImmutableList());
+    if (namespaces.isEmpty()) {
+      throw new NoSuchNamespaceException("Namespace does not exist: %s", namespace);
+    }
+
+    return namespaces;
+  }
+
+  @Override
+  public boolean dropNamespace(Namespace namespace) {
+    try {
+      client.delete(toDatasetReference(namespace));
+      // We don't delete the data folder for safety, which aligns with Hive Metastore's default
+      // behavior.
+      // We can support database or catalog level config controlling file deletion in the future.
+      return true;
+    } catch (NoSuchNamespaceException e) {
+      return false;
+    }
+  }
+
+  @Override
+  public boolean setProperties(Namespace namespace, Map<String, String> properties) {
+    Dataset dataset = client.load(toDatasetReference(namespace));
+
+    ExternalCatalogDatasetOptions existingOptions = dataset.getExternalCatalogDatasetOptions();
+    Map<String, String> existingParameters =
+        existingOptions != null ? existingOptions.getParameters() : null;
+
+    Map<String, String> newParameters = Maps.newHashMap();
+    if (existingParameters != null) {
+      newParameters.putAll(existingParameters);
+    }
+
+    newParameters.putAll(properties);
+
+    if (Objects.equals(existingParameters, newParameters)) {
+      // No change in parameters detected
+      return false;
+    }
+
+    client.setParameters(toDatasetReference(namespace), properties);
+    return true;
+  }
+
+  @Override
+  public boolean removeProperties(Namespace namespace, Set<String> properties) {
+    client.removeParameters(toDatasetReference(namespace), properties);
+    return true;
+  }
+
+  @Override
+  public Map<String, String> loadNamespaceMetadata(Namespace namespace) {
+    try {
+      return toMetadata(client.load(toDatasetReference(namespace)));
+    } catch (IllegalArgumentException e) {
+      throw new NoSuchNamespaceException("%s", e.getMessage());
+    }
+  }
+
+  @Override
+  public String name() {
+    return catalogName;
+  }
+
+  @Override
+  protected Map<String, String> properties() {
+    return catalogProperties == null ? ImmutableMap.of() : catalogProperties;
+  }
+
+  @Override
+  public void setConf(Configuration conf) {
+    this.conf = new Configuration(conf);
+  }
+
+  @Override
+  public Configuration getConf() {
+    return this.conf;
+  }
+
+  private String createDefaultStorageLocationUri(String dbId) {
+    Preconditions.checkArgument(
+        this.warehouseLocation != null,
+        String.format(
+            "Invalid data warehouse location: %s not set", CatalogProperties.WAREHOUSE_LOCATION));
+    return String.format("%s/%s.db", LocationUtil.stripTrailingSlash(warehouseLocation), dbId);
+  }
+
+  private Namespace toNamespace(Datasets dataset) {
+    return Namespace.of(dataset.getDatasetReference().getDatasetId());
+  }
+
+  private DatasetReference toDatasetReference(Namespace namespace) {
+    validateNamespace(namespace);
+    return new DatasetReference().setProjectId(projectId).setDatasetId(namespace.level(0));
+  }
+
+  private TableReference toTableReference(TableIdentifier tableIdentifier) {
+    DatasetReference datasetReference = toDatasetReference(tableIdentifier.namespace());
+    return new TableReference()
+        .setProjectId(datasetReference.getProjectId())
+        .setDatasetId(datasetReference.getDatasetId())
+        .setTableId(tableIdentifier.name());
+  }
+
+  private Map<String, String> toMetadata(Dataset dataset) {
+    ExternalCatalogDatasetOptions options = dataset.getExternalCatalogDatasetOptions();
+    Map<String, String> metadata = Maps.newHashMap();
+    if (options != null) {
+      if (options.getParameters() != null) {
+        metadata.putAll(options.getParameters());
+      }
+
+      if (!Strings.isNullOrEmpty(options.getDefaultStorageLocationUri())) {
+        metadata.put("location", options.getDefaultStorageLocationUri());
+      }
+    }
+
+    return metadata;
+  }
+
+  private void validateNamespace(Namespace namespace) {
+    Preconditions.checkArgument(

Review Comment:
   Those check happens in BigQuery Client. We dont need to check that in here. 



-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org

For queries about this service, please contact Infrastructure at:
users@infra.apache.org


---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For additional commands, e-mail: issues-help@iceberg.apache.org


Re: [PR] Catalog: Add BigQuery Metastore Catalog Support [iceberg]

Posted by "talatuyarer (via GitHub)" <gi...@apache.org>.
talatuyarer commented on code in PR #12808:
URL: https://github.com/apache/iceberg/pull/12808#discussion_r2067652720


##########
build.gradle:
##########
@@ -657,6 +657,47 @@ project(':iceberg-delta-lake') {
   }
 }
 
+project(':iceberg-bigquery') {
+  test {
+    useJUnitPlatform()
+  }
+
+  dependencies {
+    implementation project(path: ':iceberg-bundled-guava', configuration: 'shadow')
+    implementation project(':iceberg-gcp')

Review Comment:
   Sorry i accidentally closed this. I used this dependency while testing against BigQuery Prod. I removed 



-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org

For queries about this service, please contact Infrastructure at:
users@infra.apache.org


---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For additional commands, e-mail: issues-help@iceberg.apache.org


Re: [PR] Catalog: Add BigQuery Metastore Catalog Support [iceberg]

Posted by "ebyhr (via GitHub)" <gi...@apache.org>.
ebyhr commented on code in PR #12808:
URL: https://github.com/apache/iceberg/pull/12808#discussion_r2074830051


##########
build.gradle:
##########
@@ -657,6 +657,46 @@ project(':iceberg-delta-lake') {
   }
 }
 
+project(':iceberg-bigquery') {
+  test {
+    useJUnitPlatform()
+  }
+
+  dependencies {
+    api project(':iceberg-api')
+    implementation project(':iceberg-common')
+    implementation project(':iceberg-core')
+
+    implementation project(path: ':iceberg-bundled-guava', configuration: 'shadow')
+
+    implementation platform(libs.google.libraries.bom)
+    compileOnly "com.google.cloud:google-cloud-storage"
+    implementation "com.google.cloud:google-cloud-bigquery"
+    implementation "com.google.cloud:google-cloud-core"
+
+    compileOnly libs.hadoop3.common

Review Comment:
   Note that Hadoop dependency will prevent Trino from adopting BigQuery metastore catalog since we banned those libraries in https://github.com/trinodb/trino/issues/15921. 



##########
build.gradle:
##########
@@ -657,6 +657,46 @@ project(':iceberg-delta-lake') {
   }
 }
 
+project(':iceberg-bigquery') {
+  test {
+    useJUnitPlatform()
+  }
+
+  dependencies {
+    api project(':iceberg-api')
+    implementation project(':iceberg-common')
+    implementation project(':iceberg-core')
+
+    implementation project(path: ':iceberg-bundled-guava', configuration: 'shadow')
+
+    implementation platform(libs.google.libraries.bom)
+    compileOnly "com.google.cloud:google-cloud-storage"
+    implementation "com.google.cloud:google-cloud-bigquery"
+    implementation "com.google.cloud:google-cloud-core"
+
+    compileOnly libs.hadoop3.common
+    compileOnly (libs.hive2.metastore) {

Review Comment:
   We can remove `libs.hive2.metastore` dependency once we inline constants of `org.apache.hadoop.hive.common.StatsSetupConst`, right? Adding this dependency just for constants looks overkill to me. 



-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org

For queries about this service, please contact Infrastructure at:
users@infra.apache.org


---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For additional commands, e-mail: issues-help@iceberg.apache.org


Re: [PR] Catalog: Add BigQuery Metastore Catalog Support [iceberg]

Posted by "nastra (via GitHub)" <gi...@apache.org>.
nastra commented on code in PR #12808:
URL: https://github.com/apache/iceberg/pull/12808#discussion_r2055559412


##########
bigquery/src/main/java/org/apache/iceberg/gcp/bigquery/BigQueryMetastoreClient.java:
##########
@@ -0,0 +1,121 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.iceberg.gcp.bigquery;
+
+import com.google.api.services.bigquery.model.Dataset;
+import com.google.api.services.bigquery.model.DatasetList.Datasets;
+import com.google.api.services.bigquery.model.DatasetReference;
+import com.google.api.services.bigquery.model.Table;
+import com.google.api.services.bigquery.model.TableList.Tables;
+import com.google.api.services.bigquery.model.TableReference;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+
+/**
+ * A client of Google BigQuery Metastore functions over the BigQuery service. Uses the Google
+ * BigQuery API.
+ */
+public interface BigQueryMetastoreClient {
+
+  /**
+   * Creates and returns a new dataset.
+   *
+   * @param dataset the dataset to create
+   */
+  Dataset createDataset(Dataset dataset);
+
+  /**
+   * Returns a dataset.
+   *
+   * @param datasetReference full dataset reference
+   */
+  Dataset getDataset(DatasetReference datasetReference);
+
+  /**
+   * Deletes a dataset.
+   *
+   * @param datasetReference full dataset reference
+   */
+  void deleteDataset(DatasetReference datasetReference);
+
+  /**
+   * Updates parameters of a dataset or adds them if did not exist, leaving already-existing ones
+   * intact.
+   *
+   * @param datasetReference full dataset reference
+   * @param parameters metadata parameters to add
+   * @return dataset after patch
+   */
+  Dataset setDatasetParameters(DatasetReference datasetReference, Map<String, String> parameters);
+
+  /**
+   * Removes given set of keys of parameters of a dataset. Ignores keys that do not exist already.
+   *
+   * @param datasetReference full dataset reference
+   * @param parameters metadata parameter keys to remove
+   * @return dataset after patch
+   */
+  Dataset removeDatasetParameters(DatasetReference datasetReference, Set<String> parameters);
+
+  /**
+   * Lists datasets under a given project
+   *
+   * @param projectId the identifier of the project to list datasets under
+   */
+  List<Datasets> listDatasets(String projectId);
+
+  /**
+   * Creates and returns a new table.
+   *
+   * @param table body of the table to create
+   */
+  Table createTable(Table table);
+
+  /**
+   * Returns a table.
+   *
+   * @param tableReference full table reference
+   */
+  Table getTable(TableReference tableReference);

Review Comment:
   ```suggestion
     Table loadTable(TableReference tableReference);
   ```
   the codebase typically tries to not use `get` prefixes, so `load` might be better here



-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org

For queries about this service, please contact Infrastructure at:
users@infra.apache.org


---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For additional commands, e-mail: issues-help@iceberg.apache.org


Re: [PR] Catalog: Add BigQuery Metastore Catalog Support [iceberg]

Posted by "nastra (via GitHub)" <gi...@apache.org>.
nastra commented on code in PR #12808:
URL: https://github.com/apache/iceberg/pull/12808#discussion_r2055557189


##########
bigquery/src/main/java/org/apache/iceberg/gcp/bigquery/BigQueryMetastoreClient.java:
##########
@@ -0,0 +1,121 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.iceberg.gcp.bigquery;
+
+import com.google.api.services.bigquery.model.Dataset;
+import com.google.api.services.bigquery.model.DatasetList.Datasets;
+import com.google.api.services.bigquery.model.DatasetReference;
+import com.google.api.services.bigquery.model.Table;
+import com.google.api.services.bigquery.model.TableList.Tables;
+import com.google.api.services.bigquery.model.TableReference;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+
+/**
+ * A client of Google BigQuery Metastore functions over the BigQuery service. Uses the Google
+ * BigQuery API.
+ */
+public interface BigQueryMetastoreClient {
+
+  /**
+   * Creates and returns a new dataset.
+   *
+   * @param dataset the dataset to create
+   */
+  Dataset createDataset(Dataset dataset);
+
+  /**
+   * Returns a dataset.
+   *
+   * @param datasetReference full dataset reference
+   */
+  Dataset getDataset(DatasetReference datasetReference);
+
+  /**
+   * Deletes a dataset.
+   *
+   * @param datasetReference full dataset reference
+   */
+  void deleteDataset(DatasetReference datasetReference);

Review Comment:
   same as above, maybe just name it `delete(DatasetReference datasetReference)`?



-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org

For queries about this service, please contact Infrastructure at:
users@infra.apache.org


---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For additional commands, e-mail: issues-help@iceberg.apache.org


Re: [PR] Catalog: Add BigQuery Metastore Catalog Support [iceberg]

Posted by "nastra (via GitHub)" <gi...@apache.org>.
nastra commented on code in PR #12808:
URL: https://github.com/apache/iceberg/pull/12808#discussion_r2055557727


##########
bigquery/src/main/java/org/apache/iceberg/gcp/bigquery/BigQueryMetastoreClient.java:
##########
@@ -0,0 +1,121 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.iceberg.gcp.bigquery;
+
+import com.google.api.services.bigquery.model.Dataset;
+import com.google.api.services.bigquery.model.DatasetList.Datasets;
+import com.google.api.services.bigquery.model.DatasetReference;
+import com.google.api.services.bigquery.model.Table;
+import com.google.api.services.bigquery.model.TableList.Tables;
+import com.google.api.services.bigquery.model.TableReference;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+
+/**
+ * A client of Google BigQuery Metastore functions over the BigQuery service. Uses the Google
+ * BigQuery API.
+ */
+public interface BigQueryMetastoreClient {
+
+  /**
+   * Creates and returns a new dataset.
+   *
+   * @param dataset the dataset to create
+   */
+  Dataset createDataset(Dataset dataset);
+
+  /**
+   * Returns a dataset.
+   *
+   * @param datasetReference full dataset reference
+   */
+  Dataset getDataset(DatasetReference datasetReference);

Review Comment:
   `load(DatasetReference datasetReference)`?



-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org

For queries about this service, please contact Infrastructure at:
users@infra.apache.org


---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For additional commands, e-mail: issues-help@iceberg.apache.org


Re: [PR] Catalog: Add BigQuery Metastore Catalog Support [iceberg]

Posted by "nastra (via GitHub)" <gi...@apache.org>.
nastra commented on code in PR #12808:
URL: https://github.com/apache/iceberg/pull/12808#discussion_r2055570860


##########
bigquery/src/main/java/org/apache/iceberg/gcp/bigquery/BigQueryMetastoreCatalog.java:
##########
@@ -0,0 +1,400 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.iceberg.gcp.bigquery;
+
+import com.google.api.services.bigquery.model.Dataset;
+import com.google.api.services.bigquery.model.DatasetList.Datasets;
+import com.google.api.services.bigquery.model.DatasetReference;
+import com.google.api.services.bigquery.model.ExternalCatalogDatasetOptions;
+import com.google.api.services.bigquery.model.TableReference;
+import com.google.cloud.ServiceOptions;
+import com.google.cloud.bigquery.BigQueryOptions;
+import java.io.IOException;
+import java.security.GeneralSecurityException;
+import java.util.List;
+import java.util.Locale;
+import java.util.Map;
+import java.util.Objects;
+import java.util.Set;
+import org.apache.hadoop.conf.Configurable;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.iceberg.BaseMetastoreCatalog;
+import org.apache.iceberg.CatalogProperties;
+import org.apache.iceberg.CatalogUtil;
+import org.apache.iceberg.TableMetadata;
+import org.apache.iceberg.TableOperations;
+import org.apache.iceberg.catalog.Namespace;
+import org.apache.iceberg.catalog.SupportsNamespaces;
+import org.apache.iceberg.catalog.TableIdentifier;
+import org.apache.iceberg.exceptions.AlreadyExistsException;
+import org.apache.iceberg.exceptions.NoSuchNamespaceException;
+import org.apache.iceberg.exceptions.NoSuchTableException;
+import org.apache.iceberg.exceptions.ServiceFailureException;
+import org.apache.iceberg.exceptions.ValidationException;
+import org.apache.iceberg.io.FileIO;
+import org.apache.iceberg.relocated.com.google.common.annotations.VisibleForTesting;
+import org.apache.iceberg.relocated.com.google.common.base.Preconditions;
+import org.apache.iceberg.relocated.com.google.common.base.Strings;
+import org.apache.iceberg.relocated.com.google.common.collect.ImmutableList;
+import org.apache.iceberg.relocated.com.google.common.collect.ImmutableMap;
+import org.apache.iceberg.relocated.com.google.common.collect.Maps;
+import org.apache.iceberg.util.LocationUtil;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/** Iceberg Bigquery Metastore Catalog implementation. */
+public final class BigQueryMetastoreCatalog extends BaseMetastoreCatalog
+    implements SupportsNamespaces, Configurable {
+
+  // User provided properties.
+  public static final String PROJECT_ID = "gcp-project";
+  public static final String GCP_LOCATION = "gcp-location";
+  public static final String FILTER_UNSUPPORTED_TABLES = "filter-unsupported-tables";
+  public static final String TESTING_ENABLED = "testing-enabled";
+
+  public static final String HIVE_METASTORE_WAREHOUSE_DIR = "hive.metastore.warehouse.dir";
+
+  private static final Logger LOG = LoggerFactory.getLogger(BigQueryMetastoreCatalog.class);
+
+  private static final String DEFAULT_GCP_LOCATION = "us";
+
+  private String catalogName;
+  private Map<String, String> catalogProperties;
+  private FileIO fileIO;
+  private Configuration conf;
+  private String projectId;
+  private String location;
+  private BigQueryMetastoreClient client;
+  private boolean filterUnsupportedTables;
+
+  // Must have a no-arg constructor to be dynamically loaded
+  // initialize(String name, Map<String, String> properties) will be called to complete
+  // initialization
+  public BigQueryMetastoreCatalog() {}
+
+  @Override
+  public void initialize(String inputName, Map<String, String> properties) {
+    Preconditions.checkArgument(
+        properties.containsKey(PROJECT_ID), "GCP project must be specified");
+
+    projectId = properties.get(PROJECT_ID);
+    location = properties.getOrDefault(GCP_LOCATION, DEFAULT_GCP_LOCATION);
+    boolean testingEnabled =

Review Comment:
   TBH I don't think this flag is actually needed. Since you already have defined another `initialize` method that takes `BigQueryMetastoreClient` as a parameter, you can pass the fake client there for testing



-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org

For queries about this service, please contact Infrastructure at:
users@infra.apache.org


---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For additional commands, e-mail: issues-help@iceberg.apache.org


Re: [PR] Catalog: Add BigQuery Metastore Catalog Support [iceberg]

Posted by "nastra (via GitHub)" <gi...@apache.org>.
nastra commented on code in PR #12808:
URL: https://github.com/apache/iceberg/pull/12808#discussion_r2055661860


##########
bigquery/src/main/java/org/apache/iceberg/gcp/bigquery/BigQueryTableOperations.java:
##########
@@ -0,0 +1,297 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.iceberg.gcp.bigquery;
+
+import com.google.api.services.bigquery.model.ExternalCatalogTableOptions;
+import com.google.api.services.bigquery.model.Table;
+import com.google.api.services.bigquery.model.TableReference;
+import java.util.Locale;
+import java.util.Map;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hive.common.StatsSetupConst;
+import org.apache.iceberg.BaseMetastoreOperations;
+import org.apache.iceberg.BaseMetastoreTableOperations;
+import org.apache.iceberg.SnapshotSummary;
+import org.apache.iceberg.TableMetadata;
+import org.apache.iceberg.TableProperties;
+import org.apache.iceberg.exceptions.AlreadyExistsException;
+import org.apache.iceberg.exceptions.CommitFailedException;
+import org.apache.iceberg.exceptions.CommitStateUnknownException;
+import org.apache.iceberg.exceptions.NoSuchTableException;
+import org.apache.iceberg.exceptions.ValidationException;
+import org.apache.iceberg.hadoop.ConfigProperties;
+import org.apache.iceberg.io.FileIO;
+import org.apache.iceberg.relocated.com.google.common.collect.Maps;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/** Handles BigQuery metastore table operations. */
+public final class BigQueryTableOperations extends BaseMetastoreTableOperations {
+
+  private static final Logger LOG = LoggerFactory.getLogger(BigQueryTableOperations.class);
+
+  public static final String TABLE_PROPERTIES_BQ_CONNECTION = "bq_connection";

Review Comment:
   this can be private as it's not used anywhere else



-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org

For queries about this service, please contact Infrastructure at:
users@infra.apache.org


---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For additional commands, e-mail: issues-help@iceberg.apache.org


Re: [PR] Catalog: Add BigQuery Metastore Catalog Support [iceberg]

Posted by "talatuyarer (via GitHub)" <gi...@apache.org>.
talatuyarer commented on code in PR #12808:
URL: https://github.com/apache/iceberg/pull/12808#discussion_r2081048540


##########
build.gradle:
##########
@@ -657,6 +657,46 @@ project(':iceberg-delta-lake') {
   }
 }
 
+project(':iceberg-bigquery') {
+  test {
+    useJUnitPlatform()
+  }
+
+  dependencies {
+    api project(':iceberg-api')
+    implementation project(':iceberg-common')
+    implementation project(':iceberg-core')
+
+    implementation project(path: ':iceberg-bundled-guava', configuration: 'shadow')
+
+    implementation platform(libs.google.libraries.bom)
+    compileOnly "com.google.cloud:google-cloud-storage"
+    implementation "com.google.cloud:google-cloud-bigquery"
+    implementation "com.google.cloud:google-cloud-core"
+
+    compileOnly libs.hadoop3.common
+    compileOnly (libs.hive2.metastore) {

Review Comment:
   Removed Hive dependencies. 



-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org

For queries about this service, please contact Infrastructure at:
users@infra.apache.org


---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For additional commands, e-mail: issues-help@iceberg.apache.org


Re: [PR] Catalog: Add BigQuery Metastore Catalog Support [iceberg]

Posted by "talatuyarer (via GitHub)" <gi...@apache.org>.
talatuyarer commented on code in PR #12808:
URL: https://github.com/apache/iceberg/pull/12808#discussion_r2082102992


##########
bigquery/src/main/java/org/apache/iceberg/gcp/bigquery/BigQueryMetastoreCatalog.java:
##########
@@ -0,0 +1,374 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.iceberg.gcp.bigquery;
+
+import com.google.api.services.bigquery.model.Dataset;
+import com.google.api.services.bigquery.model.DatasetList.Datasets;
+import com.google.api.services.bigquery.model.DatasetReference;
+import com.google.api.services.bigquery.model.ExternalCatalogDatasetOptions;
+import com.google.api.services.bigquery.model.TableReference;
+import com.google.cloud.ServiceOptions;
+import com.google.cloud.bigquery.BigQueryOptions;
+import java.io.IOException;
+import java.io.UncheckedIOException;
+import java.security.GeneralSecurityException;
+import java.util.List;
+import java.util.Locale;
+import java.util.Map;
+import java.util.Objects;
+import java.util.Set;
+import org.apache.iceberg.BaseMetastoreCatalog;
+import org.apache.iceberg.CatalogProperties;
+import org.apache.iceberg.CatalogUtil;
+import org.apache.iceberg.TableMetadata;
+import org.apache.iceberg.TableOperations;
+import org.apache.iceberg.catalog.Namespace;
+import org.apache.iceberg.catalog.SupportsNamespaces;
+import org.apache.iceberg.catalog.TableIdentifier;
+import org.apache.iceberg.exceptions.NoSuchNamespaceException;
+import org.apache.iceberg.exceptions.NoSuchTableException;
+import org.apache.iceberg.hadoop.Configurable;
+import org.apache.iceberg.io.FileIO;
+import org.apache.iceberg.relocated.com.google.common.annotations.VisibleForTesting;
+import org.apache.iceberg.relocated.com.google.common.base.Preconditions;
+import org.apache.iceberg.relocated.com.google.common.base.Strings;
+import org.apache.iceberg.relocated.com.google.common.collect.ImmutableList;
+import org.apache.iceberg.relocated.com.google.common.collect.ImmutableMap;
+import org.apache.iceberg.relocated.com.google.common.collect.Maps;
+import org.apache.iceberg.util.LocationUtil;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/** Iceberg Bigquery Metastore Catalog implementation. */
+public class BigQueryMetastoreCatalog extends BaseMetastoreCatalog
+    implements SupportsNamespaces, Configurable<Object> {
+
+  // User provided properties.
+  public static final String PROJECT_ID = "gcp.bigquery.project-id";
+  public static final String GCP_LOCATION = "gcp.bigquery.location";
+  public static final String LIST_ALL_TABLES = "gcp.bigquery.list-all-tables";
+
+  private static final Logger LOG = LoggerFactory.getLogger(BigQueryMetastoreCatalog.class);
+
+  private static final String DEFAULT_GCP_LOCATION = "us";
+
+  private String catalogName;
+  private Map<String, String> catalogProperties;
+  private FileIO fileIO;
+  private Object conf;
+  private String projectId;
+  private String projectLocation;
+  private BigQueryMetastoreClient client;
+  private boolean listAllTables;
+  private String warehouseLocation;
+
+  public BigQueryMetastoreCatalog() {}
+
+  @Override
+  public void initialize(String name, Map<String, String> properties) {
+    Preconditions.checkArgument(
+        properties.containsKey(PROJECT_ID),
+        "Invalid GCP project: %s must be specified",
+        PROJECT_ID);
+
+    this.projectId = properties.get(PROJECT_ID);
+    this.projectLocation = properties.getOrDefault(GCP_LOCATION, DEFAULT_GCP_LOCATION);
+
+    BigQueryOptions options =
+        BigQueryOptions.newBuilder()
+            .setProjectId(projectId)
+            .setLocation(projectLocation)
+            .setRetrySettings(ServiceOptions.getDefaultRetrySettings())
+            .build();
+
+    try {
+      client = new BigQueryMetastoreClientImpl(options);
+    } catch (IOException e) {
+      throw new UncheckedIOException("Creating BigQuery client failed", e);
+    } catch (GeneralSecurityException e) {
+      throw new RuntimeException("Creating BigQuery client failed due to a security issue", e);
+    }
+
+    initialize(name, properties, projectId, projectLocation, client);
+  }
+
+  @VisibleForTesting
+  void initialize(
+      String name,
+      Map<String, String> properties,
+      String initialProjectId,
+      String initialLocation,
+      BigQueryMetastoreClient bigQueryMetaStoreClient) {
+    Preconditions.checkArgument(bigQueryMetaStoreClient != null, "Invalid BigQuery client: null");
+    this.catalogName = name;
+    this.catalogProperties = ImmutableMap.copyOf(properties);
+    this.projectId = initialProjectId;
+    this.projectLocation = initialLocation;
+    this.client = bigQueryMetaStoreClient;
+
+    LOG.info("Using BigQuery Metastore Iceberg Catalog: {}", name);
+
+    if (properties.containsKey(CatalogProperties.WAREHOUSE_LOCATION)) {
+      this.warehouseLocation =
+          LocationUtil.stripTrailingSlash(properties.get(CatalogProperties.WAREHOUSE_LOCATION));
+    }
+
+    this.fileIO =
+        CatalogUtil.loadFileIO(
+            properties.getOrDefault(
+                CatalogProperties.FILE_IO_IMPL, "org.apache.iceberg.io.ResolvingFileIO"),
+            properties,
+            conf);
+
+    this.listAllTables = Boolean.parseBoolean(properties.getOrDefault(LIST_ALL_TABLES, "true"));
+  }
+
+  @Override
+  protected TableOperations newTableOps(TableIdentifier identifier) {
+    return new BigQueryTableOperations(client, fileIO, toTableReference(identifier));
+  }
+
+  @Override
+  protected String defaultWarehouseLocation(TableIdentifier identifier) {
+    String locationUri = null;
+    DatasetReference datasetReference = toDatasetReference(identifier.namespace());
+    Dataset dataset = client.load(datasetReference);
+    if (dataset != null && dataset.getExternalCatalogDatasetOptions() != null) {
+      locationUri = dataset.getExternalCatalogDatasetOptions().getDefaultStorageLocationUri();
+    }
+
+    return String.format(
+        "%s/%s",
+        Strings.isNullOrEmpty(locationUri)
+            ? createDefaultStorageLocationUri(datasetReference.getDatasetId())
+            : LocationUtil.stripTrailingSlash(locationUri),
+        identifier.name());
+  }
+
+  @Override
+  public List<TableIdentifier> listTables(Namespace namespace) {
+    validateNamespace(namespace);
+
+    return client.list(toDatasetReference(namespace), listAllTables).stream()
+        .map(
+            table -> TableIdentifier.of(namespace.level(0), table.getTableReference().getTableId()))
+        .collect(ImmutableList.toImmutableList());
+  }
+
+  @Override
+  public boolean dropTable(TableIdentifier identifier, boolean purge) {
+    try {
+      TableOperations ops = newTableOps(identifier);
+      TableMetadata lastMetadata = ops.current();
+
+      client.delete(toTableReference(identifier));
+
+      if (purge && lastMetadata != null) {
+        CatalogUtil.dropTableData(ops.io(), lastMetadata);
+      }
+    } catch (NoSuchTableException e) {
+      return false;
+    }
+
+    return true;
+  }
+
+  @Override
+  public void renameTable(TableIdentifier from, TableIdentifier to) {
+    // TODO: Enable once supported by BigQuery API.
+    throw new UnsupportedOperationException("Table rename operation is unsupported.");
+  }
+
+  @Override
+  public void createNamespace(Namespace namespace, Map<String, String> metadata) {
+    Dataset builder = new Dataset();
+    DatasetReference datasetReference = toDatasetReference(namespace);
+    builder.setLocation(this.projectLocation);
+    builder.setDatasetReference(datasetReference);
+    builder.setExternalCatalogDatasetOptions(
+        BigQueryMetastoreUtils.createExternalCatalogDatasetOptions(
+            createDefaultStorageLocationUri(datasetReference.getDatasetId()), metadata));
+
+    client.create(builder);
+  }
+
+  @Override
+  public List<Namespace> listNamespaces() {
+    try {
+      return listNamespaces(Namespace.empty());
+    } catch (NoSuchNamespaceException e) {
+      return ImmutableList.of();
+    }
+  }
+
+  /**
+   * Since this catalog only supports one-level namespaces, it always returns an empty list unless
+   * passed an empty namespace to list all namespaces within the catalog.
+   */
+  @Override
+  public List<Namespace> listNamespaces(Namespace namespace) {
+    if (!namespace.isEmpty()) {
+      throw new NoSuchNamespaceException("Namespace does not exist: %s", namespace);
+    }
+    if (!namespace.isEmpty()) {

Review Comment:
   Thank you. @nastra 



-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org

For queries about this service, please contact Infrastructure at:
users@infra.apache.org


---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For additional commands, e-mail: issues-help@iceberg.apache.org


Re: [PR] Catalog: Add BigQuery Metastore Catalog Support [iceberg]

Posted by "talatuyarer (via GitHub)" <gi...@apache.org>.
talatuyarer commented on code in PR #12808:
URL: https://github.com/apache/iceberg/pull/12808#discussion_r2059698850


##########
bigquery/src/main/java/org/apache/iceberg/gcp/bigquery/BigQueryMetastoreCatalog.java:
##########
@@ -0,0 +1,400 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.iceberg.gcp.bigquery;
+
+import com.google.api.services.bigquery.model.Dataset;
+import com.google.api.services.bigquery.model.DatasetList.Datasets;
+import com.google.api.services.bigquery.model.DatasetReference;
+import com.google.api.services.bigquery.model.ExternalCatalogDatasetOptions;
+import com.google.api.services.bigquery.model.TableReference;
+import com.google.cloud.ServiceOptions;
+import com.google.cloud.bigquery.BigQueryOptions;
+import java.io.IOException;
+import java.security.GeneralSecurityException;
+import java.util.List;
+import java.util.Locale;
+import java.util.Map;
+import java.util.Objects;
+import java.util.Set;
+import org.apache.hadoop.conf.Configurable;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.iceberg.BaseMetastoreCatalog;
+import org.apache.iceberg.CatalogProperties;
+import org.apache.iceberg.CatalogUtil;
+import org.apache.iceberg.TableMetadata;
+import org.apache.iceberg.TableOperations;
+import org.apache.iceberg.catalog.Namespace;
+import org.apache.iceberg.catalog.SupportsNamespaces;
+import org.apache.iceberg.catalog.TableIdentifier;
+import org.apache.iceberg.exceptions.AlreadyExistsException;
+import org.apache.iceberg.exceptions.NoSuchNamespaceException;
+import org.apache.iceberg.exceptions.NoSuchTableException;
+import org.apache.iceberg.exceptions.ServiceFailureException;
+import org.apache.iceberg.exceptions.ValidationException;
+import org.apache.iceberg.io.FileIO;
+import org.apache.iceberg.relocated.com.google.common.annotations.VisibleForTesting;
+import org.apache.iceberg.relocated.com.google.common.base.Preconditions;
+import org.apache.iceberg.relocated.com.google.common.base.Strings;
+import org.apache.iceberg.relocated.com.google.common.collect.ImmutableList;
+import org.apache.iceberg.relocated.com.google.common.collect.ImmutableMap;
+import org.apache.iceberg.relocated.com.google.common.collect.Maps;
+import org.apache.iceberg.util.LocationUtil;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/** Iceberg Bigquery Metastore Catalog implementation. */
+public final class BigQueryMetastoreCatalog extends BaseMetastoreCatalog
+    implements SupportsNamespaces, Configurable {
+
+  // User provided properties.
+  public static final String PROJECT_ID = "gcp-project";
+  public static final String GCP_LOCATION = "gcp-location";
+  public static final String FILTER_UNSUPPORTED_TABLES = "filter-unsupported-tables";
+  public static final String TESTING_ENABLED = "testing-enabled";
+
+  public static final String HIVE_METASTORE_WAREHOUSE_DIR = "hive.metastore.warehouse.dir";
+
+  private static final Logger LOG = LoggerFactory.getLogger(BigQueryMetastoreCatalog.class);
+
+  private static final String DEFAULT_GCP_LOCATION = "us";
+
+  private String catalogName;
+  private Map<String, String> catalogProperties;
+  private FileIO fileIO;
+  private Configuration conf;
+  private String projectId;
+  private String location;
+  private BigQueryMetastoreClient client;
+  private boolean filterUnsupportedTables;
+
+  // Must have a no-arg constructor to be dynamically loaded
+  // initialize(String name, Map<String, String> properties) will be called to complete
+  // initialization
+  public BigQueryMetastoreCatalog() {}
+
+  @Override
+  public void initialize(String inputName, Map<String, String> properties) {
+    Preconditions.checkArgument(
+        properties.containsKey(PROJECT_ID), "GCP project must be specified");
+
+    projectId = properties.get(PROJECT_ID);
+    location = properties.getOrDefault(GCP_LOCATION, DEFAULT_GCP_LOCATION);
+    boolean testingEnabled =

Review Comment:
   I wanted to test via CatalogUtils. Thats why I set this flag. It gives Unit tests to run overall workflow. Can I keep this ? 



-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org

For queries about this service, please contact Infrastructure at:
users@infra.apache.org


---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For additional commands, e-mail: issues-help@iceberg.apache.org


Re: [PR] Catalog: Add BigQuery Metastore Catalog Support [iceberg]

Posted by "nastra (via GitHub)" <gi...@apache.org>.
nastra commented on code in PR #12808:
URL: https://github.com/apache/iceberg/pull/12808#discussion_r2059740604


##########
bigquery/src/test/java/org/apache/iceberg/gcp/bigquery/TestBigQueryCatalog.java:
##########
@@ -0,0 +1,673 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.iceberg.gcp.bigquery;
+
+import static org.apache.iceberg.CatalogUtil.ICEBERG_CATALOG_TYPE;
+import static org.apache.iceberg.CatalogUtil.ICEBERG_CATALOG_TYPE_BIGQUERY;
+import static org.apache.iceberg.gcp.bigquery.BigQueryMetastoreCatalog.PROJECT_ID;
+import static org.apache.iceberg.gcp.bigquery.BigQueryMetastoreCatalog.TESTING_ENABLED;
+import static org.apache.iceberg.types.Types.NestedField.optional;
+import static org.assertj.core.api.Assertions.assertThat;
+import static org.assertj.core.api.Assertions.assertThatThrownBy;
+
+import java.io.File;
+import java.util.List;
+import java.util.Map;
+import org.apache.iceberg.CatalogProperties;
+import org.apache.iceberg.CatalogUtil;
+import org.apache.iceberg.PartitionSpec;
+import org.apache.iceberg.Schema;
+import org.apache.iceberg.SortOrder;
+import org.apache.iceberg.Table;
+import org.apache.iceberg.TableUtil;
+import org.apache.iceberg.Transaction;
+import org.apache.iceberg.catalog.CatalogTests;
+import org.apache.iceberg.catalog.Namespace;
+import org.apache.iceberg.catalog.TableIdentifier;
+import org.apache.iceberg.exceptions.NoSuchNamespaceException;
+import org.apache.iceberg.exceptions.NoSuchTableException;
+import org.apache.iceberg.expressions.Expressions;
+import org.apache.iceberg.relocated.com.google.common.collect.ImmutableMap;
+import org.apache.iceberg.types.Types;
+import org.junit.jupiter.api.AfterEach;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Disabled;
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.io.TempDir;
+import org.junit.jupiter.params.ParameterizedTest;
+import org.junit.jupiter.params.provider.ValueSource;
+
+public class TestBigQueryCatalog extends CatalogTests<BigQueryMetastoreCatalog> {
+
+  private static final String GCP_PROJECT = "project-id";
+  private static final String CATALOG_ID = "catalog-name";
+
+  // another schema that is not the same
+  protected static final Schema OTHER_SCHEMA_OPTIONAL =
+      new Schema(optional(1, "some_id", Types.IntegerType.get()));
+
+  // Schema passed to create tables
+  protected static final Schema SCHEMA_OPTIONAL =
+      new Schema(
+          optional(3, "id", Types.IntegerType.get(), "unique ID 🤪"),
+          optional(4, "data", Types.StringType.get()));
+
+  // This is the actual schema for the table, with column IDs reassigned
+  protected static final Schema REPLACE_SCHEMA_OPTIONAL =
+      new Schema(
+          optional(2, "id", Types.IntegerType.get(), "unique ID 🤪"),
+          optional(3, "data", Types.StringType.get()));
+
+  protected static final PartitionSpec REPLACE_SPEC_OPTIONAL =
+      PartitionSpec.builderFor(REPLACE_SCHEMA_OPTIONAL).bucket("id", 16).withSpecId(1).build();
+
+  protected static final SortOrder REPLACE_WRITE_ORDER_OPTIONAL =
+      SortOrder.builderFor(REPLACE_SCHEMA_OPTIONAL)
+          .asc(Expressions.bucket("id", 16))
+          .asc("id")
+          .build();
+
+  @TempDir private File tempFolder;
+  private BigQueryMetastoreCatalog catalog;
+  private String warehouseLocation;
+
+  @BeforeEach
+  public void before() throws Exception {
+    catalog = initCatalog(CATALOG_ID, ImmutableMap.of());
+  }
+
+  @AfterEach
+  public void after() throws Exception {
+    // Drop all tables in all datasets
+    List<Namespace> namespaces = catalog().listNamespaces(Namespace.empty());
+    for (Namespace namespace : namespaces) {
+      List<TableIdentifier> tables = catalog().listTables(namespace);
+      for (TableIdentifier table : tables) {
+        try {
+          catalog().dropTable(table, true); // drop with purge
+        } catch (NoSuchTableException e) {
+          // Table might be dropped by a previous iteration, ignore
+        }
+      }
+
+      // Drop all datasets (except any initial ones)
+      try {
+        catalog().dropNamespace(namespace);
+      } catch (NoSuchNamespaceException e) {
+        // Namespace might be dropped by a previous iteration, ignore
+      }
+    }
+  }
+
+  @Override
+  protected boolean requiresNamespaceCreate() {
+    return true;
+  }
+
+  @Override
+  protected boolean supportsNamesWithSlashes() {
+    return false;
+  }
+
+  @Override
+  protected boolean supportsNamesWithDot() {
+    return false;
+  }
+
+  @Override
+  public BigQueryMetastoreCatalog catalog() {
+    return catalog;
+  }
+
+  @Override
+  protected BigQueryMetastoreCatalog initCatalog(
+      String catalogName, Map<String, String> additionalProperties) {
+
+    warehouseLocation = tempFolder.toPath().resolve("hive-warehouse").toString();
+
+    Map<String, String> properties =
+        Map.of(
+            ICEBERG_CATALOG_TYPE,
+            ICEBERG_CATALOG_TYPE_BIGQUERY,
+            PROJECT_ID,
+            GCP_PROJECT,
+            CatalogProperties.WAREHOUSE_LOCATION,
+            warehouseLocation,
+            TESTING_ENABLED,
+            "true",
+            CatalogProperties.TABLE_DEFAULT_PREFIX + "default-key1",
+            "catalog-default-key1",
+            CatalogProperties.TABLE_DEFAULT_PREFIX + "default-key2",
+            "catalog-default-key2",
+            CatalogProperties.TABLE_DEFAULT_PREFIX + "override-key3",
+            "catalog-default-key3",
+            CatalogProperties.TABLE_OVERRIDE_PREFIX + "override-key3",
+            "catalog-override-key3",
+            CatalogProperties.TABLE_OVERRIDE_PREFIX + "override-key4",
+            "catalog-override-key4");
+
+    BigQueryMetastoreCatalog tmpCatalog =
+        (BigQueryMetastoreCatalog)
+            CatalogUtil.buildIcebergCatalog(
+                catalogName,
+                ImmutableMap.<String, String>builder()
+                    .putAll(properties)
+                    .putAll(additionalProperties)
+                    .build(),
+                null);
+
+    return tmpCatalog;
+  }
+
+  @Test
+  public void testRenameTable() {
+    if (requiresNamespaceCreate()) {
+      catalog.createNamespace(NS);
+    }
+
+    assertThat(catalog.tableExists(TABLE))
+        .as("Source table should not exist before create")
+        .isFalse();
+
+    catalog.buildTable(TABLE, SCHEMA).create();
+    assertThat(catalog.tableExists(TABLE)).as("Table should exist after create").isTrue();
+
+    assertThat(catalog.tableExists(RENAMED_TABLE))
+        .as("Destination table should not exist before rename")
+        .isFalse();
+
+    assertThatThrownBy(() -> catalog.renameTable(TABLE, RENAMED_TABLE))
+        .isInstanceOf(UnsupportedOperationException.class)
+        .hasMessageContaining("Table rename operation is unsupported");
+
+    catalog.dropTable(TABLE);
+  }
+
+  @Disabled(
+      "BigQuery does not allow adding a REQUIRED column to an existing table schema. "
+          + "When this operation is attempted, an exception gets thrown.")
+  @Test
+  public void testOverrideTablePropertiesReplaceTransaction() {}

Review Comment:
   is this still true? when I removed the `Disabled` flag the test would pass



-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org

For queries about this service, please contact Infrastructure at:
users@infra.apache.org


---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For additional commands, e-mail: issues-help@iceberg.apache.org


Re: [PR] Catalog: Add BigQuery Metastore Catalog Support [iceberg]

Posted by "nastra (via GitHub)" <gi...@apache.org>.
nastra commented on code in PR #12808:
URL: https://github.com/apache/iceberg/pull/12808#discussion_r2059737734


##########
bigquery/src/test/java/org/apache/iceberg/gcp/bigquery/TestBigQueryCatalog.java:
##########
@@ -0,0 +1,673 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.iceberg.gcp.bigquery;
+
+import static org.apache.iceberg.CatalogUtil.ICEBERG_CATALOG_TYPE;
+import static org.apache.iceberg.CatalogUtil.ICEBERG_CATALOG_TYPE_BIGQUERY;
+import static org.apache.iceberg.gcp.bigquery.BigQueryMetastoreCatalog.PROJECT_ID;
+import static org.apache.iceberg.gcp.bigquery.BigQueryMetastoreCatalog.TESTING_ENABLED;
+import static org.apache.iceberg.types.Types.NestedField.optional;
+import static org.assertj.core.api.Assertions.assertThat;
+import static org.assertj.core.api.Assertions.assertThatThrownBy;
+
+import java.io.File;
+import java.util.List;
+import java.util.Map;
+import org.apache.iceberg.CatalogProperties;
+import org.apache.iceberg.CatalogUtil;
+import org.apache.iceberg.PartitionSpec;
+import org.apache.iceberg.Schema;
+import org.apache.iceberg.SortOrder;
+import org.apache.iceberg.Table;
+import org.apache.iceberg.TableUtil;
+import org.apache.iceberg.Transaction;
+import org.apache.iceberg.catalog.CatalogTests;
+import org.apache.iceberg.catalog.Namespace;
+import org.apache.iceberg.catalog.TableIdentifier;
+import org.apache.iceberg.exceptions.NoSuchNamespaceException;
+import org.apache.iceberg.exceptions.NoSuchTableException;
+import org.apache.iceberg.expressions.Expressions;
+import org.apache.iceberg.relocated.com.google.common.collect.ImmutableMap;
+import org.apache.iceberg.types.Types;
+import org.junit.jupiter.api.AfterEach;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Disabled;
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.io.TempDir;
+import org.junit.jupiter.params.ParameterizedTest;
+import org.junit.jupiter.params.provider.ValueSource;
+
+public class TestBigQueryCatalog extends CatalogTests<BigQueryMetastoreCatalog> {
+
+  private static final String GCP_PROJECT = "project-id";
+  private static final String CATALOG_ID = "catalog-name";

Review Comment:
   is this one needed given that it's only used at a single place?



-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org

For queries about this service, please contact Infrastructure at:
users@infra.apache.org


---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For additional commands, e-mail: issues-help@iceberg.apache.org


Re: [PR] Catalog: Add BigQuery Metastore Catalog Support [iceberg]

Posted by "nastra (via GitHub)" <gi...@apache.org>.
nastra commented on code in PR #12808:
URL: https://github.com/apache/iceberg/pull/12808#discussion_r2059745760


##########
bigquery/src/test/java/org/apache/iceberg/gcp/bigquery/TestBigQueryCatalog.java:
##########
@@ -0,0 +1,673 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.iceberg.gcp.bigquery;
+
+import static org.apache.iceberg.CatalogUtil.ICEBERG_CATALOG_TYPE;
+import static org.apache.iceberg.CatalogUtil.ICEBERG_CATALOG_TYPE_BIGQUERY;
+import static org.apache.iceberg.gcp.bigquery.BigQueryMetastoreCatalog.PROJECT_ID;
+import static org.apache.iceberg.gcp.bigquery.BigQueryMetastoreCatalog.TESTING_ENABLED;
+import static org.apache.iceberg.types.Types.NestedField.optional;
+import static org.assertj.core.api.Assertions.assertThat;
+import static org.assertj.core.api.Assertions.assertThatThrownBy;
+
+import java.io.File;
+import java.util.List;
+import java.util.Map;
+import org.apache.iceberg.CatalogProperties;
+import org.apache.iceberg.CatalogUtil;
+import org.apache.iceberg.PartitionSpec;
+import org.apache.iceberg.Schema;
+import org.apache.iceberg.SortOrder;
+import org.apache.iceberg.Table;
+import org.apache.iceberg.TableUtil;
+import org.apache.iceberg.Transaction;
+import org.apache.iceberg.catalog.CatalogTests;
+import org.apache.iceberg.catalog.Namespace;
+import org.apache.iceberg.catalog.TableIdentifier;
+import org.apache.iceberg.exceptions.NoSuchNamespaceException;
+import org.apache.iceberg.exceptions.NoSuchTableException;
+import org.apache.iceberg.expressions.Expressions;
+import org.apache.iceberg.relocated.com.google.common.collect.ImmutableMap;
+import org.apache.iceberg.types.Types;
+import org.junit.jupiter.api.AfterEach;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Disabled;
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.io.TempDir;
+import org.junit.jupiter.params.ParameterizedTest;
+import org.junit.jupiter.params.provider.ValueSource;
+
+public class TestBigQueryCatalog extends CatalogTests<BigQueryMetastoreCatalog> {
+
+  private static final String GCP_PROJECT = "project-id";
+  private static final String CATALOG_ID = "catalog-name";
+
+  // another schema that is not the same
+  protected static final Schema OTHER_SCHEMA_OPTIONAL =
+      new Schema(optional(1, "some_id", Types.IntegerType.get()));
+
+  // Schema passed to create tables
+  protected static final Schema SCHEMA_OPTIONAL =
+      new Schema(
+          optional(3, "id", Types.IntegerType.get(), "unique ID 🤪"),
+          optional(4, "data", Types.StringType.get()));
+
+  // This is the actual schema for the table, with column IDs reassigned
+  protected static final Schema REPLACE_SCHEMA_OPTIONAL =
+      new Schema(
+          optional(2, "id", Types.IntegerType.get(), "unique ID 🤪"),
+          optional(3, "data", Types.StringType.get()));
+
+  protected static final PartitionSpec REPLACE_SPEC_OPTIONAL =
+      PartitionSpec.builderFor(REPLACE_SCHEMA_OPTIONAL).bucket("id", 16).withSpecId(1).build();
+
+  protected static final SortOrder REPLACE_WRITE_ORDER_OPTIONAL =
+      SortOrder.builderFor(REPLACE_SCHEMA_OPTIONAL)
+          .asc(Expressions.bucket("id", 16))
+          .asc("id")
+          .build();
+
+  @TempDir private File tempFolder;
+  private BigQueryMetastoreCatalog catalog;
+  private String warehouseLocation;
+
+  @BeforeEach
+  public void before() throws Exception {
+    catalog = initCatalog(CATALOG_ID, ImmutableMap.of());
+  }
+
+  @AfterEach
+  public void after() throws Exception {
+    // Drop all tables in all datasets
+    List<Namespace> namespaces = catalog().listNamespaces(Namespace.empty());
+    for (Namespace namespace : namespaces) {
+      List<TableIdentifier> tables = catalog().listTables(namespace);
+      for (TableIdentifier table : tables) {
+        try {
+          catalog().dropTable(table, true); // drop with purge
+        } catch (NoSuchTableException e) {
+          // Table might be dropped by a previous iteration, ignore
+        }
+      }
+
+      // Drop all datasets (except any initial ones)
+      try {
+        catalog().dropNamespace(namespace);
+      } catch (NoSuchNamespaceException e) {
+        // Namespace might be dropped by a previous iteration, ignore
+      }
+    }
+  }
+
+  @Override
+  protected boolean requiresNamespaceCreate() {
+    return true;
+  }
+
+  @Override
+  protected boolean supportsNamesWithSlashes() {
+    return false;
+  }
+
+  @Override
+  protected boolean supportsNamesWithDot() {
+    return false;
+  }
+
+  @Override
+  public BigQueryMetastoreCatalog catalog() {
+    return catalog;
+  }
+
+  @Override
+  protected BigQueryMetastoreCatalog initCatalog(
+      String catalogName, Map<String, String> additionalProperties) {
+
+    warehouseLocation = tempFolder.toPath().resolve("hive-warehouse").toString();
+
+    Map<String, String> properties =
+        Map.of(
+            ICEBERG_CATALOG_TYPE,
+            ICEBERG_CATALOG_TYPE_BIGQUERY,
+            PROJECT_ID,
+            GCP_PROJECT,
+            CatalogProperties.WAREHOUSE_LOCATION,
+            warehouseLocation,
+            TESTING_ENABLED,
+            "true",
+            CatalogProperties.TABLE_DEFAULT_PREFIX + "default-key1",
+            "catalog-default-key1",
+            CatalogProperties.TABLE_DEFAULT_PREFIX + "default-key2",
+            "catalog-default-key2",
+            CatalogProperties.TABLE_DEFAULT_PREFIX + "override-key3",
+            "catalog-default-key3",
+            CatalogProperties.TABLE_OVERRIDE_PREFIX + "override-key3",
+            "catalog-override-key3",
+            CatalogProperties.TABLE_OVERRIDE_PREFIX + "override-key4",
+            "catalog-override-key4");
+
+    BigQueryMetastoreCatalog tmpCatalog =
+        (BigQueryMetastoreCatalog)
+            CatalogUtil.buildIcebergCatalog(
+                catalogName,
+                ImmutableMap.<String, String>builder()
+                    .putAll(properties)
+                    .putAll(additionalProperties)
+                    .build(),
+                null);
+
+    return tmpCatalog;
+  }
+
+  @Test
+  public void testRenameTable() {
+    if (requiresNamespaceCreate()) {
+      catalog.createNamespace(NS);
+    }
+
+    assertThat(catalog.tableExists(TABLE))
+        .as("Source table should not exist before create")
+        .isFalse();
+
+    catalog.buildTable(TABLE, SCHEMA).create();
+    assertThat(catalog.tableExists(TABLE)).as("Table should exist after create").isTrue();
+
+    assertThat(catalog.tableExists(RENAMED_TABLE))
+        .as("Destination table should not exist before rename")
+        .isFalse();
+
+    assertThatThrownBy(() -> catalog.renameTable(TABLE, RENAMED_TABLE))
+        .isInstanceOf(UnsupportedOperationException.class)
+        .hasMessageContaining("Table rename operation is unsupported");
+
+    catalog.dropTable(TABLE);
+  }
+
+  @Disabled(
+      "BigQuery does not allow adding a REQUIRED column to an existing table schema. "
+          + "When this operation is attempted, an exception gets thrown.")
+  @Test
+  public void testOverrideTablePropertiesReplaceTransaction() {}

Review Comment:
   given that the tests here that live in the Iceberg codebase are always running against the Fake client, I would suggest to remove all of those `Disabled` tests, since they all pass against the Fake client. If you want to run the catalog tests against an actual client, you can then basically just create a new `TestRealBigQueryCatalog` (I'm sure there's a better name) that initializes the correct client where you disable the tests that don't work



-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org

For queries about this service, please contact Infrastructure at:
users@infra.apache.org


---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For additional commands, e-mail: issues-help@iceberg.apache.org


Re: [PR] Catalog: Add BigQuery Metastore Catalog Support [iceberg]

Posted by "talatuyarer (via GitHub)" <gi...@apache.org>.
talatuyarer commented on code in PR #12808:
URL: https://github.com/apache/iceberg/pull/12808#discussion_r2059867329


##########
bigquery/src/main/java/org/apache/iceberg/gcp/bigquery/BigQueryMetastoreCatalog.java:
##########
@@ -0,0 +1,400 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.iceberg.gcp.bigquery;
+
+import com.google.api.services.bigquery.model.Dataset;
+import com.google.api.services.bigquery.model.DatasetList.Datasets;
+import com.google.api.services.bigquery.model.DatasetReference;
+import com.google.api.services.bigquery.model.ExternalCatalogDatasetOptions;
+import com.google.api.services.bigquery.model.TableReference;
+import com.google.cloud.ServiceOptions;
+import com.google.cloud.bigquery.BigQueryOptions;
+import java.io.IOException;
+import java.security.GeneralSecurityException;
+import java.util.List;
+import java.util.Locale;
+import java.util.Map;
+import java.util.Objects;
+import java.util.Set;
+import org.apache.hadoop.conf.Configurable;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.iceberg.BaseMetastoreCatalog;
+import org.apache.iceberg.CatalogProperties;
+import org.apache.iceberg.CatalogUtil;
+import org.apache.iceberg.TableMetadata;
+import org.apache.iceberg.TableOperations;
+import org.apache.iceberg.catalog.Namespace;
+import org.apache.iceberg.catalog.SupportsNamespaces;
+import org.apache.iceberg.catalog.TableIdentifier;
+import org.apache.iceberg.exceptions.AlreadyExistsException;
+import org.apache.iceberg.exceptions.NoSuchNamespaceException;
+import org.apache.iceberg.exceptions.NoSuchTableException;
+import org.apache.iceberg.exceptions.ServiceFailureException;
+import org.apache.iceberg.exceptions.ValidationException;
+import org.apache.iceberg.io.FileIO;
+import org.apache.iceberg.relocated.com.google.common.annotations.VisibleForTesting;
+import org.apache.iceberg.relocated.com.google.common.base.Preconditions;
+import org.apache.iceberg.relocated.com.google.common.base.Strings;
+import org.apache.iceberg.relocated.com.google.common.collect.ImmutableList;
+import org.apache.iceberg.relocated.com.google.common.collect.ImmutableMap;
+import org.apache.iceberg.relocated.com.google.common.collect.Maps;
+import org.apache.iceberg.util.LocationUtil;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/** Iceberg Bigquery Metastore Catalog implementation. */
+public final class BigQueryMetastoreCatalog extends BaseMetastoreCatalog
+    implements SupportsNamespaces, Configurable {
+
+  // User provided properties.
+  public static final String PROJECT_ID = "gcp-project";
+  public static final String GCP_LOCATION = "gcp-location";
+  public static final String FILTER_UNSUPPORTED_TABLES = "filter-unsupported-tables";
+  public static final String TESTING_ENABLED = "testing-enabled";
+
+  public static final String HIVE_METASTORE_WAREHOUSE_DIR = "hive.metastore.warehouse.dir";
+
+  private static final Logger LOG = LoggerFactory.getLogger(BigQueryMetastoreCatalog.class);
+
+  private static final String DEFAULT_GCP_LOCATION = "us";
+
+  private String catalogName;
+  private Map<String, String> catalogProperties;
+  private FileIO fileIO;
+  private Configuration conf;
+  private String projectId;
+  private String location;
+  private BigQueryMetastoreClient client;
+  private boolean filterUnsupportedTables;
+
+  // Must have a no-arg constructor to be dynamically loaded
+  // initialize(String name, Map<String, String> properties) will be called to complete
+  // initialization
+  public BigQueryMetastoreCatalog() {}
+
+  @Override
+  public void initialize(String inputName, Map<String, String> properties) {
+    Preconditions.checkArgument(
+        properties.containsKey(PROJECT_ID), "GCP project must be specified");
+
+    projectId = properties.get(PROJECT_ID);
+    location = properties.getOrDefault(GCP_LOCATION, DEFAULT_GCP_LOCATION);
+    boolean testingEnabled =
+        Boolean.parseBoolean(properties.getOrDefault(TESTING_ENABLED, "false"));
+
+    BigQueryOptions options =
+        BigQueryOptions.newBuilder()
+            .setProjectId(projectId)
+            .setLocation(location)
+            .setRetrySettings(ServiceOptions.getDefaultRetrySettings())
+            .build();
+
+    try {
+      if (testingEnabled) {
+        client = new FakeBigQueryMetastoreClient(options);
+
+      } else {
+        client = new BigQueryMetastoreClientImpl(options);
+      }
+    } catch (IOException e) {
+      throw new ServiceFailureException(e, "Creating BigQuery client failed");
+    } catch (GeneralSecurityException e) {
+      throw new ValidationException(e, "Creating BigQuery client failed due to a security issue");
+    }
+    initialize(inputName, properties, projectId, location, client);
+  }
+
+  @VisibleForTesting
+  void initialize(
+      String inputName,
+      Map<String, String> properties,
+      String initialProjectId,
+      String initialLocation,
+      BigQueryMetastoreClient bigQueryMetaStoreClient) {
+    Preconditions.checkArgument(bigQueryMetaStoreClient != null, "BigQuery client can not be null");
+    this.catalogName = inputName;
+    this.catalogProperties = ImmutableMap.copyOf(properties);
+    this.projectId = initialProjectId;
+    this.location = initialLocation;
+    this.client = bigQueryMetaStoreClient;
+
+    if (this.conf == null) {
+      LOG.warn("No configuration was set, using the default environment Configuration");
+      this.conf = new Configuration();
+    }
+
+    LOG.info("Using BigQuery Metastore Iceberg Catalog: {}", inputName);
+
+    if (properties.containsKey(CatalogProperties.WAREHOUSE_LOCATION)) {
+      // Iceberg always removes trailing slash to avoid paths like "<folder>//data" in file systems
+      // like s3.
+      this.conf.set(
+          HIVE_METASTORE_WAREHOUSE_DIR,
+          LocationUtil.stripTrailingSlash(properties.get(CatalogProperties.WAREHOUSE_LOCATION)));
+    }
+
+    String fileIoImpl =
+        properties.getOrDefault(
+            CatalogProperties.FILE_IO_IMPL, "org.apache.iceberg.hadoop.HadoopFileIO");
+    this.fileIO = CatalogUtil.loadFileIO(fileIoImpl, properties, conf);
+
+    this.filterUnsupportedTables =
+        Boolean.parseBoolean(properties.getOrDefault(FILTER_UNSUPPORTED_TABLES, "false"));
+  }
+
+  @Override
+  protected TableOperations newTableOps(TableIdentifier identifier) {
+    return new BigQueryTableOperations(
+        client,
+        fileIO,
+        projectId,
+        // Sometimes extensions have the namespace contain the table name too, so we are forced to
+        // allow invalid namespace and just take the first part here like other catalog
+        // implementations do.
+        identifier.namespace().level(0),
+        identifier.name(),
+        conf);
+  }
+
+  @Override
+  protected String defaultWarehouseLocation(TableIdentifier identifier) {
+    String locationUri = null;
+    DatasetReference datasetReference = toDatasetReference(identifier.namespace());
+    Dataset dataset = client.getDataset(datasetReference);
+    if (dataset != null && dataset.getExternalCatalogDatasetOptions() != null) {
+      locationUri = dataset.getExternalCatalogDatasetOptions().getDefaultStorageLocationUri();
+    }
+    return String.format(
+        "%s/%s",
+        Strings.isNullOrEmpty(locationUri)
+            ? datasetReference.getDatasetId()
+            : LocationUtil.stripTrailingSlash(locationUri),
+        identifier.name());
+  }
+
+  @Override
+  public List<TableIdentifier> listTables(Namespace namespace) {
+    validateNamespace(namespace);
+
+    return client.listTables(toDatasetReference(namespace), filterUnsupportedTables).stream()
+        .map(
+            table -> TableIdentifier.of(namespace.level(0), table.getTableReference().getTableId()))
+        .collect(ImmutableList.toImmutableList());
+  }
+
+  @Override
+  public boolean dropTable(TableIdentifier identifier, boolean purge) {
+    try {
+      TableOperations ops = newTableOps(identifier);
+      TableMetadata lastMetadata = ops.current();
+
+      client.deleteTable(toTableReference(identifier));
+
+      if (purge && lastMetadata != null) {
+        CatalogUtil.dropTableData(ops.io(), lastMetadata);
+      }
+    } catch (NoSuchTableException e) { // Not catching a NoSuchIcebergTableException on purpose
+      return false; // The documentation says just return false in this case
+    }
+    return true;
+  }
+
+  @Override
+  public void renameTable(TableIdentifier from, TableIdentifier to) {
+    validateNamespace(from.namespace());
+    validateNamespace(to.namespace());
+
+    try {
+      client.getDataset(toDatasetReference(to.namespace()));
+      client.getDataset(toDatasetReference(from.namespace()));
+    } catch (NoSuchNamespaceException e) {
+      throw e;
+    }
+
+    TableReference fromTableRef = toTableReference(from);
+    TableReference toTableRef = toTableReference(to);
+
+    // Check if the source table exists
+    try {
+      client.getTable(fromTableRef);
+    } catch (NoSuchTableException e) {
+      throw new NoSuchTableException("Table does not exist: %s", from.name());
+    }
+
+    // Check if the destination table already exists
+    try {
+      client.getTable(toTableRef);
+      // If getTable succeeds, the destination table exists
+      throw new AlreadyExistsException("Table already exists: %s", to.name());
+    } catch (NoSuchTableException e) {
+      // Destination table does not exist, proceed with throwing exception
+    }
+
+    if (!from.namespace().equals(to.namespace())) {
+      throw new ValidationException("New table name must be in the same namespace");
+    }
+
+    // TODO(b/354981675): Enable once supported by the API.
+    throw new UnsupportedOperationException("Table rename operation is unsupported.");
+  }
+
+  @Override
+  public void createNamespace(Namespace namespace, Map<String, String> metadata) {
+    Dataset builder = new Dataset();
+    DatasetReference datasetReference = toDatasetReference(namespace);
+    builder.setLocation(this.location);
+    builder.setDatasetReference(datasetReference);
+    builder.setExternalCatalogDatasetOptions(
+        BigQueryMetastoreUtils.createExternalCatalogDatasetOptions(
+            datasetReference.getDatasetId(), metadata));
+
+    client.createDataset(builder);
+  }
+
+  /**
+   * Since this catalog only supports one-level namespaces, it always returns an empty list unless
+   * passed an empty namespace to list all namespaces within the catalog.
+   */
+  @Override
+  public List<Namespace> listNamespaces(Namespace namespace) {

Review Comment:
   BigQuery MetaStore is the first catalog that supports this. I am proud of it 😃 



-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org

For queries about this service, please contact Infrastructure at:
users@infra.apache.org


---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For additional commands, e-mail: issues-help@iceberg.apache.org


Re: [PR] Catalog: Add BigQuery Metastore Catalog Support [iceberg]

Posted by "nastra (via GitHub)" <gi...@apache.org>.
nastra commented on code in PR #12808:
URL: https://github.com/apache/iceberg/pull/12808#discussion_r2055516878


##########
build.gradle:
##########
@@ -657,6 +657,47 @@ project(':iceberg-delta-lake') {
   }
 }
 
+project(':iceberg-bigquery') {
+  test {
+    useJUnitPlatform()
+  }
+
+  dependencies {
+    implementation project(path: ':iceberg-bundled-guava', configuration: 'shadow')
+    implementation project(':iceberg-gcp')
+
+    implementation platform(libs.google.libraries.bom)
+    compileOnly "com.google.cloud:google-cloud-storage"
+    implementation "com.google.cloud:google-cloud-bigquery"
+    implementation "com.google.cloud:google-cloud-core"
+
+    api project(':iceberg-api')
+    implementation project(':iceberg-common')
+    implementation project(':iceberg-core')
+
+    compileOnly libs.hadoop3.common
+    compileOnly("org.apache.hive:hive-metastore:4.0.1") {
+      exclude group: 'org.apache.avro', module: 'avro'
+      exclude group: 'org.slf4j', module: 'slf4j-log4j12'
+      exclude group: 'org.pentaho' // missing dependency

Review Comment:
   what does the comment mean here?



-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org

For queries about this service, please contact Infrastructure at:
users@infra.apache.org


---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For additional commands, e-mail: issues-help@iceberg.apache.org


Re: [PR] Catalog: Add BigQuery Metastore Catalog Support [iceberg]

Posted by "nastra (via GitHub)" <gi...@apache.org>.
nastra commented on code in PR #12808:
URL: https://github.com/apache/iceberg/pull/12808#discussion_r2055627940


##########
bigquery/src/main/java/org/apache/iceberg/gcp/bigquery/BigQueryMetastoreCatalog.java:
##########
@@ -0,0 +1,400 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.iceberg.gcp.bigquery;
+
+import com.google.api.services.bigquery.model.Dataset;
+import com.google.api.services.bigquery.model.DatasetList.Datasets;
+import com.google.api.services.bigquery.model.DatasetReference;
+import com.google.api.services.bigquery.model.ExternalCatalogDatasetOptions;
+import com.google.api.services.bigquery.model.TableReference;
+import com.google.cloud.ServiceOptions;
+import com.google.cloud.bigquery.BigQueryOptions;
+import java.io.IOException;
+import java.security.GeneralSecurityException;
+import java.util.List;
+import java.util.Locale;
+import java.util.Map;
+import java.util.Objects;
+import java.util.Set;
+import org.apache.hadoop.conf.Configurable;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.iceberg.BaseMetastoreCatalog;
+import org.apache.iceberg.CatalogProperties;
+import org.apache.iceberg.CatalogUtil;
+import org.apache.iceberg.TableMetadata;
+import org.apache.iceberg.TableOperations;
+import org.apache.iceberg.catalog.Namespace;
+import org.apache.iceberg.catalog.SupportsNamespaces;
+import org.apache.iceberg.catalog.TableIdentifier;
+import org.apache.iceberg.exceptions.AlreadyExistsException;
+import org.apache.iceberg.exceptions.NoSuchNamespaceException;
+import org.apache.iceberg.exceptions.NoSuchTableException;
+import org.apache.iceberg.exceptions.ServiceFailureException;
+import org.apache.iceberg.exceptions.ValidationException;
+import org.apache.iceberg.io.FileIO;
+import org.apache.iceberg.relocated.com.google.common.annotations.VisibleForTesting;
+import org.apache.iceberg.relocated.com.google.common.base.Preconditions;
+import org.apache.iceberg.relocated.com.google.common.base.Strings;
+import org.apache.iceberg.relocated.com.google.common.collect.ImmutableList;
+import org.apache.iceberg.relocated.com.google.common.collect.ImmutableMap;
+import org.apache.iceberg.relocated.com.google.common.collect.Maps;
+import org.apache.iceberg.util.LocationUtil;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/** Iceberg Bigquery Metastore Catalog implementation. */
+public final class BigQueryMetastoreCatalog extends BaseMetastoreCatalog
+    implements SupportsNamespaces, Configurable {
+
+  // User provided properties.
+  public static final String PROJECT_ID = "gcp-project";
+  public static final String GCP_LOCATION = "gcp-location";
+  public static final String FILTER_UNSUPPORTED_TABLES = "filter-unsupported-tables";
+  public static final String TESTING_ENABLED = "testing-enabled";
+
+  public static final String HIVE_METASTORE_WAREHOUSE_DIR = "hive.metastore.warehouse.dir";
+
+  private static final Logger LOG = LoggerFactory.getLogger(BigQueryMetastoreCatalog.class);
+
+  private static final String DEFAULT_GCP_LOCATION = "us";
+
+  private String catalogName;
+  private Map<String, String> catalogProperties;
+  private FileIO fileIO;
+  private Configuration conf;
+  private String projectId;
+  private String location;
+  private BigQueryMetastoreClient client;
+  private boolean filterUnsupportedTables;
+
+  // Must have a no-arg constructor to be dynamically loaded
+  // initialize(String name, Map<String, String> properties) will be called to complete
+  // initialization
+  public BigQueryMetastoreCatalog() {}
+
+  @Override
+  public void initialize(String inputName, Map<String, String> properties) {
+    Preconditions.checkArgument(
+        properties.containsKey(PROJECT_ID), "GCP project must be specified");
+
+    projectId = properties.get(PROJECT_ID);
+    location = properties.getOrDefault(GCP_LOCATION, DEFAULT_GCP_LOCATION);
+    boolean testingEnabled =
+        Boolean.parseBoolean(properties.getOrDefault(TESTING_ENABLED, "false"));
+
+    BigQueryOptions options =
+        BigQueryOptions.newBuilder()
+            .setProjectId(projectId)
+            .setLocation(location)
+            .setRetrySettings(ServiceOptions.getDefaultRetrySettings())
+            .build();
+
+    try {
+      if (testingEnabled) {
+        client = new FakeBigQueryMetastoreClient(options);
+
+      } else {
+        client = new BigQueryMetastoreClientImpl(options);
+      }
+    } catch (IOException e) {
+      throw new ServiceFailureException(e, "Creating BigQuery client failed");
+    } catch (GeneralSecurityException e) {
+      throw new ValidationException(e, "Creating BigQuery client failed due to a security issue");
+    }
+    initialize(inputName, properties, projectId, location, client);
+  }
+
+  @VisibleForTesting
+  void initialize(
+      String inputName,
+      Map<String, String> properties,
+      String initialProjectId,
+      String initialLocation,
+      BigQueryMetastoreClient bigQueryMetaStoreClient) {
+    Preconditions.checkArgument(bigQueryMetaStoreClient != null, "BigQuery client can not be null");
+    this.catalogName = inputName;
+    this.catalogProperties = ImmutableMap.copyOf(properties);
+    this.projectId = initialProjectId;
+    this.location = initialLocation;
+    this.client = bigQueryMetaStoreClient;
+
+    if (this.conf == null) {
+      LOG.warn("No configuration was set, using the default environment Configuration");
+      this.conf = new Configuration();
+    }
+
+    LOG.info("Using BigQuery Metastore Iceberg Catalog: {}", inputName);
+
+    if (properties.containsKey(CatalogProperties.WAREHOUSE_LOCATION)) {
+      // Iceberg always removes trailing slash to avoid paths like "<folder>//data" in file systems
+      // like s3.
+      this.conf.set(
+          HIVE_METASTORE_WAREHOUSE_DIR,
+          LocationUtil.stripTrailingSlash(properties.get(CatalogProperties.WAREHOUSE_LOCATION)));
+    }
+
+    String fileIoImpl =
+        properties.getOrDefault(
+            CatalogProperties.FILE_IO_IMPL, "org.apache.iceberg.hadoop.HadoopFileIO");
+    this.fileIO = CatalogUtil.loadFileIO(fileIoImpl, properties, conf);
+
+    this.filterUnsupportedTables =
+        Boolean.parseBoolean(properties.getOrDefault(FILTER_UNSUPPORTED_TABLES, "false"));
+  }
+
+  @Override
+  protected TableOperations newTableOps(TableIdentifier identifier) {
+    return new BigQueryTableOperations(
+        client,
+        fileIO,
+        projectId,
+        // Sometimes extensions have the namespace contain the table name too, so we are forced to
+        // allow invalid namespace and just take the first part here like other catalog
+        // implementations do.
+        identifier.namespace().level(0),
+        identifier.name(),
+        conf);
+  }
+
+  @Override
+  protected String defaultWarehouseLocation(TableIdentifier identifier) {
+    String locationUri = null;
+    DatasetReference datasetReference = toDatasetReference(identifier.namespace());
+    Dataset dataset = client.getDataset(datasetReference);
+    if (dataset != null && dataset.getExternalCatalogDatasetOptions() != null) {
+      locationUri = dataset.getExternalCatalogDatasetOptions().getDefaultStorageLocationUri();
+    }
+    return String.format(
+        "%s/%s",
+        Strings.isNullOrEmpty(locationUri)
+            ? datasetReference.getDatasetId()
+            : LocationUtil.stripTrailingSlash(locationUri),
+        identifier.name());
+  }
+
+  @Override
+  public List<TableIdentifier> listTables(Namespace namespace) {
+    validateNamespace(namespace);
+
+    return client.listTables(toDatasetReference(namespace), filterUnsupportedTables).stream()
+        .map(
+            table -> TableIdentifier.of(namespace.level(0), table.getTableReference().getTableId()))
+        .collect(ImmutableList.toImmutableList());
+  }
+
+  @Override
+  public boolean dropTable(TableIdentifier identifier, boolean purge) {
+    try {
+      TableOperations ops = newTableOps(identifier);
+      TableMetadata lastMetadata = ops.current();
+
+      client.deleteTable(toTableReference(identifier));
+
+      if (purge && lastMetadata != null) {
+        CatalogUtil.dropTableData(ops.io(), lastMetadata);
+      }
+    } catch (NoSuchTableException e) { // Not catching a NoSuchIcebergTableException on purpose
+      return false; // The documentation says just return false in this case
+    }
+    return true;
+  }
+
+  @Override
+  public void renameTable(TableIdentifier from, TableIdentifier to) {
+    validateNamespace(from.namespace());
+    validateNamespace(to.namespace());
+
+    try {
+      client.getDataset(toDatasetReference(to.namespace()));
+      client.getDataset(toDatasetReference(from.namespace()));
+    } catch (NoSuchNamespaceException e) {
+      throw e;
+    }
+
+    TableReference fromTableRef = toTableReference(from);
+    TableReference toTableRef = toTableReference(to);
+
+    // Check if the source table exists
+    try {
+      client.getTable(fromTableRef);
+    } catch (NoSuchTableException e) {
+      throw new NoSuchTableException("Table does not exist: %s", from.name());
+    }
+
+    // Check if the destination table already exists
+    try {
+      client.getTable(toTableRef);
+      // If getTable succeeds, the destination table exists
+      throw new AlreadyExistsException("Table already exists: %s", to.name());
+    } catch (NoSuchTableException e) {
+      // Destination table does not exist, proceed with throwing exception
+    }
+
+    if (!from.namespace().equals(to.namespace())) {
+      throw new ValidationException("New table name must be in the same namespace");
+    }
+
+    // TODO(b/354981675): Enable once supported by the API.
+    throw new UnsupportedOperationException("Table rename operation is unsupported.");
+  }
+
+  @Override
+  public void createNamespace(Namespace namespace, Map<String, String> metadata) {
+    Dataset builder = new Dataset();
+    DatasetReference datasetReference = toDatasetReference(namespace);
+    builder.setLocation(this.location);
+    builder.setDatasetReference(datasetReference);
+    builder.setExternalCatalogDatasetOptions(
+        BigQueryMetastoreUtils.createExternalCatalogDatasetOptions(
+            datasetReference.getDatasetId(), metadata));
+
+    client.createDataset(builder);
+  }
+
+  /**
+   * Since this catalog only supports one-level namespaces, it always returns an empty list unless
+   * passed an empty namespace to list all namespaces within the catalog.
+   */
+  @Override
+  public List<Namespace> listNamespaces(Namespace namespace) {
+    if (namespace.levels().length != 0) {
+      // BQMS does not support namespaces under database or tables, returns empty.
+      // It is called when dropping a namespace to make sure it's empty (listTables is called as
+      // well), returns empty to unblock deletion.
+      return ImmutableList.of();
+    }
+    return client.listDatasets(projectId).stream()
+        .map(BigQueryMetastoreCatalog::getNamespace)
+        .collect(ImmutableList.toImmutableList());
+  }
+
+  @Override
+  public boolean dropNamespace(Namespace namespace) {
+    try {
+      client.deleteDataset(toDatasetReference(namespace));
+      // We don't delete the data folder for safety, which aligns with Hive Metastore's default
+      // behavior.
+      // We can support database or catalog level config controlling file deletion in the future.
+      return true;
+    } catch (NoSuchNamespaceException e) {
+      return false;
+    }
+  }
+
+  @Override
+  public boolean setProperties(Namespace namespace, Map<String, String> properties) {
+    Dataset dataset = client.getDataset(toDatasetReference(namespace));
+
+    ExternalCatalogDatasetOptions existingOptions = dataset.getExternalCatalogDatasetOptions();
+    Map<String, String> existingParameters =
+        existingOptions != null ? existingOptions.getParameters() : null;
+
+    Map<String, String> newParameters = Maps.newHashMap();
+    if (existingParameters != null) {
+      newParameters.putAll(existingParameters);
+    }
+    newParameters.putAll(properties);
+
+    if (Objects.equals(existingParameters, newParameters)) {
+      // No change in parameters detected
+      return false;
+    }
+
+    client.setDatasetParameters(toDatasetReference(namespace), properties);
+    return true;
+  }
+
+  @Override
+  public boolean removeProperties(Namespace namespace, Set<String> properties) {
+    client.removeDatasetParameters(toDatasetReference(namespace), properties);
+    return true;
+  }
+
+  @SuppressWarnings("FormatStringAnnotation")
+  @Override
+  public Map<String, String> loadNamespaceMetadata(Namespace namespace) {
+    try {
+      return getMetadata(client.getDataset(toDatasetReference(namespace)));
+    } catch (IllegalArgumentException e) {
+      throw new NoSuchNamespaceException(e.getMessage());
+    }
+  }
+
+  @Override
+  public String name() {
+    return catalogName;
+  }
+
+  @Override
+  protected Map<String, String> properties() {
+    return catalogProperties == null ? ImmutableMap.of() : catalogProperties;
+  }
+
+  @Override
+  public void setConf(Configuration conf) {
+    this.conf = new Configuration(conf);
+  }
+
+  @Override
+  public Configuration getConf() {
+    return this.conf;
+  }
+
+  private static Namespace getNamespace(Datasets dataset) {
+    return Namespace.of(dataset.getDatasetReference().getDatasetId());
+  }
+
+  private DatasetReference toDatasetReference(Namespace namespace) {
+    validateNamespace(namespace);
+    return new DatasetReference().setProjectId(projectId).setDatasetId(namespace.level(0));
+  }
+
+  private TableReference toTableReference(TableIdentifier tableIdentifier) {
+    DatasetReference datasetReference = toDatasetReference(tableIdentifier.namespace());
+    return new TableReference()
+        .setProjectId(datasetReference.getProjectId())
+        .setDatasetId(datasetReference.getDatasetId())
+        .setTableId(tableIdentifier.name());
+  }
+
+  private static Map<String, String> getMetadata(Dataset dataset) {
+    ExternalCatalogDatasetOptions options = dataset.getExternalCatalogDatasetOptions();
+    Map<String, String> metadata = Maps.newHashMap();
+    if (options != null) {
+      if (options.getParameters() != null) {
+        metadata.putAll(options.getParameters());
+      }
+      if (!Strings.isNullOrEmpty(options.getDefaultStorageLocationUri())) {
+        metadata.put("location", options.getDefaultStorageLocationUri());
+      }
+    }
+    return metadata;
+  }
+
+  private static void validateNamespace(Namespace namespace) {

Review Comment:
   ah nvm, I missed that the validation is already part of `toDatasetReference`



-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org

For queries about this service, please contact Infrastructure at:
users@infra.apache.org


---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For additional commands, e-mail: issues-help@iceberg.apache.org


Re: [PR] Catalog: Add BigQuery Metastore Catalog Support [iceberg]

Posted by "nastra (via GitHub)" <gi...@apache.org>.
nastra commented on code in PR #12808:
URL: https://github.com/apache/iceberg/pull/12808#discussion_r2055636208


##########
bigquery/src/main/java/org/apache/iceberg/gcp/bigquery/BigQueryMetastoreCatalog.java:
##########
@@ -0,0 +1,400 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.iceberg.gcp.bigquery;
+
+import com.google.api.services.bigquery.model.Dataset;
+import com.google.api.services.bigquery.model.DatasetList.Datasets;
+import com.google.api.services.bigquery.model.DatasetReference;
+import com.google.api.services.bigquery.model.ExternalCatalogDatasetOptions;
+import com.google.api.services.bigquery.model.TableReference;
+import com.google.cloud.ServiceOptions;
+import com.google.cloud.bigquery.BigQueryOptions;
+import java.io.IOException;
+import java.security.GeneralSecurityException;
+import java.util.List;
+import java.util.Locale;
+import java.util.Map;
+import java.util.Objects;
+import java.util.Set;
+import org.apache.hadoop.conf.Configurable;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.iceberg.BaseMetastoreCatalog;
+import org.apache.iceberg.CatalogProperties;
+import org.apache.iceberg.CatalogUtil;
+import org.apache.iceberg.TableMetadata;
+import org.apache.iceberg.TableOperations;
+import org.apache.iceberg.catalog.Namespace;
+import org.apache.iceberg.catalog.SupportsNamespaces;
+import org.apache.iceberg.catalog.TableIdentifier;
+import org.apache.iceberg.exceptions.AlreadyExistsException;
+import org.apache.iceberg.exceptions.NoSuchNamespaceException;
+import org.apache.iceberg.exceptions.NoSuchTableException;
+import org.apache.iceberg.exceptions.ServiceFailureException;
+import org.apache.iceberg.exceptions.ValidationException;
+import org.apache.iceberg.io.FileIO;
+import org.apache.iceberg.relocated.com.google.common.annotations.VisibleForTesting;
+import org.apache.iceberg.relocated.com.google.common.base.Preconditions;
+import org.apache.iceberg.relocated.com.google.common.base.Strings;
+import org.apache.iceberg.relocated.com.google.common.collect.ImmutableList;
+import org.apache.iceberg.relocated.com.google.common.collect.ImmutableMap;
+import org.apache.iceberg.relocated.com.google.common.collect.Maps;
+import org.apache.iceberg.util.LocationUtil;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/** Iceberg Bigquery Metastore Catalog implementation. */
+public final class BigQueryMetastoreCatalog extends BaseMetastoreCatalog
+    implements SupportsNamespaces, Configurable {
+
+  // User provided properties.
+  public static final String PROJECT_ID = "gcp-project";
+  public static final String GCP_LOCATION = "gcp-location";
+  public static final String FILTER_UNSUPPORTED_TABLES = "filter-unsupported-tables";
+  public static final String TESTING_ENABLED = "testing-enabled";
+
+  public static final String HIVE_METASTORE_WAREHOUSE_DIR = "hive.metastore.warehouse.dir";
+
+  private static final Logger LOG = LoggerFactory.getLogger(BigQueryMetastoreCatalog.class);
+
+  private static final String DEFAULT_GCP_LOCATION = "us";
+
+  private String catalogName;
+  private Map<String, String> catalogProperties;
+  private FileIO fileIO;
+  private Configuration conf;
+  private String projectId;
+  private String location;
+  private BigQueryMetastoreClient client;
+  private boolean filterUnsupportedTables;
+
+  // Must have a no-arg constructor to be dynamically loaded
+  // initialize(String name, Map<String, String> properties) will be called to complete
+  // initialization
+  public BigQueryMetastoreCatalog() {}
+
+  @Override
+  public void initialize(String inputName, Map<String, String> properties) {
+    Preconditions.checkArgument(
+        properties.containsKey(PROJECT_ID), "GCP project must be specified");
+
+    projectId = properties.get(PROJECT_ID);
+    location = properties.getOrDefault(GCP_LOCATION, DEFAULT_GCP_LOCATION);
+    boolean testingEnabled =
+        Boolean.parseBoolean(properties.getOrDefault(TESTING_ENABLED, "false"));
+
+    BigQueryOptions options =
+        BigQueryOptions.newBuilder()
+            .setProjectId(projectId)
+            .setLocation(location)
+            .setRetrySettings(ServiceOptions.getDefaultRetrySettings())
+            .build();
+
+    try {
+      if (testingEnabled) {
+        client = new FakeBigQueryMetastoreClient(options);
+
+      } else {
+        client = new BigQueryMetastoreClientImpl(options);
+      }
+    } catch (IOException e) {
+      throw new ServiceFailureException(e, "Creating BigQuery client failed");
+    } catch (GeneralSecurityException e) {
+      throw new ValidationException(e, "Creating BigQuery client failed due to a security issue");
+    }
+    initialize(inputName, properties, projectId, location, client);
+  }
+
+  @VisibleForTesting
+  void initialize(
+      String inputName,
+      Map<String, String> properties,
+      String initialProjectId,
+      String initialLocation,
+      BigQueryMetastoreClient bigQueryMetaStoreClient) {
+    Preconditions.checkArgument(bigQueryMetaStoreClient != null, "BigQuery client can not be null");
+    this.catalogName = inputName;
+    this.catalogProperties = ImmutableMap.copyOf(properties);
+    this.projectId = initialProjectId;
+    this.location = initialLocation;
+    this.client = bigQueryMetaStoreClient;
+
+    if (this.conf == null) {
+      LOG.warn("No configuration was set, using the default environment Configuration");
+      this.conf = new Configuration();
+    }
+
+    LOG.info("Using BigQuery Metastore Iceberg Catalog: {}", inputName);
+
+    if (properties.containsKey(CatalogProperties.WAREHOUSE_LOCATION)) {
+      // Iceberg always removes trailing slash to avoid paths like "<folder>//data" in file systems
+      // like s3.
+      this.conf.set(
+          HIVE_METASTORE_WAREHOUSE_DIR,
+          LocationUtil.stripTrailingSlash(properties.get(CatalogProperties.WAREHOUSE_LOCATION)));
+    }
+
+    String fileIoImpl =
+        properties.getOrDefault(
+            CatalogProperties.FILE_IO_IMPL, "org.apache.iceberg.hadoop.HadoopFileIO");
+    this.fileIO = CatalogUtil.loadFileIO(fileIoImpl, properties, conf);
+
+    this.filterUnsupportedTables =
+        Boolean.parseBoolean(properties.getOrDefault(FILTER_UNSUPPORTED_TABLES, "false"));
+  }
+
+  @Override
+  protected TableOperations newTableOps(TableIdentifier identifier) {
+    return new BigQueryTableOperations(
+        client,
+        fileIO,
+        projectId,
+        // Sometimes extensions have the namespace contain the table name too, so we are forced to
+        // allow invalid namespace and just take the first part here like other catalog
+        // implementations do.
+        identifier.namespace().level(0),
+        identifier.name(),
+        conf);
+  }
+
+  @Override
+  protected String defaultWarehouseLocation(TableIdentifier identifier) {
+    String locationUri = null;
+    DatasetReference datasetReference = toDatasetReference(identifier.namespace());
+    Dataset dataset = client.getDataset(datasetReference);
+    if (dataset != null && dataset.getExternalCatalogDatasetOptions() != null) {
+      locationUri = dataset.getExternalCatalogDatasetOptions().getDefaultStorageLocationUri();
+    }
+    return String.format(
+        "%s/%s",
+        Strings.isNullOrEmpty(locationUri)
+            ? datasetReference.getDatasetId()
+            : LocationUtil.stripTrailingSlash(locationUri),
+        identifier.name());
+  }
+
+  @Override
+  public List<TableIdentifier> listTables(Namespace namespace) {
+    validateNamespace(namespace);
+
+    return client.listTables(toDatasetReference(namespace), filterUnsupportedTables).stream()
+        .map(
+            table -> TableIdentifier.of(namespace.level(0), table.getTableReference().getTableId()))
+        .collect(ImmutableList.toImmutableList());
+  }
+
+  @Override
+  public boolean dropTable(TableIdentifier identifier, boolean purge) {
+    try {
+      TableOperations ops = newTableOps(identifier);
+      TableMetadata lastMetadata = ops.current();
+
+      client.deleteTable(toTableReference(identifier));
+
+      if (purge && lastMetadata != null) {
+        CatalogUtil.dropTableData(ops.io(), lastMetadata);
+      }
+    } catch (NoSuchTableException e) { // Not catching a NoSuchIcebergTableException on purpose
+      return false; // The documentation says just return false in this case
+    }
+    return true;
+  }
+
+  @Override
+  public void renameTable(TableIdentifier from, TableIdentifier to) {
+    validateNamespace(from.namespace());
+    validateNamespace(to.namespace());
+
+    try {
+      client.getDataset(toDatasetReference(to.namespace()));
+      client.getDataset(toDatasetReference(from.namespace()));
+    } catch (NoSuchNamespaceException e) {
+      throw e;
+    }
+
+    TableReference fromTableRef = toTableReference(from);
+    TableReference toTableRef = toTableReference(to);
+
+    // Check if the source table exists
+    try {
+      client.getTable(fromTableRef);
+    } catch (NoSuchTableException e) {
+      throw new NoSuchTableException("Table does not exist: %s", from.name());
+    }
+
+    // Check if the destination table already exists
+    try {
+      client.getTable(toTableRef);
+      // If getTable succeeds, the destination table exists
+      throw new AlreadyExistsException("Table already exists: %s", to.name());
+    } catch (NoSuchTableException e) {
+      // Destination table does not exist, proceed with throwing exception
+    }
+
+    if (!from.namespace().equals(to.namespace())) {
+      throw new ValidationException("New table name must be in the same namespace");
+    }
+
+    // TODO(b/354981675): Enable once supported by the API.
+    throw new UnsupportedOperationException("Table rename operation is unsupported.");
+  }
+
+  @Override
+  public void createNamespace(Namespace namespace, Map<String, String> metadata) {
+    Dataset builder = new Dataset();
+    DatasetReference datasetReference = toDatasetReference(namespace);
+    builder.setLocation(this.location);
+    builder.setDatasetReference(datasetReference);
+    builder.setExternalCatalogDatasetOptions(
+        BigQueryMetastoreUtils.createExternalCatalogDatasetOptions(
+            datasetReference.getDatasetId(), metadata));
+
+    client.createDataset(builder);
+  }
+
+  /**
+   * Since this catalog only supports one-level namespaces, it always returns an empty list unless
+   * passed an empty namespace to list all namespaces within the catalog.
+   */
+  @Override
+  public List<Namespace> listNamespaces(Namespace namespace) {
+    if (namespace.levels().length != 0) {
+      // BQMS does not support namespaces under database or tables, returns empty.
+      // It is called when dropping a namespace to make sure it's empty (listTables is called as
+      // well), returns empty to unblock deletion.
+      return ImmutableList.of();
+    }
+    return client.listDatasets(projectId).stream()
+        .map(BigQueryMetastoreCatalog::getNamespace)
+        .collect(ImmutableList.toImmutableList());
+  }
+
+  @Override
+  public boolean dropNamespace(Namespace namespace) {
+    try {
+      client.deleteDataset(toDatasetReference(namespace));
+      // We don't delete the data folder for safety, which aligns with Hive Metastore's default
+      // behavior.
+      // We can support database or catalog level config controlling file deletion in the future.
+      return true;
+    } catch (NoSuchNamespaceException e) {
+      return false;
+    }
+  }
+
+  @Override
+  public boolean setProperties(Namespace namespace, Map<String, String> properties) {
+    Dataset dataset = client.getDataset(toDatasetReference(namespace));
+
+    ExternalCatalogDatasetOptions existingOptions = dataset.getExternalCatalogDatasetOptions();
+    Map<String, String> existingParameters =
+        existingOptions != null ? existingOptions.getParameters() : null;
+
+    Map<String, String> newParameters = Maps.newHashMap();
+    if (existingParameters != null) {
+      newParameters.putAll(existingParameters);
+    }
+    newParameters.putAll(properties);
+
+    if (Objects.equals(existingParameters, newParameters)) {
+      // No change in parameters detected
+      return false;
+    }
+
+    client.setDatasetParameters(toDatasetReference(namespace), properties);
+    return true;
+  }
+
+  @Override
+  public boolean removeProperties(Namespace namespace, Set<String> properties) {
+    client.removeDatasetParameters(toDatasetReference(namespace), properties);
+    return true;
+  }
+
+  @SuppressWarnings("FormatStringAnnotation")
+  @Override
+  public Map<String, String> loadNamespaceMetadata(Namespace namespace) {
+    try {
+      return getMetadata(client.getDataset(toDatasetReference(namespace)));
+    } catch (IllegalArgumentException e) {
+      throw new NoSuchNamespaceException(e.getMessage());
+    }
+  }
+
+  @Override
+  public String name() {
+    return catalogName;
+  }
+
+  @Override
+  protected Map<String, String> properties() {
+    return catalogProperties == null ? ImmutableMap.of() : catalogProperties;
+  }
+
+  @Override
+  public void setConf(Configuration conf) {
+    this.conf = new Configuration(conf);
+  }
+
+  @Override
+  public Configuration getConf() {
+    return this.conf;
+  }
+
+  private static Namespace getNamespace(Datasets dataset) {
+    return Namespace.of(dataset.getDatasetReference().getDatasetId());
+  }
+
+  private DatasetReference toDatasetReference(Namespace namespace) {
+    validateNamespace(namespace);
+    return new DatasetReference().setProjectId(projectId).setDatasetId(namespace.level(0));
+  }
+
+  private TableReference toTableReference(TableIdentifier tableIdentifier) {
+    DatasetReference datasetReference = toDatasetReference(tableIdentifier.namespace());
+    return new TableReference()
+        .setProjectId(datasetReference.getProjectId())
+        .setDatasetId(datasetReference.getDatasetId())
+        .setTableId(tableIdentifier.name());
+  }
+
+  private static Map<String, String> getMetadata(Dataset dataset) {

Review Comment:
   ```suggestion
     privateMap<String, String> toMetadata(Dataset dataset) {
   ```



-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org

For queries about this service, please contact Infrastructure at:
users@infra.apache.org


---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For additional commands, e-mail: issues-help@iceberg.apache.org


Re: [PR] Catalog: Add BigQuery Metastore Catalog Support [iceberg]

Posted by "nastra (via GitHub)" <gi...@apache.org>.
nastra commented on code in PR #12808:
URL: https://github.com/apache/iceberg/pull/12808#discussion_r2054367736


##########
bigquery/src/main/java/org/apache/iceberg/gcp/bigquery/BigQueryMetastoreUtils.java:
##########
@@ -0,0 +1,97 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.iceberg.gcp.bigquery;
+
+import com.google.api.services.bigquery.model.ExternalCatalogDatasetOptions;
+import com.google.api.services.bigquery.model.ExternalCatalogTableOptions;
+import com.google.api.services.bigquery.model.SerDeInfo;
+import com.google.api.services.bigquery.model.StorageDescriptor;
+import java.util.Map;
+
+/** Shared utilities for BigQuery Metastore specific functions and constants. */
+public final class BigQueryMetastoreUtils {
+
+  private BigQueryMetastoreUtils() {}
+
+  public static final String HIVE_SERIALIZATION_LIBRARY =
+      "org.apache.iceberg.mr.hive.HiveIcebergSerDe";
+  public static final String HIVE_FILE_INPUT_FORMAT =
+      "org.apache.iceberg.mr.hive.HiveIcebergInputFormat";
+  public static final String HIVE_FILE_OUTPUT_FORMAT =
+      "org.apache.iceberg.mr.hive.HiveIcebergOutputFormat";
+
+  // TODO(b/341933455): Consider using "org.apache.iceberg.mr.hive.HiveIcebergSerDe" when

Review Comment:
   TBH I find those TODOs with that custom tag a bit confusing for readers of the code as it's like adding references to internal JIRA tickets. I understand that this is meant for Google engineers but I don't think stuff like this should be added to Iceberg becuse only a subset of people know what it actually refers to.



-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org

For queries about this service, please contact Infrastructure at:
users@infra.apache.org


---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For additional commands, e-mail: issues-help@iceberg.apache.org


Re: [PR] Catalog: Add BigQuery Metastore Catalog Support [iceberg]

Posted by "nastra (via GitHub)" <gi...@apache.org>.
nastra commented on code in PR #12808:
URL: https://github.com/apache/iceberg/pull/12808#discussion_r2055657656


##########
bigquery/src/main/java/org/apache/iceberg/gcp/bigquery/BigQueryMetastoreCatalog.java:
##########
@@ -0,0 +1,400 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.iceberg.gcp.bigquery;
+
+import com.google.api.services.bigquery.model.Dataset;
+import com.google.api.services.bigquery.model.DatasetList.Datasets;
+import com.google.api.services.bigquery.model.DatasetReference;
+import com.google.api.services.bigquery.model.ExternalCatalogDatasetOptions;
+import com.google.api.services.bigquery.model.TableReference;
+import com.google.cloud.ServiceOptions;
+import com.google.cloud.bigquery.BigQueryOptions;
+import java.io.IOException;
+import java.security.GeneralSecurityException;
+import java.util.List;
+import java.util.Locale;
+import java.util.Map;
+import java.util.Objects;
+import java.util.Set;
+import org.apache.hadoop.conf.Configurable;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.iceberg.BaseMetastoreCatalog;
+import org.apache.iceberg.CatalogProperties;
+import org.apache.iceberg.CatalogUtil;
+import org.apache.iceberg.TableMetadata;
+import org.apache.iceberg.TableOperations;
+import org.apache.iceberg.catalog.Namespace;
+import org.apache.iceberg.catalog.SupportsNamespaces;
+import org.apache.iceberg.catalog.TableIdentifier;
+import org.apache.iceberg.exceptions.AlreadyExistsException;
+import org.apache.iceberg.exceptions.NoSuchNamespaceException;
+import org.apache.iceberg.exceptions.NoSuchTableException;
+import org.apache.iceberg.exceptions.ServiceFailureException;
+import org.apache.iceberg.exceptions.ValidationException;
+import org.apache.iceberg.io.FileIO;
+import org.apache.iceberg.relocated.com.google.common.annotations.VisibleForTesting;
+import org.apache.iceberg.relocated.com.google.common.base.Preconditions;
+import org.apache.iceberg.relocated.com.google.common.base.Strings;
+import org.apache.iceberg.relocated.com.google.common.collect.ImmutableList;
+import org.apache.iceberg.relocated.com.google.common.collect.ImmutableMap;
+import org.apache.iceberg.relocated.com.google.common.collect.Maps;
+import org.apache.iceberg.util.LocationUtil;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/** Iceberg Bigquery Metastore Catalog implementation. */
+public final class BigQueryMetastoreCatalog extends BaseMetastoreCatalog
+    implements SupportsNamespaces, Configurable {
+
+  // User provided properties.
+  public static final String PROJECT_ID = "gcp-project";
+  public static final String GCP_LOCATION = "gcp-location";
+  public static final String FILTER_UNSUPPORTED_TABLES = "filter-unsupported-tables";
+  public static final String TESTING_ENABLED = "testing-enabled";
+
+  public static final String HIVE_METASTORE_WAREHOUSE_DIR = "hive.metastore.warehouse.dir";
+
+  private static final Logger LOG = LoggerFactory.getLogger(BigQueryMetastoreCatalog.class);
+
+  private static final String DEFAULT_GCP_LOCATION = "us";
+
+  private String catalogName;
+  private Map<String, String> catalogProperties;
+  private FileIO fileIO;
+  private Configuration conf;
+  private String projectId;
+  private String location;
+  private BigQueryMetastoreClient client;
+  private boolean filterUnsupportedTables;
+
+  // Must have a no-arg constructor to be dynamically loaded
+  // initialize(String name, Map<String, String> properties) will be called to complete
+  // initialization
+  public BigQueryMetastoreCatalog() {}
+
+  @Override
+  public void initialize(String inputName, Map<String, String> properties) {
+    Preconditions.checkArgument(
+        properties.containsKey(PROJECT_ID), "GCP project must be specified");
+
+    projectId = properties.get(PROJECT_ID);
+    location = properties.getOrDefault(GCP_LOCATION, DEFAULT_GCP_LOCATION);
+    boolean testingEnabled =
+        Boolean.parseBoolean(properties.getOrDefault(TESTING_ENABLED, "false"));
+
+    BigQueryOptions options =
+        BigQueryOptions.newBuilder()
+            .setProjectId(projectId)
+            .setLocation(location)
+            .setRetrySettings(ServiceOptions.getDefaultRetrySettings())
+            .build();
+
+    try {
+      if (testingEnabled) {
+        client = new FakeBigQueryMetastoreClient(options);
+
+      } else {
+        client = new BigQueryMetastoreClientImpl(options);
+      }
+    } catch (IOException e) {
+      throw new ServiceFailureException(e, "Creating BigQuery client failed");
+    } catch (GeneralSecurityException e) {
+      throw new ValidationException(e, "Creating BigQuery client failed due to a security issue");
+    }
+    initialize(inputName, properties, projectId, location, client);
+  }
+
+  @VisibleForTesting
+  void initialize(
+      String inputName,
+      Map<String, String> properties,
+      String initialProjectId,
+      String initialLocation,
+      BigQueryMetastoreClient bigQueryMetaStoreClient) {
+    Preconditions.checkArgument(bigQueryMetaStoreClient != null, "BigQuery client can not be null");
+    this.catalogName = inputName;
+    this.catalogProperties = ImmutableMap.copyOf(properties);
+    this.projectId = initialProjectId;
+    this.location = initialLocation;
+    this.client = bigQueryMetaStoreClient;
+
+    if (this.conf == null) {
+      LOG.warn("No configuration was set, using the default environment Configuration");
+      this.conf = new Configuration();
+    }
+
+    LOG.info("Using BigQuery Metastore Iceberg Catalog: {}", inputName);
+
+    if (properties.containsKey(CatalogProperties.WAREHOUSE_LOCATION)) {
+      // Iceberg always removes trailing slash to avoid paths like "<folder>//data" in file systems
+      // like s3.
+      this.conf.set(
+          HIVE_METASTORE_WAREHOUSE_DIR,
+          LocationUtil.stripTrailingSlash(properties.get(CatalogProperties.WAREHOUSE_LOCATION)));
+    }
+
+    String fileIoImpl =
+        properties.getOrDefault(
+            CatalogProperties.FILE_IO_IMPL, "org.apache.iceberg.hadoop.HadoopFileIO");
+    this.fileIO = CatalogUtil.loadFileIO(fileIoImpl, properties, conf);
+
+    this.filterUnsupportedTables =
+        Boolean.parseBoolean(properties.getOrDefault(FILTER_UNSUPPORTED_TABLES, "false"));
+  }
+
+  @Override
+  protected TableOperations newTableOps(TableIdentifier identifier) {
+    return new BigQueryTableOperations(
+        client,
+        fileIO,
+        projectId,
+        // Sometimes extensions have the namespace contain the table name too, so we are forced to
+        // allow invalid namespace and just take the first part here like other catalog
+        // implementations do.
+        identifier.namespace().level(0),
+        identifier.name(),
+        conf);
+  }
+
+  @Override
+  protected String defaultWarehouseLocation(TableIdentifier identifier) {
+    String locationUri = null;
+    DatasetReference datasetReference = toDatasetReference(identifier.namespace());
+    Dataset dataset = client.getDataset(datasetReference);
+    if (dataset != null && dataset.getExternalCatalogDatasetOptions() != null) {
+      locationUri = dataset.getExternalCatalogDatasetOptions().getDefaultStorageLocationUri();
+    }
+    return String.format(
+        "%s/%s",
+        Strings.isNullOrEmpty(locationUri)
+            ? datasetReference.getDatasetId()
+            : LocationUtil.stripTrailingSlash(locationUri),
+        identifier.name());
+  }
+
+  @Override
+  public List<TableIdentifier> listTables(Namespace namespace) {
+    validateNamespace(namespace);
+
+    return client.listTables(toDatasetReference(namespace), filterUnsupportedTables).stream()
+        .map(
+            table -> TableIdentifier.of(namespace.level(0), table.getTableReference().getTableId()))
+        .collect(ImmutableList.toImmutableList());
+  }
+
+  @Override
+  public boolean dropTable(TableIdentifier identifier, boolean purge) {
+    try {
+      TableOperations ops = newTableOps(identifier);
+      TableMetadata lastMetadata = ops.current();
+
+      client.deleteTable(toTableReference(identifier));
+
+      if (purge && lastMetadata != null) {
+        CatalogUtil.dropTableData(ops.io(), lastMetadata);
+      }
+    } catch (NoSuchTableException e) { // Not catching a NoSuchIcebergTableException on purpose
+      return false; // The documentation says just return false in this case
+    }
+    return true;
+  }
+
+  @Override
+  public void renameTable(TableIdentifier from, TableIdentifier to) {
+    validateNamespace(from.namespace());
+    validateNamespace(to.namespace());
+
+    try {
+      client.getDataset(toDatasetReference(to.namespace()));
+      client.getDataset(toDatasetReference(from.namespace()));
+    } catch (NoSuchNamespaceException e) {
+      throw e;
+    }
+
+    TableReference fromTableRef = toTableReference(from);
+    TableReference toTableRef = toTableReference(to);
+
+    // Check if the source table exists
+    try {
+      client.getTable(fromTableRef);
+    } catch (NoSuchTableException e) {
+      throw new NoSuchTableException("Table does not exist: %s", from.name());
+    }
+
+    // Check if the destination table already exists
+    try {
+      client.getTable(toTableRef);
+      // If getTable succeeds, the destination table exists
+      throw new AlreadyExistsException("Table already exists: %s", to.name());
+    } catch (NoSuchTableException e) {
+      // Destination table does not exist, proceed with throwing exception
+    }
+
+    if (!from.namespace().equals(to.namespace())) {

Review Comment:
   I would probably move this check up to when the other namespace checks are done



-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org

For queries about this service, please contact Infrastructure at:
users@infra.apache.org


---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For additional commands, e-mail: issues-help@iceberg.apache.org


Re: [PR] Catalog: Add BigQuery Metastore Catalog Support [iceberg]

Posted by "talatuyarer (via GitHub)" <gi...@apache.org>.
talatuyarer commented on code in PR #12808:
URL: https://github.com/apache/iceberg/pull/12808#discussion_r2056511962


##########
bigquery/src/test/java/org/apache/iceberg/gcp/bigquery/TestBigQueryCatalog.java:
##########
@@ -0,0 +1,673 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.iceberg.gcp.bigquery;
+
+import static org.apache.iceberg.CatalogUtil.ICEBERG_CATALOG_TYPE;
+import static org.apache.iceberg.CatalogUtil.ICEBERG_CATALOG_TYPE_BIGQUERY;
+import static org.apache.iceberg.gcp.bigquery.BigQueryMetastoreCatalog.PROJECT_ID;
+import static org.apache.iceberg.gcp.bigquery.BigQueryMetastoreCatalog.TESTING_ENABLED;
+import static org.apache.iceberg.types.Types.NestedField.optional;
+import static org.assertj.core.api.Assertions.assertThat;
+import static org.assertj.core.api.Assertions.assertThatThrownBy;
+
+import java.io.File;
+import java.util.List;
+import java.util.Map;
+import org.apache.iceberg.CatalogProperties;
+import org.apache.iceberg.CatalogUtil;
+import org.apache.iceberg.PartitionSpec;
+import org.apache.iceberg.Schema;
+import org.apache.iceberg.SortOrder;
+import org.apache.iceberg.Table;
+import org.apache.iceberg.TableUtil;
+import org.apache.iceberg.Transaction;
+import org.apache.iceberg.catalog.CatalogTests;
+import org.apache.iceberg.catalog.Namespace;
+import org.apache.iceberg.catalog.TableIdentifier;
+import org.apache.iceberg.exceptions.NoSuchNamespaceException;
+import org.apache.iceberg.exceptions.NoSuchTableException;
+import org.apache.iceberg.expressions.Expressions;
+import org.apache.iceberg.relocated.com.google.common.collect.ImmutableMap;
+import org.apache.iceberg.types.Types;
+import org.junit.jupiter.api.AfterEach;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Disabled;
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.io.TempDir;
+import org.junit.jupiter.params.ParameterizedTest;
+import org.junit.jupiter.params.provider.ValueSource;
+
+public class TestBigQueryCatalog extends CatalogTests<BigQueryMetastoreCatalog> {
+
+  private static final String GCP_PROJECT = "project-id";
+  private static final String CATALOG_ID = "catalog-name";
+
+  // another schema that is not the same
+  protected static final Schema OTHER_SCHEMA_OPTIONAL =
+      new Schema(optional(1, "some_id", Types.IntegerType.get()));
+
+  // Schema passed to create tables
+  protected static final Schema SCHEMA_OPTIONAL =
+      new Schema(
+          optional(3, "id", Types.IntegerType.get(), "unique ID 🤪"),
+          optional(4, "data", Types.StringType.get()));
+
+  // This is the actual schema for the table, with column IDs reassigned
+  protected static final Schema REPLACE_SCHEMA_OPTIONAL =
+      new Schema(
+          optional(2, "id", Types.IntegerType.get(), "unique ID 🤪"),
+          optional(3, "data", Types.StringType.get()));
+
+  protected static final PartitionSpec REPLACE_SPEC_OPTIONAL =
+      PartitionSpec.builderFor(REPLACE_SCHEMA_OPTIONAL).bucket("id", 16).withSpecId(1).build();
+
+  protected static final SortOrder REPLACE_WRITE_ORDER_OPTIONAL =
+      SortOrder.builderFor(REPLACE_SCHEMA_OPTIONAL)
+          .asc(Expressions.bucket("id", 16))
+          .asc("id")
+          .build();
+
+  @TempDir private File tempFolder;
+  private BigQueryMetastoreCatalog catalog;
+  private BigQueryMetastoreClient fakeBigQueryMetastoreClient;
+  private String warehouseLocation;
+
+  @BeforeEach
+  public void before() throws Exception {
+    catalog = initCatalog(CATALOG_ID, ImmutableMap.of());
+  }
+
+  @AfterEach
+  public void after() throws Exception {
+    // Drop all tables in all datasets
+    List<Namespace> namespaces = catalog().listNamespaces(Namespace.empty());
+    for (Namespace namespace : namespaces) {
+      List<TableIdentifier> tables = catalog().listTables(namespace);
+      for (TableIdentifier table : tables) {
+        try {
+          catalog().dropTable(table, true); // drop with purge
+        } catch (NoSuchTableException e) {
+          // Table might be dropped by a previous iteration, ignore
+        }
+      }
+
+      // Drop all datasets (except any initial ones)
+      try {
+        catalog().dropNamespace(namespace);
+      } catch (NoSuchNamespaceException e) {
+        // Namespace might be dropped by a previous iteration, ignore
+      }
+    }
+  }
+
+  @Override
+  protected boolean requiresNamespaceCreate() {
+    return true;
+  }
+
+  @Override
+  protected boolean supportsNamesWithSlashes() {
+    return false;
+  }
+
+  @Override
+  protected boolean supportsNamesWithDot() {
+    return false;
+  }
+
+  @Override
+  public BigQueryMetastoreCatalog catalog() {
+    return catalog;
+  }
+
+  @Override
+  protected BigQueryMetastoreCatalog initCatalog(
+      String catalogName, Map<String, String> additionalProperties) {
+
+    warehouseLocation = tempFolder.toPath().resolve("hive-warehouse").toString();
+
+    Map<String, String> properties =
+        Map.of(
+            ICEBERG_CATALOG_TYPE,
+            ICEBERG_CATALOG_TYPE_BIGQUERY,
+            PROJECT_ID,
+            GCP_PROJECT,
+            CatalogProperties.WAREHOUSE_LOCATION,
+            warehouseLocation,
+            TESTING_ENABLED,
+            "true",
+            CatalogProperties.TABLE_DEFAULT_PREFIX + "default-key1",
+            "catalog-default-key1",
+            CatalogProperties.TABLE_DEFAULT_PREFIX + "default-key2",
+            "catalog-default-key2",
+            CatalogProperties.TABLE_DEFAULT_PREFIX + "override-key3",
+            "catalog-default-key3",
+            CatalogProperties.TABLE_OVERRIDE_PREFIX + "override-key3",
+            "catalog-override-key3",
+            CatalogProperties.TABLE_OVERRIDE_PREFIX + "override-key4",
+            "catalog-override-key4");
+
+    BigQueryMetastoreCatalog tmpCatalog =
+        (BigQueryMetastoreCatalog)
+            CatalogUtil.buildIcebergCatalog(
+                catalogName,
+                ImmutableMap.<String, String>builder()
+                    .putAll(properties)
+                    .putAll(additionalProperties)
+                    .build(),
+                null);
+
+    return tmpCatalog;
+  }
+
+  @Test
+  public void testRenameTable() {
+    if (requiresNamespaceCreate()) {
+      catalog.createNamespace(NS);
+    }
+
+    assertThat(catalog.tableExists(TABLE))
+        .as("Source table should not exist before create")
+        .isFalse();
+
+    catalog.buildTable(TABLE, SCHEMA).create();
+    assertThat(catalog.tableExists(TABLE)).as("Table should exist after create").isTrue();
+
+    assertThat(catalog.tableExists(RENAMED_TABLE))
+        .as("Destination table should not exist before rename")
+        .isFalse();
+
+    assertThatThrownBy(() -> catalog.renameTable(TABLE, RENAMED_TABLE))
+        .isInstanceOf(UnsupportedOperationException.class)
+        .hasMessageContaining("Table rename operation is unsupported");
+
+    catalog.dropTable(TABLE);
+  }
+
+  @Disabled(
+      "BigQuery does not allow adding a REQUIRED column to an existing table schema. "
+          + "When this operation is attempted, an exception gets thrown.")
+  @Test
+  public void testOverrideTablePropertiesReplaceTransaction() {}
+
+  @Test
+  public void testConcurrentReplaceTransactionSchema() {
+
+    if (requiresNamespaceCreate()) {
+      catalog.createNamespace(NS);
+    }
+
+    Transaction transaction = catalog.buildTable(TABLE, OTHER_SCHEMA_OPTIONAL).createTransaction();
+    transaction.newFastAppend().appendFile(FILE_A).commit();
+    transaction.commitTransaction();
+
+    Table original = catalog.loadTable(TABLE);
+    assertFiles(original, FILE_A);
+
+    Transaction secondReplace =
+        catalog.buildTable(TABLE, OTHER_SCHEMA_OPTIONAL).replaceTransaction();
+    secondReplace.newFastAppend().appendFile(FILE_C).commit();
+
+    Transaction firstReplace = catalog.buildTable(TABLE, SCHEMA_OPTIONAL).replaceTransaction();
+    firstReplace.newFastAppend().appendFile(FILE_B).commit();
+    firstReplace.commitTransaction();
+
+    Table afterFirstReplace = catalog.loadTable(TABLE);
+    assertThat(afterFirstReplace.schema().asStruct())
+        .as("Table schema should match the new schema")
+        .isEqualTo(REPLACE_SCHEMA_OPTIONAL.asStruct());
+    assertUUIDsMatch(original, afterFirstReplace);
+    assertFiles(afterFirstReplace, FILE_B);
+
+    secondReplace.commitTransaction();
+
+    Table afterSecondReplace = catalog.loadTable(TABLE);
+    assertThat(afterSecondReplace.schema().asStruct())
+        .as("Table schema should match the original schema")
+        .isEqualTo(original.schema().asStruct());
+    assertUUIDsMatch(original, afterSecondReplace);
+    assertFiles(afterSecondReplace, FILE_C);
+  }
+
+  @Test
+  public void testCompleteCreateTransaction() {
+    if (requiresNamespaceCreate()) {
+      catalog.createNamespace(NS);
+    }
+
+    String tableLocation = warehouseLocation + "/tmp/ns/table";
+
+    Map<String, String> properties =
+        ImmutableMap.of("user", "someone", "created-at", "2022-02-25T00:38:19");
+    Transaction create =
+        catalog
+            .buildTable(TABLE, SCHEMA)
+            .withLocation(tableLocation)
+            .withPartitionSpec(SPEC)
+            .withSortOrder(WRITE_ORDER)
+            .withProperties(properties)
+            .createTransaction();
+
+    assertThat(catalog.tableExists(TABLE))
+        .as("Table should not exist after createTransaction")
+        .isFalse();
+
+    create.newFastAppend().appendFile(FILE_A).commit();
+
+    assertThat(catalog.tableExists(TABLE))
+        .as("Table should not exist after append commit")
+        .isFalse();
+
+    create.commitTransaction();
+
+    assertThat(catalog.tableExists(TABLE)).as("Table should exist after append commit").isTrue();
+
+    Table table = catalog.loadTable(TABLE);
+    assertThat(table.schema().asStruct())
+        .as("Table schema should match the new schema")
+        .isEqualTo(TABLE_SCHEMA.asStruct());
+    assertThat(table.spec().fields())
+        .as("Table should have create partition spec")
+        .isEqualTo(TABLE_SPEC.fields());
+    assertThat(table.sortOrder())
+        .as("Table should have create sort order")
+        .isEqualTo(TABLE_WRITE_ORDER);
+    assertThat(table.properties().entrySet())
+        .as("Table properties should be a superset of the requested properties")
+        .containsAll(properties.entrySet());
+    if (!overridesRequestedLocation()) {
+      assertThat(table.location())
+          .as("Table location should match requested")
+          .isEqualTo(tableLocation);
+    }
+    assertFiles(table, FILE_A);
+    assertFilesPartitionSpec(table);
+    assertPreviousMetadataFileCount(table, 0);
+  }
+
+  @Disabled(
+      "BigQuery does not allow adding a REQUIRED column to an existing table schema. "
+          + "When this operation is attempted, an exception gets thrown.")
+  @Test
+  public void testCreateOrReplaceReplaceTransactionReplace() {}
+
+  // TODO: Remove this test when BigQuery supports required field change
+  @Test
+  public void testCreateOrReplaceReplaceTransactionReplaceWithOptionalFields() {
+
+    if (requiresNamespaceCreate()) {
+      catalog.createNamespace(NS);
+    }
+
+    Table original = catalog.buildTable(TABLE, OTHER_SCHEMA_OPTIONAL).create();
+
+    assertThat(catalog.tableExists(TABLE))
+        .as("Table should exist before replaceTransaction")
+        .isTrue();
+
+    Transaction createOrReplace =
+        catalog.buildTable(TABLE, SCHEMA_OPTIONAL).createOrReplaceTransaction();
+
+    assertThat(catalog.tableExists(TABLE))
+        .as("Table should still exist after replaceTransaction")
+        .isTrue();
+
+    createOrReplace.newFastAppend().appendFile(FILE_A).commit();
+
+    // validate table has not changed
+    Table table = catalog.loadTable(TABLE);
+
+    assertThat(table.schema().asStruct())
+        .as("Table schema should match concurrent create")
+        .isEqualTo(OTHER_SCHEMA_OPTIONAL.asStruct());
+
+    assertUUIDsMatch(original, table);
+    assertNoFiles(table);
+
+    createOrReplace.commitTransaction();
+
+    // validate the table after replace
+    assertThat(catalog.tableExists(TABLE)).as("Table should exist after append commit").isTrue();
+    table.refresh(); // refresh should work with UUID validation
+
+    Table loaded = catalog.loadTable(TABLE);
+
+    assertThat(loaded.schema().asStruct())
+        .as("Table schema should match the new schema")
+        .isEqualTo(REPLACE_SCHEMA_OPTIONAL.asStruct());
+    assertUUIDsMatch(original, loaded);
+    assertFiles(loaded, FILE_A);
+    assertPreviousMetadataFileCount(loaded, 1);
+  }
+
+  @Disabled(
+      "BigQuery does not allow adding a REQUIRED column to an existing table schema. "
+          + "When this operation is attempted, an exception gets thrown.")
+  @Test
+  public void testConcurrentReplaceTransactionSchema2() {}
+
+  // TODO: Remove this test when BigQuery supports required field change
+  @Test
+  public void testConcurrentReplaceTransactionSchema2WithOptionalFields() {

Review Comment:
   These tests were also run against the Production BigQuery environment using the real BigQuery client. This helped identify a limitation within BigQuery itself. To work around this limitation, I've implemented optional field tests.



-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org

For queries about this service, please contact Infrastructure at:
users@infra.apache.org


---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For additional commands, e-mail: issues-help@iceberg.apache.org


Re: [PR] Catalog: Add BigQuery Metastore Catalog Support [iceberg]

Posted by "nastra (via GitHub)" <gi...@apache.org>.
nastra commented on code in PR #12808:
URL: https://github.com/apache/iceberg/pull/12808#discussion_r2054362591


##########
bigquery/src/main/java/org/apache/iceberg/gcp/bigquery/BigQueryMetastoreClientImpl.java:
##########
@@ -0,0 +1,581 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.iceberg.gcp.bigquery;
+
+import com.google.api.client.googleapis.javanet.GoogleNetHttpTransport;
+import com.google.api.client.http.HttpHeaders;
+import com.google.api.client.http.HttpResponse;
+import com.google.api.client.http.HttpResponseException;
+import com.google.api.client.http.HttpStatusCodes;
+import com.google.api.client.json.gson.GsonFactory;
+import com.google.api.client.util.Data;
+import com.google.api.services.bigquery.Bigquery;
+import com.google.api.services.bigquery.BigqueryScopes;
+import com.google.api.services.bigquery.model.Dataset;
+import com.google.api.services.bigquery.model.DatasetList;
+import com.google.api.services.bigquery.model.DatasetList.Datasets;
+import com.google.api.services.bigquery.model.DatasetReference;
+import com.google.api.services.bigquery.model.ExternalCatalogDatasetOptions;
+import com.google.api.services.bigquery.model.ExternalCatalogTableOptions;
+import com.google.api.services.bigquery.model.Table;
+import com.google.api.services.bigquery.model.TableList;
+import com.google.api.services.bigquery.model.TableList.Tables;
+import com.google.api.services.bigquery.model.TableReference;
+import com.google.api.services.bigquery.model.TableSchema;
+import com.google.auth.http.HttpCredentialsAdapter;
+import com.google.auth.oauth2.GoogleCredentials;
+import com.google.cloud.BaseServiceException;
+import com.google.cloud.ExceptionHandler;
+import com.google.cloud.bigquery.BigQueryErrorMessages;
+import com.google.cloud.bigquery.BigQueryOptions;
+import com.google.cloud.bigquery.BigQueryRetryConfig;
+import com.google.cloud.bigquery.BigQueryRetryHelper;
+import java.io.IOException;
+import java.nio.charset.StandardCharsets;
+import java.security.GeneralSecurityException;
+import java.util.List;
+import java.util.Locale;
+import java.util.Map;
+import java.util.Set;
+import java.util.stream.Collectors;
+import java.util.stream.Stream;
+import org.apache.iceberg.BaseMetastoreTableOperations;
+import org.apache.iceberg.exceptions.AlreadyExistsException;
+import org.apache.iceberg.exceptions.BadRequestException;
+import org.apache.iceberg.exceptions.ForbiddenException;
+import org.apache.iceberg.exceptions.NoSuchIcebergTableException;
+import org.apache.iceberg.exceptions.NoSuchNamespaceException;
+import org.apache.iceberg.exceptions.NoSuchTableException;
+import org.apache.iceberg.exceptions.NotAuthorizedException;
+import org.apache.iceberg.exceptions.RuntimeIOException;
+import org.apache.iceberg.exceptions.ServiceFailureException;
+import org.apache.iceberg.exceptions.ServiceUnavailableException;
+import org.apache.iceberg.exceptions.ValidationException;
+import org.apache.iceberg.relocated.com.google.common.base.Preconditions;
+import org.apache.iceberg.relocated.com.google.common.collect.Lists;
+import org.apache.iceberg.relocated.com.google.common.collect.Maps;
+
+/** A client of Google Bigquery Metastore functions over the BigQuery service. */
+public final class BigQueryMetastoreClientImpl implements BigQueryMetastoreClient {
+
+  public static final ExceptionHandler.Interceptor EXCEPTION_HANDLER_INTERCEPTOR =
+      new ExceptionHandler.Interceptor() {
+
+        private static final long serialVersionUID = -8429573486870467828L;
+
+        @Override
+        public RetryResult afterEval(Exception exception, RetryResult retryResult) {
+          return ExceptionHandler.Interceptor.RetryResult.CONTINUE_EVALUATION;
+        }
+
+        @Override
+        public RetryResult beforeEval(Exception exception) {
+          if (exception instanceof BaseServiceException) {
+            boolean retriable = ((BaseServiceException) exception).isRetryable();
+            return retriable
+                ? ExceptionHandler.Interceptor.RetryResult.RETRY
+                : ExceptionHandler.Interceptor.RetryResult.CONTINUE_EVALUATION;
+          }
+          return ExceptionHandler.Interceptor.RetryResult.CONTINUE_EVALUATION;
+        }
+      };
+
+  private final Bigquery client;
+  private final BigQueryOptions bigqueryOptions;
+
+  // Retry config with error messages and regex for rate limit exceeded errors.
+  private static final BigQueryRetryConfig DEFAULT_RETRY_CONFIG =
+      BigQueryRetryConfig.newBuilder()
+          .retryOnMessage(BigQueryErrorMessages.RATE_LIMIT_EXCEEDED_MSG)
+          .retryOnMessage(BigQueryErrorMessages.JOB_RATE_LIMIT_EXCEEDED_MSG)
+          .retryOnRegEx(BigQueryErrorMessages.RetryRegExPatterns.RATE_LIMIT_EXCEEDED_REGEX)
+          .build();
+
+  public static final ExceptionHandler BIGQUERY_EXCEPTION_HANDLER =
+      ExceptionHandler.newBuilder()
+          .abortOn(RuntimeException.class)
+          .retryOn(
+              java.net.ConnectException
+                  .class) // Retry on connection failures due to transient network issues.
+          .retryOn(
+              java.net.UnknownHostException
+                  .class) // Retry to recover from temporary DNS resolution failures.
+          .addInterceptors(EXCEPTION_HANDLER_INTERCEPTOR)
+          .build();
+
+  /** Constructs a client of the Google BigQuery service. */
+  public BigQueryMetastoreClientImpl(BigQueryOptions options)
+      throws IOException, GeneralSecurityException {
+    // Initialize client that will be used to send requests. This client only needs to be created
+    // once, and can be reused for multiple requests
+    HttpCredentialsAdapter httpCredentialsAdapter =
+        new HttpCredentialsAdapter(
+            GoogleCredentials.getApplicationDefault().createScoped(BigqueryScopes.all()));
+    this.client =
+        new Bigquery.Builder(
+                GoogleNetHttpTransport.newTrustedTransport(),
+                GsonFactory.getDefaultInstance(),
+                httpRequest -> {
+                  httpCredentialsAdapter.initialize(httpRequest);
+                  // Instead of throwing exceptions, analyze the HttpResponse object and inspect its
+                  // status code. This will allow BigQuery API errors to be converted into Iceberg
+                  // exceptions.
+                  httpRequest.setThrowExceptionOnExecuteError(false);
+                })
+            .setApplicationName("BigQuery Metastore Iceberg Catalog Plugin")
+            .build();
+    this.bigqueryOptions = options;
+  }
+
+  @Override
+  public Dataset createDataset(Dataset dataset) {
+    Dataset response = null;
+    try {
+      response =
+          BigQueryRetryHelper.runWithRetries(
+              () -> create(dataset),
+              bigqueryOptions.getRetrySettings(),
+              BIGQUERY_EXCEPTION_HANDLER,
+              bigqueryOptions.getClock(),
+              DEFAULT_RETRY_CONFIG);
+    } catch (BigQueryRetryHelper.BigQueryRetryHelperException e) {
+      handleBigQueryRetryException(e);
+    }
+    return response;
+  }
+
+  @SuppressWarnings("FormatStringAnnotation")
+  private Dataset create(Dataset dataset) {
+    try {
+      HttpResponse response =
+          client
+              .datasets()
+              .insert(dataset.getDatasetReference().getProjectId(), dataset)
+              .executeUnparsed();
+      return convertExceptionIfUnsuccessful(response).parseAs(Dataset.class);
+    } catch (IOException e) {
+      throw new RuntimeIOException(e);
+    } catch (AlreadyExistsException e) {
+      throw new AlreadyExistsException("Namespace already exists: " + dataset.getId());
+    }
+  }
+
+  @Override
+  @SuppressWarnings("FormatStringAnnotation")
+  public Dataset getDataset(DatasetReference datasetReference) {
+    try {
+      HttpResponse response =
+          client
+              .datasets()
+              .get(datasetReference.getProjectId(), datasetReference.getDatasetId())
+              .executeUnparsed();
+      if (response.getStatusCode() == HttpStatusCodes.STATUS_CODE_NOT_FOUND) {
+        throw new NoSuchNamespaceException(
+            "Namespace does not exist: " + datasetReference.getDatasetId());
+      }
+      return convertExceptionIfUnsuccessful(response).parseAs(Dataset.class);
+    } catch (IOException e) {
+      throw new RuntimeIOException(e);
+    }
+  }
+
+  @Override
+  @SuppressWarnings("FormatStringAnnotation")
+  public void deleteDataset(DatasetReference datasetReference) {
+    try {
+      HttpResponse response =
+          client
+              .datasets()
+              .delete(datasetReference.getProjectId(), datasetReference.getDatasetId())
+              .executeUnparsed();
+      if (response.getStatusCode() == HttpStatusCodes.STATUS_CODE_NOT_FOUND) {
+        throw new NoSuchNamespaceException(
+            "Namespace does not exist: " + datasetReference.getDatasetId());
+      }
+      convertExceptionIfUnsuccessful(response);
+    } catch (IOException e) {
+      throw new RuntimeIOException(e);
+    }
+  }
+
+  @Override
+  public Dataset setDatasetParameters(
+      DatasetReference datasetReference, Map<String, String> parameters) {
+    Dataset dataset = getDataset(datasetReference);
+    ExternalCatalogDatasetOptions externalCatalogDatasetOptions =
+        dataset.getExternalCatalogDatasetOptions() == null
+            ? new ExternalCatalogDatasetOptions()
+            : dataset.getExternalCatalogDatasetOptions();
+    Map<String, String> finalParameters =
+        externalCatalogDatasetOptions.getParameters() == null
+            ? Maps.newHashMap()
+            : externalCatalogDatasetOptions.getParameters();
+    finalParameters.putAll(parameters);
+
+    dataset.setExternalCatalogDatasetOptions(
+        externalCatalogDatasetOptions.setParameters(finalParameters));
+
+    return updateDataset(dataset);
+  }
+
+  @Override
+  public Dataset removeDatasetParameters(
+      DatasetReference datasetReference, Set<String> parameters) {
+    Dataset dataset = getDataset(datasetReference);
+    ExternalCatalogDatasetOptions externalCatalogDatasetOptions =
+        dataset.getExternalCatalogDatasetOptions() == null
+            ? new ExternalCatalogDatasetOptions()
+            : dataset.getExternalCatalogDatasetOptions();
+    Map<String, String> finalParameters =
+        externalCatalogDatasetOptions.getParameters() == null
+            ? Maps.newHashMap()
+            : externalCatalogDatasetOptions.getParameters();
+    parameters.forEach(finalParameters::remove);
+
+    dataset.setExternalCatalogDatasetOptions(
+        externalCatalogDatasetOptions.setParameters(finalParameters));
+
+    return updateDataset(dataset);
+  }
+
+  @Override
+  public List<Datasets> listDatasets(String projectId) {
+    try {
+      String nextPageToken = null;
+      List<Datasets> datasets = Lists.newArrayList();
+      do {
+        HttpResponse pageResponse =
+            client.datasets().list(projectId).setPageToken(nextPageToken).executeUnparsed();
+        DatasetList result =
+            convertExceptionIfUnsuccessful(pageResponse).parseAs(DatasetList.class);
+        nextPageToken = result.getNextPageToken();
+        if (result.getDatasets() != null) {
+          datasets.addAll(result.getDatasets());
+        }
+      } while (nextPageToken != null && !nextPageToken.isEmpty());
+      return datasets;
+    } catch (IOException e) {
+      throw new RuntimeIOException(e);
+    }
+  }
+
+  @Override
+  public Table createTable(Table table) {
+    // Ensure it is an Iceberg table supported by the BigQuery metastore catalog.
+    validateTable(table);
+    // TODO: Ensure table creation is idempotent when handling retries.
+    Table response = null;
+    try {
+      response =
+          BigQueryRetryHelper.runWithRetries(
+              () -> create(table),
+              bigqueryOptions.getRetrySettings(),
+              BIGQUERY_EXCEPTION_HANDLER,
+              bigqueryOptions.getClock(),
+              DEFAULT_RETRY_CONFIG);
+    } catch (BigQueryRetryHelper.BigQueryRetryHelperException e) {
+      handleBigQueryRetryException(e);
+    }
+    return response;
+  }
+
+  @SuppressWarnings("FormatStringAnnotation")
+  private Table create(Table table) {
+    try {
+      HttpResponse response =
+          client
+              .tables()
+              .insert(
+                  Preconditions.checkNotNull(table.getTableReference()).getProjectId(),
+                  Preconditions.checkNotNull(table.getTableReference()).getDatasetId(),
+                  table)
+              .executeUnparsed();
+      return convertExceptionIfUnsuccessful(response).parseAs(Table.class);
+    } catch (IOException e) {
+      throw new RuntimeIOException(e);
+    } catch (AlreadyExistsException e) {
+      throw new AlreadyExistsException(String.format("Table %s already exists", table), e);
+    }
+  }
+
+  @Override
+  @SuppressWarnings("FormatStringAnnotation")
+  public Table getTable(TableReference tableReference) {
+    try {
+      HttpResponse response =
+          client
+              .tables()
+              .get(
+                  tableReference.getProjectId(),
+                  tableReference.getDatasetId(),
+                  tableReference.getTableId())
+              .executeUnparsed();
+      if (response.getStatusCode() == HttpStatusCodes.STATUS_CODE_NOT_FOUND) {
+        throw new NoSuchTableException(response.getStatusMessage());
+      }
+      return validateTable(convertExceptionIfUnsuccessful(response).parseAs(Table.class));
+    } catch (IOException e) {
+      throw new RuntimeIOException(e);
+    }
+  }
+
+  @Override
+  @SuppressWarnings("FormatStringAnnotation")
+  public Table patchTable(TableReference tableReference, Table table) {
+    // Ensure it is an Iceberg table supported by the BQ metastore catalog.
+    validateTable(table);
+
+    ExternalCatalogTableOptions newExternalCatalogTableOptions =
+        new ExternalCatalogTableOptions()
+            .setStorageDescriptor(table.getExternalCatalogTableOptions().getStorageDescriptor())
+            .setConnectionId(table.getExternalCatalogTableOptions().getConnectionId())
+            .setParameters(table.getExternalCatalogTableOptions().getParameters());
+    Table updatedTable =
+        new Table()
+            .setExternalCatalogTableOptions(newExternalCatalogTableOptions)
+            // Must set the schema as null for using schema auto-detect.
+            .setSchema(Data.nullOf(TableSchema.class));
+
+    Table response = null;
+    try {
+      response =
+          BigQueryRetryHelper.runWithRetries(
+              () -> patch(tableReference, updatedTable, table.getEtag()),
+              bigqueryOptions.getRetrySettings(),
+              BIGQUERY_EXCEPTION_HANDLER,
+              bigqueryOptions.getClock(),
+              DEFAULT_RETRY_CONFIG);
+    } catch (BigQueryRetryHelper.BigQueryRetryHelperException e) {
+      handleBigQueryRetryException(e);
+    }
+    return response;
+  }
+
+  @SuppressWarnings("FormatStringAnnotation")
+  private Table patch(TableReference tableReference, Table table, String etag) {
+    try {
+      HttpResponse response =
+          client
+              .tables()
+              .patch(
+                  tableReference.getProjectId(),
+                  tableReference.getDatasetId(),
+                  tableReference.getTableId(),
+                  table)
+              .setRequestHeaders(new HttpHeaders().setIfMatch(etag))
+              .executeUnparsed();
+
+      if (response.getStatusCode() == HttpStatusCodes.STATUS_CODE_NOT_FOUND) {
+        String responseString = response.parseAsString();
+        if (responseString.toLowerCase(Locale.ENGLISH).contains("not found: connection")) {
+          throw new BadRequestException(responseString);
+        }
+        throw new NoSuchTableException(response.getStatusMessage());
+      }
+      return convertExceptionIfUnsuccessful(response).parseAs(Table.class);
+    } catch (IOException e) {
+      throw new RuntimeIOException(e);
+    }
+  }
+
+  @Override
+  @SuppressWarnings("FormatStringAnnotation")
+  public void deleteTable(TableReference tableReference) {
+    try {
+      getTable(tableReference); // Fetching it to validate it is a BigQuery Metastore table first
+
+      HttpResponse response =
+          client
+              .tables()
+              .delete(
+                  tableReference.getProjectId(),
+                  tableReference.getDatasetId(),
+                  tableReference.getTableId())
+              .executeUnparsed();
+
+      if (response.getStatusCode() == HttpStatusCodes.STATUS_CODE_NOT_FOUND) {
+        throw new NoSuchTableException(response.getStatusMessage());
+      }
+      convertExceptionIfUnsuccessful(response);
+    } catch (IOException e) {
+      throw new RuntimeIOException(e);
+    }
+  }
+
+  @Override
+  @SuppressWarnings("FormatStringAnnotation")
+  public List<Tables> listTables(
+      DatasetReference datasetReference, boolean filterUnsupportedTables) {
+    try {
+      String nextPageToken = null;
+      Stream<Tables> tablesStream = Stream.empty();
+      do {
+        HttpResponse pageResponse =
+            client
+                .tables()
+                .list(datasetReference.getProjectId(), datasetReference.getDatasetId())
+                .setPageToken(nextPageToken)
+                .executeUnparsed();
+        if (pageResponse.getStatusCode() == HttpStatusCodes.STATUS_CODE_NOT_FOUND) {
+          throw new NoSuchNamespaceException(pageResponse.getStatusMessage());
+        }
+        TableList result = convertExceptionIfUnsuccessful(pageResponse).parseAs(TableList.class);
+        nextPageToken = result.getNextPageToken();
+        List<Tables> tablesPage = result.getTables();
+        Stream<Tables> tablesPageStream =
+            tablesPage == null ? Stream.empty() : result.getTables().stream();
+        tablesStream = Stream.concat(tablesStream, tablesPageStream);
+      } while (nextPageToken != null && !nextPageToken.isEmpty());
+
+      // The server should return more metadata here (e.g. BigQuery Non Iceberg tables) to
+      // distinguish Iceberg
+      // tables for us to filter out those results since invoking `getTable` on them would
+      // correctly raise a `NoSuchIcebergTableException` for being inoperable by this plugin.
+      if (filterUnsupportedTables) {
+        tablesStream =
+            tablesStream
+                .parallel()
+                .filter(
+                    table -> {
+                      try {
+                        getTable(table.getTableReference());
+                      } catch (NoSuchTableException e) {
+                        return false;
+                      }
+                      return true;
+                    });
+      }
+
+      return tablesStream.collect(Collectors.toList());
+    } catch (IOException e) {
+      throw new RuntimeIOException(e);
+    }
+  }
+
+  @SuppressWarnings("FormatStringAnnotation")
+  private Dataset updateDataset(Dataset dataset) {
+    try {
+      HttpResponse response =
+          client
+              .datasets()
+              .update(
+                  Preconditions.checkNotNull(dataset.getDatasetReference()).getProjectId(),

Review Comment:
   IMO this should be rather `Preconditions.checkArgument(.. != null, errorMsg)` as it is clearer for a reader of the error msg to know why something isn't a valid argument. 



-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org

For queries about this service, please contact Infrastructure at:
users@infra.apache.org


---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For additional commands, e-mail: issues-help@iceberg.apache.org


Re: [PR] Catalog: Add BigQuery Metastore Catalog Support [iceberg]

Posted by "nastra (via GitHub)" <gi...@apache.org>.
nastra commented on code in PR #12808:
URL: https://github.com/apache/iceberg/pull/12808#discussion_r2055650817


##########
bigquery/src/main/java/org/apache/iceberg/gcp/bigquery/BigQueryMetastoreCatalog.java:
##########
@@ -0,0 +1,400 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.iceberg.gcp.bigquery;
+
+import com.google.api.services.bigquery.model.Dataset;
+import com.google.api.services.bigquery.model.DatasetList.Datasets;
+import com.google.api.services.bigquery.model.DatasetReference;
+import com.google.api.services.bigquery.model.ExternalCatalogDatasetOptions;
+import com.google.api.services.bigquery.model.TableReference;
+import com.google.cloud.ServiceOptions;
+import com.google.cloud.bigquery.BigQueryOptions;
+import java.io.IOException;
+import java.security.GeneralSecurityException;
+import java.util.List;
+import java.util.Locale;
+import java.util.Map;
+import java.util.Objects;
+import java.util.Set;
+import org.apache.hadoop.conf.Configurable;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.iceberg.BaseMetastoreCatalog;
+import org.apache.iceberg.CatalogProperties;
+import org.apache.iceberg.CatalogUtil;
+import org.apache.iceberg.TableMetadata;
+import org.apache.iceberg.TableOperations;
+import org.apache.iceberg.catalog.Namespace;
+import org.apache.iceberg.catalog.SupportsNamespaces;
+import org.apache.iceberg.catalog.TableIdentifier;
+import org.apache.iceberg.exceptions.AlreadyExistsException;
+import org.apache.iceberg.exceptions.NoSuchNamespaceException;
+import org.apache.iceberg.exceptions.NoSuchTableException;
+import org.apache.iceberg.exceptions.ServiceFailureException;
+import org.apache.iceberg.exceptions.ValidationException;
+import org.apache.iceberg.io.FileIO;
+import org.apache.iceberg.relocated.com.google.common.annotations.VisibleForTesting;
+import org.apache.iceberg.relocated.com.google.common.base.Preconditions;
+import org.apache.iceberg.relocated.com.google.common.base.Strings;
+import org.apache.iceberg.relocated.com.google.common.collect.ImmutableList;
+import org.apache.iceberg.relocated.com.google.common.collect.ImmutableMap;
+import org.apache.iceberg.relocated.com.google.common.collect.Maps;
+import org.apache.iceberg.util.LocationUtil;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/** Iceberg Bigquery Metastore Catalog implementation. */
+public final class BigQueryMetastoreCatalog extends BaseMetastoreCatalog
+    implements SupportsNamespaces, Configurable {
+
+  // User provided properties.
+  public static final String PROJECT_ID = "gcp-project";
+  public static final String GCP_LOCATION = "gcp-location";
+  public static final String FILTER_UNSUPPORTED_TABLES = "filter-unsupported-tables";
+  public static final String TESTING_ENABLED = "testing-enabled";
+
+  public static final String HIVE_METASTORE_WAREHOUSE_DIR = "hive.metastore.warehouse.dir";
+
+  private static final Logger LOG = LoggerFactory.getLogger(BigQueryMetastoreCatalog.class);
+
+  private static final String DEFAULT_GCP_LOCATION = "us";
+
+  private String catalogName;
+  private Map<String, String> catalogProperties;
+  private FileIO fileIO;
+  private Configuration conf;
+  private String projectId;
+  private String location;
+  private BigQueryMetastoreClient client;
+  private boolean filterUnsupportedTables;
+
+  // Must have a no-arg constructor to be dynamically loaded
+  // initialize(String name, Map<String, String> properties) will be called to complete
+  // initialization
+  public BigQueryMetastoreCatalog() {}
+
+  @Override
+  public void initialize(String inputName, Map<String, String> properties) {
+    Preconditions.checkArgument(
+        properties.containsKey(PROJECT_ID), "GCP project must be specified");
+
+    projectId = properties.get(PROJECT_ID);
+    location = properties.getOrDefault(GCP_LOCATION, DEFAULT_GCP_LOCATION);
+    boolean testingEnabled =
+        Boolean.parseBoolean(properties.getOrDefault(TESTING_ENABLED, "false"));
+
+    BigQueryOptions options =
+        BigQueryOptions.newBuilder()
+            .setProjectId(projectId)
+            .setLocation(location)
+            .setRetrySettings(ServiceOptions.getDefaultRetrySettings())
+            .build();
+
+    try {
+      if (testingEnabled) {
+        client = new FakeBigQueryMetastoreClient(options);
+
+      } else {
+        client = new BigQueryMetastoreClientImpl(options);
+      }
+    } catch (IOException e) {
+      throw new ServiceFailureException(e, "Creating BigQuery client failed");
+    } catch (GeneralSecurityException e) {
+      throw new ValidationException(e, "Creating BigQuery client failed due to a security issue");
+    }
+    initialize(inputName, properties, projectId, location, client);
+  }
+
+  @VisibleForTesting
+  void initialize(
+      String inputName,
+      Map<String, String> properties,
+      String initialProjectId,
+      String initialLocation,
+      BigQueryMetastoreClient bigQueryMetaStoreClient) {
+    Preconditions.checkArgument(bigQueryMetaStoreClient != null, "BigQuery client can not be null");
+    this.catalogName = inputName;
+    this.catalogProperties = ImmutableMap.copyOf(properties);
+    this.projectId = initialProjectId;
+    this.location = initialLocation;
+    this.client = bigQueryMetaStoreClient;
+
+    if (this.conf == null) {
+      LOG.warn("No configuration was set, using the default environment Configuration");
+      this.conf = new Configuration();
+    }
+
+    LOG.info("Using BigQuery Metastore Iceberg Catalog: {}", inputName);
+
+    if (properties.containsKey(CatalogProperties.WAREHOUSE_LOCATION)) {
+      // Iceberg always removes trailing slash to avoid paths like "<folder>//data" in file systems
+      // like s3.
+      this.conf.set(
+          HIVE_METASTORE_WAREHOUSE_DIR,
+          LocationUtil.stripTrailingSlash(properties.get(CatalogProperties.WAREHOUSE_LOCATION)));
+    }
+
+    String fileIoImpl =
+        properties.getOrDefault(
+            CatalogProperties.FILE_IO_IMPL, "org.apache.iceberg.hadoop.HadoopFileIO");
+    this.fileIO = CatalogUtil.loadFileIO(fileIoImpl, properties, conf);
+
+    this.filterUnsupportedTables =
+        Boolean.parseBoolean(properties.getOrDefault(FILTER_UNSUPPORTED_TABLES, "false"));
+  }
+
+  @Override
+  protected TableOperations newTableOps(TableIdentifier identifier) {
+    return new BigQueryTableOperations(
+        client,
+        fileIO,
+        projectId,
+        // Sometimes extensions have the namespace contain the table name too, so we are forced to
+        // allow invalid namespace and just take the first part here like other catalog
+        // implementations do.
+        identifier.namespace().level(0),
+        identifier.name(),
+        conf);
+  }
+
+  @Override
+  protected String defaultWarehouseLocation(TableIdentifier identifier) {
+    String locationUri = null;
+    DatasetReference datasetReference = toDatasetReference(identifier.namespace());
+    Dataset dataset = client.getDataset(datasetReference);
+    if (dataset != null && dataset.getExternalCatalogDatasetOptions() != null) {
+      locationUri = dataset.getExternalCatalogDatasetOptions().getDefaultStorageLocationUri();
+    }
+    return String.format(
+        "%s/%s",
+        Strings.isNullOrEmpty(locationUri)
+            ? datasetReference.getDatasetId()
+            : LocationUtil.stripTrailingSlash(locationUri),
+        identifier.name());
+  }
+
+  @Override
+  public List<TableIdentifier> listTables(Namespace namespace) {
+    validateNamespace(namespace);
+
+    return client.listTables(toDatasetReference(namespace), filterUnsupportedTables).stream()
+        .map(
+            table -> TableIdentifier.of(namespace.level(0), table.getTableReference().getTableId()))
+        .collect(ImmutableList.toImmutableList());
+  }
+
+  @Override
+  public boolean dropTable(TableIdentifier identifier, boolean purge) {
+    try {
+      TableOperations ops = newTableOps(identifier);
+      TableMetadata lastMetadata = ops.current();
+
+      client.deleteTable(toTableReference(identifier));
+
+      if (purge && lastMetadata != null) {
+        CatalogUtil.dropTableData(ops.io(), lastMetadata);
+      }
+    } catch (NoSuchTableException e) { // Not catching a NoSuchIcebergTableException on purpose
+      return false; // The documentation says just return false in this case
+    }
+    return true;
+  }
+
+  @Override
+  public void renameTable(TableIdentifier from, TableIdentifier to) {
+    validateNamespace(from.namespace());
+    validateNamespace(to.namespace());
+
+    try {

Review Comment:
   why not check if the namespaces actually exist via `namespaceExists(..)`?



-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org

For queries about this service, please contact Infrastructure at:
users@infra.apache.org


---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For additional commands, e-mail: issues-help@iceberg.apache.org


Re: [PR] Catalog: Add BigQuery Metastore Catalog Support [iceberg]

Posted by "talatuyarer (via GitHub)" <gi...@apache.org>.
talatuyarer commented on PR #12808:
URL: https://github.com/apache/iceberg/pull/12808#issuecomment-2816148094

   Thank you all @nastra @ebyhr @kravikumar @gkalra18  for your review. I addressed all your comments feel free to add more :) 


-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org

For queries about this service, please contact Infrastructure at:
users@infra.apache.org


---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For additional commands, e-mail: issues-help@iceberg.apache.org


Re: [PR] Catalog: Add BigQuery Metastore Catalog Support [iceberg]

Posted by "talatuyarer (via GitHub)" <gi...@apache.org>.
talatuyarer commented on code in PR #12808:
URL: https://github.com/apache/iceberg/pull/12808#discussion_r2051079331


##########
bigquery/src/main/java/org/apache/iceberg/gcp/bigquery/BigQueryMetaStoreClientImpl.java:
##########
@@ -0,0 +1,620 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.iceberg.gcp.bigquery;
+
+import com.google.api.client.googleapis.javanet.GoogleNetHttpTransport;
+import com.google.api.client.http.HttpHeaders;
+import com.google.api.client.http.HttpResponse;
+import com.google.api.client.http.HttpResponseException;
+import com.google.api.client.http.HttpStatusCodes;
+import com.google.api.client.json.gson.GsonFactory;
+import com.google.api.client.util.Data;
+import com.google.api.services.bigquery.Bigquery;
+import com.google.api.services.bigquery.BigqueryScopes;
+import com.google.api.services.bigquery.model.Dataset;
+import com.google.api.services.bigquery.model.DatasetList;
+import com.google.api.services.bigquery.model.DatasetList.Datasets;
+import com.google.api.services.bigquery.model.DatasetReference;
+import com.google.api.services.bigquery.model.ExternalCatalogDatasetOptions;
+import com.google.api.services.bigquery.model.ExternalCatalogTableOptions;
+import com.google.api.services.bigquery.model.Table;
+import com.google.api.services.bigquery.model.TableList;
+import com.google.api.services.bigquery.model.TableList.Tables;
+import com.google.api.services.bigquery.model.TableReference;
+import com.google.api.services.bigquery.model.TableSchema;
+import com.google.auth.http.HttpCredentialsAdapter;
+import com.google.auth.oauth2.GoogleCredentials;
+import com.google.cloud.BaseServiceException;
+import com.google.cloud.ExceptionHandler;
+import com.google.cloud.bigquery.BigQueryErrorMessages;
+import com.google.cloud.bigquery.BigQueryOptions;
+import com.google.cloud.bigquery.BigQueryRetryConfig;
+import com.google.cloud.bigquery.BigQueryRetryHelper;
+import java.io.IOException;
+import java.nio.charset.StandardCharsets;
+import java.security.GeneralSecurityException;
+import java.util.List;
+import java.util.Locale;
+import java.util.Map;
+import java.util.Set;
+import java.util.concurrent.Callable;
+import java.util.stream.Collectors;
+import java.util.stream.Stream;
+import org.apache.iceberg.BaseMetastoreTableOperations;
+import org.apache.iceberg.exceptions.AlreadyExistsException;
+import org.apache.iceberg.exceptions.BadRequestException;
+import org.apache.iceberg.exceptions.ForbiddenException;
+import org.apache.iceberg.exceptions.NoSuchIcebergTableException;
+import org.apache.iceberg.exceptions.NoSuchNamespaceException;
+import org.apache.iceberg.exceptions.NoSuchTableException;
+import org.apache.iceberg.exceptions.NotAuthorizedException;
+import org.apache.iceberg.exceptions.RuntimeIOException;
+import org.apache.iceberg.exceptions.ServiceFailureException;
+import org.apache.iceberg.exceptions.ServiceUnavailableException;
+import org.apache.iceberg.exceptions.ValidationException;
+import org.apache.iceberg.relocated.com.google.common.base.Preconditions;
+import org.apache.iceberg.relocated.com.google.common.collect.Lists;
+import org.apache.iceberg.relocated.com.google.common.collect.Maps;
+
+/** A client of Google Bigquery Metastore functions over the BigQuery service. */
+public final class BigQueryMetaStoreClientImpl implements BigQueryMetaStoreClient {
+
+  public static final ExceptionHandler.Interceptor EXCEPTION_HANDLER_INTERCEPTOR =
+      new ExceptionHandler.Interceptor() {
+
+        private static final long serialVersionUID = -8429573486870467828L;
+
+        @Override
+        public RetryResult afterEval(Exception exception, RetryResult retryResult) {
+          return ExceptionHandler.Interceptor.RetryResult.CONTINUE_EVALUATION;
+        }
+
+        @Override
+        public RetryResult beforeEval(Exception exception) {
+          if (exception instanceof BaseServiceException) {
+            boolean retriable = ((BaseServiceException) exception).isRetryable();
+            return retriable
+                ? ExceptionHandler.Interceptor.RetryResult.RETRY
+                : ExceptionHandler.Interceptor.RetryResult.CONTINUE_EVALUATION;
+          }
+          return ExceptionHandler.Interceptor.RetryResult.CONTINUE_EVALUATION;
+        }
+      };
+
+  private final Bigquery client;
+  private final BigQueryOptions bigqueryOptions;
+
+  // Retry config with error messages and regex for rate limit exceeded errors.
+  private static final BigQueryRetryConfig DEFAULT_RETRY_CONFIG =
+      BigQueryRetryConfig.newBuilder()
+          .retryOnMessage(BigQueryErrorMessages.RATE_LIMIT_EXCEEDED_MSG)
+          .retryOnMessage(BigQueryErrorMessages.JOB_RATE_LIMIT_EXCEEDED_MSG)
+          .retryOnRegEx(BigQueryErrorMessages.RetryRegExPatterns.RATE_LIMIT_EXCEEDED_REGEX)
+          .build();
+
+  public static final ExceptionHandler BIGQUERY_EXCEPTION_HANDLER =
+      ExceptionHandler.newBuilder()
+          .abortOn(RuntimeException.class)
+          .retryOn(java.net.ConnectException.class) // retry on Connection Exception
+          .retryOn(java.net.UnknownHostException.class) // retry on UnknownHostException
+          .addInterceptors(EXCEPTION_HANDLER_INTERCEPTOR)
+          .build();
+
+  /** Constructs a client of the Google BigQuery service. */
+  public BigQueryMetaStoreClientImpl(BigQueryOptions options)
+      throws IOException, GeneralSecurityException {
+    // Initialize client that will be used to send requests. This client only needs to be created
+    // once, and can be reused for multiple requests
+    HttpCredentialsAdapter httpCredentialsAdapter =
+        new HttpCredentialsAdapter(
+            GoogleCredentials.getApplicationDefault().createScoped(BigqueryScopes.all()));
+    this.client =
+        new Bigquery.Builder(
+                GoogleNetHttpTransport.newTrustedTransport(),
+                GsonFactory.getDefaultInstance(),
+                httpRequest -> {
+                  httpCredentialsAdapter.initialize(httpRequest);
+                  // Instead of throwing exceptions, analyze the HttpResponse object and inspect its
+                  // status code. This will allow BigQuery API errors to be converted into Iceberg
+                  // exceptions.
+                  httpRequest.setThrowExceptionOnExecuteError(false);
+                })
+            .setApplicationName("BigQuery Iceberg Catalog Plugin")
+            .build();
+    this.bigqueryOptions = options;
+  }
+
+  @Override
+  public Dataset createDataset(Dataset dataset) {
+    // TODO (b/399885863): Ensure Dataset creation is idempotent when handling retries.
+    Dataset response = null;
+    try {
+      response =
+          BigQueryRetryHelper.runWithRetries(
+              new Callable<Dataset>() {
+                @Override
+                public Dataset call() {
+                  return create(dataset);
+                }
+              },
+              bigqueryOptions.getRetrySettings(),
+              BIGQUERY_EXCEPTION_HANDLER,
+              bigqueryOptions.getClock(),
+              DEFAULT_RETRY_CONFIG);
+    } catch (BigQueryRetryHelper.BigQueryRetryHelperException e) {
+      handleBigQueryRetryException(e);
+    }
+    return response;
+  }
+
+  @SuppressWarnings("FormatStringAnnotation")
+  private Dataset create(Dataset dataset) {
+    try {
+      HttpResponse response =
+          client
+              .datasets()
+              .insert(dataset.getDatasetReference().getProjectId(), dataset)
+              .executeUnparsed();
+      return convertExceptionIfUnsuccessful(response).parseAs(Dataset.class);
+    } catch (IOException e) {
+      throw new RuntimeIOException(e);
+    } catch (AlreadyExistsException e) {
+      throw new AlreadyExistsException("Namespace already exists: " + dataset.getId());
+    }
+  }
+
+  @Override
+  @SuppressWarnings("FormatStringAnnotation")
+  public Dataset getDataset(DatasetReference datasetReference) {
+    try {
+      HttpResponse response =
+          client
+              .datasets()
+              .get(datasetReference.getProjectId(), datasetReference.getDatasetId())
+              .executeUnparsed();
+      if (response.getStatusCode() == HttpStatusCodes.STATUS_CODE_NOT_FOUND) {
+        throw new NoSuchNamespaceException(
+            "Namespace does not exist: " + datasetReference.getDatasetId());
+      }
+      return convertExceptionIfUnsuccessful(response).parseAs(Dataset.class);
+    } catch (IOException e) {
+      throw new RuntimeIOException(e);
+    }
+  }
+
+  @Override
+  @SuppressWarnings("FormatStringAnnotation")
+  public void deleteDataset(DatasetReference datasetReference) {
+    try {
+      HttpResponse response =
+          client
+              .datasets()
+              .delete(datasetReference.getProjectId(), datasetReference.getDatasetId())
+              .executeUnparsed();
+      if (response.getStatusCode() == HttpStatusCodes.STATUS_CODE_NOT_FOUND) {
+        throw new NoSuchNamespaceException(
+            "Namespace does not exist: " + datasetReference.getDatasetId());
+      }
+      convertExceptionIfUnsuccessful(response);
+    } catch (IOException e) {
+      throw new RuntimeIOException(e);
+    }
+  }
+
+  @Override
+  public Dataset setDatasetParameters(
+      DatasetReference datasetReference, Map<String, String> parameters) {
+    Dataset dataset = getDataset(datasetReference);
+    ExternalCatalogDatasetOptions externalCatalogDatasetOptions =
+        dataset.getExternalCatalogDatasetOptions() == null
+            ? new ExternalCatalogDatasetOptions()
+            : dataset.getExternalCatalogDatasetOptions();
+    Map<String, String> finalParameters =
+        externalCatalogDatasetOptions.getParameters() == null
+            ? Maps.newHashMap()
+            : externalCatalogDatasetOptions.getParameters();
+    finalParameters.putAll(parameters);
+
+    dataset.setExternalCatalogDatasetOptions(
+        externalCatalogDatasetOptions.setParameters(finalParameters));
+
+    return updateDataset(dataset);
+  }
+
+  @Override
+  public Dataset removeDatasetParameters(
+      DatasetReference datasetReference, Set<String> parameters) {
+    Dataset dataset = getDataset(datasetReference);
+    ExternalCatalogDatasetOptions externalCatalogDatasetOptions =
+        dataset.getExternalCatalogDatasetOptions() == null
+            ? new ExternalCatalogDatasetOptions()
+            : dataset.getExternalCatalogDatasetOptions();
+    Map<String, String> finalParameters =
+        externalCatalogDatasetOptions.getParameters() == null
+            ? Maps.newHashMap()
+            : externalCatalogDatasetOptions.getParameters();
+    parameters.forEach(finalParameters::remove);
+
+    dataset.setExternalCatalogDatasetOptions(
+        externalCatalogDatasetOptions.setParameters(finalParameters));
+
+    return updateDataset(dataset);
+  }
+
+  @Override
+  public List<Datasets> listDatasets(String projectId) {
+    try {
+      String nextPageToken = null;
+      List<Datasets> datasets = Lists.newArrayList();
+      do {
+        HttpResponse pageResponse =
+            client.datasets().list(projectId).setPageToken(nextPageToken).executeUnparsed();
+        DatasetList result =
+            convertExceptionIfUnsuccessful(pageResponse).parseAs(DatasetList.class);
+        nextPageToken = result.getNextPageToken();
+        if (result.getDatasets() != null) {
+          datasets.addAll(result.getDatasets());
+        }
+      } while (nextPageToken != null && !nextPageToken.isEmpty());
+      return datasets;
+    } catch (IOException e) {
+      throw new RuntimeIOException(e);
+    }
+  }
+
+  @Override
+  public Table createTable(Table table) {
+    // Ensure it is an Iceberg table supported by the BQ metastore catalog.
+    validateTable(table);
+    // TODO (b/399885863): Ensure table creation is idempotent when handling retries.
+    Table response = null;
+    try {
+      response =
+          BigQueryRetryHelper.runWithRetries(
+              new Callable<Table>() {
+                @Override
+                public Table call() {
+                  return create(table);
+                }
+              },
+              bigqueryOptions.getRetrySettings(),
+              BIGQUERY_EXCEPTION_HANDLER,
+              bigqueryOptions.getClock(),
+              DEFAULT_RETRY_CONFIG);
+    } catch (BigQueryRetryHelper.BigQueryRetryHelperException e) {
+      handleBigQueryRetryException(e);
+    }
+    return response;
+  }
+
+  @SuppressWarnings("FormatStringAnnotation")
+  private Table create(Table table) {
+    try {
+      HttpResponse response =
+          client
+              .tables()
+              .insert(
+                  Preconditions.checkNotNull(table.getTableReference()).getProjectId(),
+                  Preconditions.checkNotNull(table.getTableReference()).getDatasetId(),
+                  table)
+              .executeUnparsed();
+      return convertExceptionIfUnsuccessful(response).parseAs(Table.class);
+    } catch (IOException e) {
+      throw new RuntimeIOException(e);
+    } catch (AlreadyExistsException e) {
+      throw new AlreadyExistsException("Table already exists", e);
+    }
+  }
+
+  @Override
+  @SuppressWarnings("FormatStringAnnotation")
+  public Table getTable(TableReference tableReference) {
+    try {
+      HttpResponse response =
+          client
+              .tables()
+              .get(
+                  tableReference.getProjectId(),
+                  tableReference.getDatasetId(),
+                  tableReference.getTableId())
+              .executeUnparsed();
+      if (response.getStatusCode() == HttpStatusCodes.STATUS_CODE_NOT_FOUND) {
+        throw new NoSuchTableException(response.getStatusMessage());
+      }
+      return validateTable(convertExceptionIfUnsuccessful(response).parseAs(Table.class));
+    } catch (IOException e) {
+      throw new RuntimeIOException(e);
+    }
+  }
+
+  @Override
+  @SuppressWarnings("FormatStringAnnotation")
+  public Table patchTable(TableReference tableReference, Table table) {
+    // Ensure it is an Iceberg table supported by the BQ metastore catalog.
+    validateTable(table);
+
+    ExternalCatalogTableOptions newExternalCatalogTableOptions =
+        new ExternalCatalogTableOptions()
+            .setStorageDescriptor(table.getExternalCatalogTableOptions().getStorageDescriptor())
+            .setConnectionId(table.getExternalCatalogTableOptions().getConnectionId())
+            .setParameters(table.getExternalCatalogTableOptions().getParameters());
+    Table updatedTable =
+        new Table()
+            .setExternalCatalogTableOptions(newExternalCatalogTableOptions)
+            // Must set the schema as null for using schema auto-detect.
+            .setSchema(Data.nullOf(TableSchema.class));
+
+    Table response = null;
+    try {
+      response =
+          BigQueryRetryHelper.runWithRetries(
+              new Callable<Table>() {
+                @Override
+                public Table call() {
+                  return patch(tableReference, updatedTable, table.getEtag());
+                }
+              },
+              bigqueryOptions.getRetrySettings(),
+              BIGQUERY_EXCEPTION_HANDLER,
+              bigqueryOptions.getClock(),
+              DEFAULT_RETRY_CONFIG);
+    } catch (BigQueryRetryHelper.BigQueryRetryHelperException e) {
+      handleBigQueryRetryException(e);
+    }
+    return response;
+  }
+
+  @SuppressWarnings("FormatStringAnnotation")
+  private Table patch(TableReference tableReference, Table table, String etag) {
+    try {
+      HttpResponse response =
+          client
+              .tables()
+              .patch(
+                  tableReference.getProjectId(),
+                  tableReference.getDatasetId(),
+                  tableReference.getTableId(),
+                  table)
+              .setRequestHeaders(new HttpHeaders().setIfMatch(etag))
+              .executeUnparsed();
+
+      if (response.getStatusCode() == HttpStatusCodes.STATUS_CODE_NOT_FOUND) {
+        String responseString = response.parseAsString();
+        if (responseString.toLowerCase(Locale.ROOT).contains("not found: connection")) {
+          throw new BadRequestException(responseString);
+        }
+        throw new NoSuchTableException(response.getStatusMessage());
+      }
+      return convertExceptionIfUnsuccessful(response).parseAs(Table.class);
+    } catch (IOException e) {
+      throw new RuntimeIOException(e);
+    }
+  }
+
+  @Override
+  @SuppressWarnings("FormatStringAnnotation")
+  public Table renameTable(TableReference tableToRename, String newTableId) {
+    Table table = getTable(tableToRename); // Verify table first
+    Table patch =
+        new Table()
+            .setTableReference(
+                new TableReference()
+                    .setProjectId(table.getTableReference().getProjectId())
+                    .setDatasetId(table.getTableReference().getDatasetId())
+                    .setTableId(newTableId));
+
+    try {
+      HttpResponse response =
+          client
+              .tables()
+              .patch(
+                  tableToRename.getProjectId(),
+                  tableToRename.getDatasetId(),
+                  tableToRename.getTableId(),
+                  patch)
+              .setRequestHeaders(new HttpHeaders().setIfMatch(table.getEtag()))
+              .executeUnparsed();
+
+      if (response.getStatusCode() == HttpStatusCodes.STATUS_CODE_NOT_FOUND) {
+        throw new NoSuchTableException(response.getStatusMessage());
+      }
+      return convertExceptionIfUnsuccessful(response).parseAs(Table.class);
+    } catch (IOException e) {
+      throw new RuntimeIOException(e);
+    }
+  }
+
+  @Override
+  @SuppressWarnings("FormatStringAnnotation")
+  public void deleteTable(TableReference tableReference) {
+    try {
+      getTable(tableReference); // Fetching it to validate it is a BigQuery Metastore table first
+
+      HttpResponse response =
+          client
+              .tables()
+              .delete(
+                  tableReference.getProjectId(),
+                  tableReference.getDatasetId(),
+                  tableReference.getTableId())
+              .executeUnparsed();
+
+      if (response.getStatusCode() == HttpStatusCodes.STATUS_CODE_NOT_FOUND) {
+        throw new NoSuchTableException(response.getStatusMessage());
+      }
+      convertExceptionIfUnsuccessful(response);
+    } catch (IOException e) {
+      throw new RuntimeIOException(e);
+    }
+  }
+
+  @Override
+  @SuppressWarnings("FormatStringAnnotation")
+  public List<Tables> listTables(
+      DatasetReference datasetReference, boolean filterUnsupportedTables) {
+    try {
+      String nextPageToken = null;
+      Stream<Tables> tablesStream = Stream.empty();
+      do {
+        HttpResponse pageResponse =
+            client
+                .tables()
+                .list(datasetReference.getProjectId(), datasetReference.getDatasetId())
+                .setPageToken(nextPageToken)
+                .executeUnparsed();
+        if (pageResponse.getStatusCode() == HttpStatusCodes.STATUS_CODE_NOT_FOUND) {
+          throw new NoSuchNamespaceException(pageResponse.getStatusMessage());
+        }
+        TableList result = convertExceptionIfUnsuccessful(pageResponse).parseAs(TableList.class);
+        nextPageToken = result.getNextPageToken();
+        List<Tables> tablesPage = result.getTables();
+        Stream<Tables> tablesPageStream =
+            tablesPage == null ? Stream.empty() : result.getTables().stream();
+        tablesStream = Stream.concat(tablesStream, tablesPageStream);
+      } while (nextPageToken != null && !nextPageToken.isEmpty());
+
+      // TODO(b/345839927): The server should return more metadata here to distinguish Iceberg
+      // BQMS tables for us to filter out those results since invoking `getTable` on them would
+      // correctly raise a `NoSuchIcebergTableException` for being inoperable by this plugin.
+      if (filterUnsupportedTables) {
+        tablesStream =
+            tablesStream
+                .parallel()
+                .filter(
+                    table -> {
+                      try {
+                        getTable(table.getTableReference());
+                      } catch (NoSuchTableException e) {
+                        return false;
+                      }
+                      return true;
+                    });
+      }
+
+      return tablesStream.collect(Collectors.toList());
+    } catch (IOException e) {
+      throw new RuntimeIOException(e);
+    }
+  }
+
+  @SuppressWarnings("FormatStringAnnotation")
+  private Dataset updateDataset(Dataset dataset) {
+    try {
+      HttpResponse response =
+          client
+              .datasets()
+              .update(
+                  Preconditions.checkNotNull(dataset.getDatasetReference()).getProjectId(),
+                  Preconditions.checkNotNull(dataset.getDatasetReference().getDatasetId()),
+                  dataset)
+              .setRequestHeaders(new HttpHeaders().setIfMatch(dataset.getEtag()))
+              .executeUnparsed();
+      if (response.getStatusCode() == HttpStatusCodes.STATUS_CODE_NOT_FOUND) {
+        throw new NoSuchNamespaceException(response.getStatusMessage());
+      }
+      return convertExceptionIfUnsuccessful(response).parseAs(Dataset.class);
+    } catch (IOException e) {
+      throw new RuntimeIOException(e);
+    }
+  }
+
+  /**
+   * Returns true when it is a BigQuery Metastore Iceberg table, defined by having the
+   * ExternalCatalogTableOptions object and a parameter of METADATA_LOCATION_PROP as part of its
+   * parameters map.
+   *
+   * @param table to check
+   */
+  private boolean isValidIcebergTable(Table table) {
+    return table.getExternalCatalogTableOptions() != null
+        && !table.getExternalCatalogTableOptions().isEmpty()
+        && table.getExternalCatalogTableOptions().getParameters() != null
+        && table
+            .getExternalCatalogTableOptions()
+            .getParameters()
+            .containsKey(BaseMetastoreTableOperations.METADATA_LOCATION_PROP)
+        && BaseMetastoreTableOperations.ICEBERG_TABLE_TYPE_VALUE.equalsIgnoreCase(
+            table
+                .getExternalCatalogTableOptions()
+                .getParameters()
+                .get(BaseMetastoreTableOperations.TABLE_TYPE_PROP));

Review Comment:
   Done. It was very complex implementation. i changed validation to be more readable.



##########
bigquery/src/main/java/org/apache/iceberg/gcp/bigquery/BigQueryMetastoreCatalog.java:
##########
@@ -0,0 +1,403 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.iceberg.gcp.bigquery;
+
+import com.google.api.services.bigquery.model.Dataset;
+import com.google.api.services.bigquery.model.DatasetList.Datasets;
+import com.google.api.services.bigquery.model.DatasetReference;
+import com.google.api.services.bigquery.model.ExternalCatalogDatasetOptions;
+import com.google.api.services.bigquery.model.TableReference;
+import com.google.cloud.ServiceOptions;
+import com.google.cloud.bigquery.BigQueryOptions;
+import java.io.IOException;
+import java.security.GeneralSecurityException;
+import java.util.List;
+import java.util.Locale;
+import java.util.Map;
+import java.util.Objects;
+import java.util.Set;
+import org.apache.hadoop.conf.Configurable;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.iceberg.BaseMetastoreCatalog;
+import org.apache.iceberg.CatalogProperties;
+import org.apache.iceberg.CatalogUtil;
+import org.apache.iceberg.TableMetadata;
+import org.apache.iceberg.TableOperations;
+import org.apache.iceberg.catalog.Namespace;
+import org.apache.iceberg.catalog.SupportsNamespaces;
+import org.apache.iceberg.catalog.TableIdentifier;
+import org.apache.iceberg.exceptions.AlreadyExistsException;
+import org.apache.iceberg.exceptions.NoSuchNamespaceException;
+import org.apache.iceberg.exceptions.NoSuchTableException;
+import org.apache.iceberg.exceptions.ServiceFailureException;
+import org.apache.iceberg.exceptions.ValidationException;
+import org.apache.iceberg.io.FileIO;
+import org.apache.iceberg.relocated.com.google.common.annotations.VisibleForTesting;
+import org.apache.iceberg.relocated.com.google.common.base.Preconditions;
+import org.apache.iceberg.relocated.com.google.common.base.Strings;
+import org.apache.iceberg.relocated.com.google.common.collect.ImmutableList;
+import org.apache.iceberg.relocated.com.google.common.collect.ImmutableMap;
+import org.apache.iceberg.relocated.com.google.common.collect.Maps;
+import org.apache.iceberg.util.LocationUtil;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/** Iceberg Bigquery Metastore Catalog implementation. */
+public final class BigQueryMetastoreCatalog extends BaseMetastoreCatalog
+    implements SupportsNamespaces, Configurable {
+
+  // User provided properties.
+  public static final String PROPERTIES_KEY_GCP_PROJECT = "gcp-project";
+  public static final String PROPERTIES_KEY_GCP_LOCATION = "gcp-location";
+  public static final String PROPERTIES_KEY_FILTER_UNSUPPORTED_TABLES = "filter-unsupported-tables";
+  public static final String PROPERTIES_KEY_TESTING_ENABLED = "testing-enabled";
+
+  public static final String HIVE_METASTORE_WAREHOUSE_DIR = "hive.metastore.warehouse.dir";
+
+  private static final Logger LOG = LoggerFactory.getLogger(BigQueryMetastoreCatalog.class);
+
+  private static final String DEFAULT_GCP_LOCATION = "us";
+
+  private String catalogPluginName;
+  private Map<String, String> catalogProperties;
+  private FileIO fileIO;
+  private Configuration conf;
+  private String projectId;
+  private String location;
+  private BigQueryMetaStoreClient client;
+  private boolean filterUnsupportedTables;
+
+  // Must have a no-arg constructor to be dynamically loaded
+  // initialize(String name, Map<String, String> properties) will be called to complete
+  // initialization
+  public BigQueryMetastoreCatalog() {}
+
+  @Override
+  public void initialize(String inputName, Map<String, String> properties) {
+    if (!properties.containsKey(PROPERTIES_KEY_GCP_PROJECT)) {
+      throw new ValidationException("GCP project must be specified");
+    }
+    projectId = properties.get(PROPERTIES_KEY_GCP_PROJECT);
+    location = properties.getOrDefault(PROPERTIES_KEY_GCP_LOCATION, DEFAULT_GCP_LOCATION);
+    boolean testingEnabled =
+        Boolean.parseBoolean(properties.getOrDefault(PROPERTIES_KEY_TESTING_ENABLED, "false"));
+
+    BigQueryOptions options =
+        BigQueryOptions.newBuilder()
+            .setProjectId(projectId)
+            .setLocation(location)
+            .setRetrySettings(ServiceOptions.getDefaultRetrySettings())
+            .build();
+
+    try {
+      if (testingEnabled) {
+        client = new FakeBigQueryMetaStoreClient(options);
+
+      } else {
+        client = new BigQueryMetaStoreClientImpl(options);
+      }
+    } catch (IOException e) {
+      throw new ServiceFailureException(e, "Creating BigQuery client failed");
+    } catch (GeneralSecurityException e) {
+      throw new ValidationException(e, "Creating BigQuery client failed due to a security issue");
+    }
+    initialize(inputName, properties, projectId, location, client);
+  }
+
+  @VisibleForTesting
+  void initialize(
+      String inputName,
+      Map<String, String> properties,
+      String initialProjectId,
+      String initialLocation,
+      BigQueryMetaStoreClient bigQueryMetaStoreClient) {
+    this.catalogPluginName = inputName;
+    this.catalogProperties = ImmutableMap.copyOf(properties);
+    this.projectId = initialProjectId;
+    this.location = initialLocation;
+    this.client =
+        Preconditions.checkNotNull(bigQueryMetaStoreClient, "BigQuery client can not be null");
+
+    if (this.conf == null) {
+      LOG.warn("No configuration was set, using the default environment Configuration");
+      this.conf = new Configuration();
+    }
+
+    LOG.info("Using BigQuery Metastore Iceberg Catalog: {}", inputName);
+
+    if (properties.containsKey(CatalogProperties.WAREHOUSE_LOCATION)) {
+      // Iceberg always removes trailing slash to avoid paths like "<folder>//data" in file systems
+      // like s3.
+      this.conf.set(
+          HIVE_METASTORE_WAREHOUSE_DIR,
+          LocationUtil.stripTrailingSlash(properties.get(CatalogProperties.WAREHOUSE_LOCATION)));
+    }
+
+    String fileIoImpl =
+        properties.getOrDefault(
+            CatalogProperties.FILE_IO_IMPL, "org.apache.iceberg.hadoop.HadoopFileIO");
+    this.fileIO = CatalogUtil.loadFileIO(fileIoImpl, properties, conf);
+
+    this.filterUnsupportedTables =
+        Boolean.parseBoolean(
+            properties.getOrDefault(PROPERTIES_KEY_FILTER_UNSUPPORTED_TABLES, "false"));
+  }
+
+  @Override
+  protected TableOperations newTableOps(TableIdentifier identifier) {
+    return new BigQueryTableOperations(
+        client,
+        fileIO,
+        projectId,
+        // Sometimes extensions have the namespace contain the table name too, so we are forced to
+        // allow invalid namespace and just take the first part here like other catalog
+        // implementations do.
+        identifier.namespace().level(0),
+        identifier.name(),
+        conf);
+  }
+
+  @Override
+  protected String defaultWarehouseLocation(TableIdentifier identifier) {
+    String locationUri = null;
+    DatasetReference datasetReference = toDatasetReference(identifier.namespace());
+    Dataset dataset = client.getDataset(datasetReference);
+    if (dataset != null && dataset.getExternalCatalogDatasetOptions() != null) {
+      locationUri = dataset.getExternalCatalogDatasetOptions().getDefaultStorageLocationUri();
+    }
+    return String.format(
+        "%s/%s",
+        Strings.isNullOrEmpty(locationUri) ? datasetReference.getDatasetId() : locationUri,
+        identifier.name());
+  }
+
+  @Override
+  public List<TableIdentifier> listTables(Namespace namespace) {
+    validateNamespace(namespace);
+
+    return client.listTables(toDatasetReference(namespace), filterUnsupportedTables).stream()
+        .map(
+            table -> TableIdentifier.of(namespace.level(0), table.getTableReference().getTableId()))
+        .collect(ImmutableList.toImmutableList());
+  }
+
+  @Override
+  public boolean dropTable(TableIdentifier identifier, boolean purge) {
+    try {
+      TableOperations ops = newTableOps(identifier);
+      TableMetadata lastMetadata = ops.current();
+
+      client.deleteTable(toTableReference(identifier));
+
+      if (purge && lastMetadata != null) {
+        CatalogUtil.dropTableData(ops.io(), lastMetadata);
+      }
+    } catch (NoSuchTableException e) { // Not catching a NoSuchIcebergTableException on purpose
+      return false; // The documentation says just return false in this case
+    }
+    return true;
+  }
+
+  @Override
+  public void renameTable(TableIdentifier from, TableIdentifier to) {
+    validateNamespace(from.namespace());
+    validateNamespace(to.namespace());
+
+    try {
+      client.getDataset(toDatasetReference(to.namespace()));
+      client.getDataset(toDatasetReference(from.namespace()));
+    } catch (NoSuchNamespaceException e) {
+      throw e;
+    }
+
+    TableReference fromTableRef = toTableReference(from);
+    TableReference toTableRef = toTableReference(to);
+
+    // Check if the source table exists
+    try {
+      client.getTable(fromTableRef);
+    } catch (NoSuchTableException e) {
+      throw new NoSuchTableException("Table does not exist: %s", from.name());
+    }
+
+    // Check if the destination table already exists
+    try {
+      client.getTable(toTableRef);
+      // If getTable succeeds, the destination table exists
+      throw new AlreadyExistsException("Table already exists: %s", to.name());
+    } catch (NoSuchTableException e) {
+      // Destination table does not exist, proceed with throwing exception
+    }
+
+    if (!from.namespace().equals(to.namespace())) {
+      throw new ValidationException("New table name must be in the same namespace");
+    }
+
+    // TODO(b/354981675): Enable once supported by the API.
+    throw new UnsupportedOperationException(
+        String.format(
+            "Table rename operation is unsupported. Try the SQL operation directly on BigQuery: \"ALTER TABLE %s RENAME TO %s;\"",

Review Comment:
   Done



-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org

For queries about this service, please contact Infrastructure at:
users@infra.apache.org


---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For additional commands, e-mail: issues-help@iceberg.apache.org


Re: [PR] Catalog: Add BigQuery Metastore Catalog Support [iceberg]

Posted by "nastra (via GitHub)" <gi...@apache.org>.
nastra commented on code in PR #12808:
URL: https://github.com/apache/iceberg/pull/12808#discussion_r2066018919


##########
bigquery/src/main/java/org/apache/iceberg/gcp/bigquery/BigQueryMetastoreUtils.java:
##########
@@ -0,0 +1,95 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.iceberg.gcp.bigquery;
+
+import com.google.api.services.bigquery.model.ExternalCatalogDatasetOptions;
+import com.google.api.services.bigquery.model.ExternalCatalogTableOptions;
+import com.google.api.services.bigquery.model.SerDeInfo;
+import com.google.api.services.bigquery.model.StorageDescriptor;
+import java.util.Map;
+
+/** Shared utilities for BigQuery Metastore specific functions and constants. */
+public final class BigQueryMetastoreUtils {
+
+  private BigQueryMetastoreUtils() {}
+
+  public static final String HIVE_SERIALIZATION_LIBRARY =

Review Comment:
   do these need to be public or can they be private? I don't see any usages outside of this class



-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org

For queries about this service, please contact Infrastructure at:
users@infra.apache.org


---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For additional commands, e-mail: issues-help@iceberg.apache.org


Re: [PR] Catalog: Add BigQuery Metastore Catalog Support [iceberg]

Posted by "talatuyarer (via GitHub)" <gi...@apache.org>.
talatuyarer commented on code in PR #12808:
URL: https://github.com/apache/iceberg/pull/12808#discussion_r2067703701


##########
bigquery/src/main/java/org/apache/iceberg/gcp/bigquery/BigQueryMetastoreClientImpl.java:
##########
@@ -0,0 +1,594 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.iceberg.gcp.bigquery;
+
+import com.google.api.client.googleapis.javanet.GoogleNetHttpTransport;
+import com.google.api.client.http.HttpHeaders;
+import com.google.api.client.http.HttpResponse;
+import com.google.api.client.http.HttpResponseException;
+import com.google.api.client.http.HttpStatusCodes;
+import com.google.api.client.json.gson.GsonFactory;
+import com.google.api.client.util.Data;
+import com.google.api.services.bigquery.Bigquery;
+import com.google.api.services.bigquery.BigqueryScopes;
+import com.google.api.services.bigquery.model.Dataset;
+import com.google.api.services.bigquery.model.DatasetList;
+import com.google.api.services.bigquery.model.DatasetList.Datasets;
+import com.google.api.services.bigquery.model.DatasetReference;
+import com.google.api.services.bigquery.model.ExternalCatalogDatasetOptions;
+import com.google.api.services.bigquery.model.ExternalCatalogTableOptions;
+import com.google.api.services.bigquery.model.Table;
+import com.google.api.services.bigquery.model.TableList;
+import com.google.api.services.bigquery.model.TableList.Tables;
+import com.google.api.services.bigquery.model.TableReference;
+import com.google.api.services.bigquery.model.TableSchema;
+import com.google.auth.http.HttpCredentialsAdapter;
+import com.google.auth.oauth2.GoogleCredentials;
+import com.google.cloud.BaseServiceException;
+import com.google.cloud.ExceptionHandler;
+import com.google.cloud.bigquery.BigQueryErrorMessages;
+import com.google.cloud.bigquery.BigQueryOptions;
+import com.google.cloud.bigquery.BigQueryRetryConfig;
+import com.google.cloud.bigquery.BigQueryRetryHelper;
+import java.io.IOException;
+import java.nio.charset.StandardCharsets;
+import java.security.GeneralSecurityException;
+import java.util.List;
+import java.util.Locale;
+import java.util.Map;
+import java.util.Set;
+import java.util.stream.Collectors;
+import java.util.stream.Stream;
+import org.apache.iceberg.BaseMetastoreTableOperations;
+import org.apache.iceberg.exceptions.AlreadyExistsException;
+import org.apache.iceberg.exceptions.BadRequestException;
+import org.apache.iceberg.exceptions.ForbiddenException;
+import org.apache.iceberg.exceptions.NoSuchIcebergTableException;
+import org.apache.iceberg.exceptions.NoSuchNamespaceException;
+import org.apache.iceberg.exceptions.NoSuchTableException;
+import org.apache.iceberg.exceptions.NotAuthorizedException;
+import org.apache.iceberg.exceptions.RuntimeIOException;
+import org.apache.iceberg.exceptions.ServiceFailureException;
+import org.apache.iceberg.exceptions.ServiceUnavailableException;
+import org.apache.iceberg.exceptions.ValidationException;
+import org.apache.iceberg.relocated.com.google.common.base.Preconditions;
+import org.apache.iceberg.relocated.com.google.common.collect.Lists;
+import org.apache.iceberg.relocated.com.google.common.collect.Maps;
+
+/** A client of Google Bigquery Metastore functions over the BigQuery service. */
+public final class BigQueryMetastoreClientImpl implements BigQueryMetastoreClient {
+
+  public static final ExceptionHandler.Interceptor EXCEPTION_HANDLER_INTERCEPTOR =
+      new ExceptionHandler.Interceptor() {
+
+        private static final long serialVersionUID = -8429573486870467828L;
+
+        @Override
+        public RetryResult afterEval(Exception exception, RetryResult retryResult) {
+          return ExceptionHandler.Interceptor.RetryResult.CONTINUE_EVALUATION;
+        }
+
+        @Override
+        public RetryResult beforeEval(Exception exception) {
+          if (exception instanceof BaseServiceException) {
+            boolean retriable = ((BaseServiceException) exception).isRetryable();
+            return retriable
+                ? ExceptionHandler.Interceptor.RetryResult.RETRY
+                : ExceptionHandler.Interceptor.RetryResult.CONTINUE_EVALUATION;
+          }
+
+          return ExceptionHandler.Interceptor.RetryResult.CONTINUE_EVALUATION;
+        }
+      };
+
+  private final Bigquery client;
+  private final BigQueryOptions bigqueryOptions;
+
+  // Retry config with error messages and regex for rate limit exceeded errors.
+  private static final BigQueryRetryConfig DEFAULT_RETRY_CONFIG =
+      BigQueryRetryConfig.newBuilder()
+          .retryOnMessage(BigQueryErrorMessages.RATE_LIMIT_EXCEEDED_MSG)
+          .retryOnMessage(BigQueryErrorMessages.JOB_RATE_LIMIT_EXCEEDED_MSG)
+          .retryOnRegEx(BigQueryErrorMessages.RetryRegExPatterns.RATE_LIMIT_EXCEEDED_REGEX)
+          .build();
+
+  public static final ExceptionHandler BIGQUERY_EXCEPTION_HANDLER =
+      ExceptionHandler.newBuilder()
+          .abortOn(RuntimeException.class)
+          .retryOn(
+              java.net.ConnectException
+                  .class) // Retry on connection failures due to transient network issues.
+          .retryOn(
+              java.net.UnknownHostException
+                  .class) // Retry to recover from temporary DNS resolution failures.
+          .addInterceptors(EXCEPTION_HANDLER_INTERCEPTOR)
+          .build();
+
+  /** Constructs a client of the Google BigQuery service. */
+  public BigQueryMetastoreClientImpl(BigQueryOptions options)
+      throws IOException, GeneralSecurityException {
+    // Initialize client that will be used to send requests. This client only needs to be created
+    // once, and can be reused for multiple requests
+    HttpCredentialsAdapter httpCredentialsAdapter =
+        new HttpCredentialsAdapter(
+            GoogleCredentials.getApplicationDefault().createScoped(BigqueryScopes.all()));
+    this.client =
+        new Bigquery.Builder(
+                GoogleNetHttpTransport.newTrustedTransport(),
+                GsonFactory.getDefaultInstance(),
+                httpRequest -> {
+                  httpCredentialsAdapter.initialize(httpRequest);
+                  // Instead of throwing exceptions, analyze the HttpResponse object and inspect its
+                  // status code. This will allow BigQuery API errors to be converted into Iceberg
+                  // exceptions.
+                  httpRequest.setThrowExceptionOnExecuteError(false);
+                })
+            .setApplicationName("BigQuery Metastore Iceberg Catalog Plugin")
+            .build();
+    this.bigqueryOptions = options;
+  }
+
+  @Override
+  public Dataset create(Dataset dataset) {
+    Dataset response = null;
+    try {
+      response =
+          BigQueryRetryHelper.runWithRetries(
+              () -> internalCreate(dataset),
+              bigqueryOptions.getRetrySettings(),
+              BIGQUERY_EXCEPTION_HANDLER,
+              bigqueryOptions.getClock(),
+              DEFAULT_RETRY_CONFIG);
+    } catch (BigQueryRetryHelper.BigQueryRetryHelperException e) {
+      handleBigQueryRetryException(e);
+    }
+    return response;
+  }
+
+  @SuppressWarnings("FormatStringAnnotation")
+  private Dataset internalCreate(Dataset dataset) {
+    try {
+      HttpResponse response =
+          client
+              .datasets()
+              .insert(dataset.getDatasetReference().getProjectId(), dataset)
+              .executeUnparsed();
+      return convertExceptionIfUnsuccessful(response).parseAs(Dataset.class);
+    } catch (IOException e) {
+      throw new RuntimeIOException(e);
+    } catch (AlreadyExistsException e) {
+      throw new AlreadyExistsException("Namespace already exists: " + dataset.getId());
+    }
+  }
+
+  @Override
+  @SuppressWarnings("FormatStringAnnotation")
+  public Dataset load(DatasetReference datasetReference) {
+    try {
+      HttpResponse response =
+          client
+              .datasets()
+              .get(datasetReference.getProjectId(), datasetReference.getDatasetId())
+              .executeUnparsed();
+      if (response.getStatusCode() == HttpStatusCodes.STATUS_CODE_NOT_FOUND) {
+        throw new NoSuchNamespaceException(
+            "Namespace does not exist: " + datasetReference.getDatasetId());
+      }
+
+      return convertExceptionIfUnsuccessful(response).parseAs(Dataset.class);
+    } catch (IOException e) {
+      throw new RuntimeIOException(e);
+    }
+  }
+
+  @Override
+  @SuppressWarnings("FormatStringAnnotation")
+  public void delete(DatasetReference datasetReference) {
+    try {
+      HttpResponse response =
+          client
+              .datasets()
+              .delete(datasetReference.getProjectId(), datasetReference.getDatasetId())
+              .executeUnparsed();
+      if (response.getStatusCode() == HttpStatusCodes.STATUS_CODE_NOT_FOUND) {
+        throw new NoSuchNamespaceException(
+            "Namespace does not exist: " + datasetReference.getDatasetId());
+      }
+
+      convertExceptionIfUnsuccessful(response);
+    } catch (IOException e) {
+      throw new RuntimeIOException(e);
+    }
+  }
+
+  @Override
+  public Dataset setParameters(DatasetReference datasetReference, Map<String, String> parameters) {
+    Dataset dataset = load(datasetReference);
+    ExternalCatalogDatasetOptions externalCatalogDatasetOptions =
+        dataset.getExternalCatalogDatasetOptions() == null
+            ? new ExternalCatalogDatasetOptions()
+            : dataset.getExternalCatalogDatasetOptions();
+    Map<String, String> finalParameters =
+        externalCatalogDatasetOptions.getParameters() == null
+            ? Maps.newHashMap()
+            : externalCatalogDatasetOptions.getParameters();
+    finalParameters.putAll(parameters);
+
+    dataset.setExternalCatalogDatasetOptions(
+        externalCatalogDatasetOptions.setParameters(finalParameters));
+
+    return internalUpdate(dataset);
+  }
+
+  @Override
+  public Dataset removeParameters(DatasetReference datasetReference, Set<String> parameters) {
+    Dataset dataset = load(datasetReference);
+    ExternalCatalogDatasetOptions externalCatalogDatasetOptions =
+        dataset.getExternalCatalogDatasetOptions() == null
+            ? new ExternalCatalogDatasetOptions()
+            : dataset.getExternalCatalogDatasetOptions();
+    Map<String, String> finalParameters =
+        externalCatalogDatasetOptions.getParameters() == null
+            ? Maps.newHashMap()
+            : externalCatalogDatasetOptions.getParameters();
+    parameters.forEach(finalParameters::remove);
+
+    dataset.setExternalCatalogDatasetOptions(
+        externalCatalogDatasetOptions.setParameters(finalParameters));
+
+    return internalUpdate(dataset);
+  }
+
+  @Override
+  public List<Datasets> list(String projectId) {
+    try {
+      String nextPageToken = null;
+      List<Datasets> datasets = Lists.newArrayList();
+      do {
+        HttpResponse pageResponse =
+            client.datasets().list(projectId).setPageToken(nextPageToken).executeUnparsed();
+        DatasetList result =
+            convertExceptionIfUnsuccessful(pageResponse).parseAs(DatasetList.class);
+        nextPageToken = result.getNextPageToken();
+        if (result.getDatasets() != null) {
+          datasets.addAll(result.getDatasets());
+        }
+
+      } while (nextPageToken != null && !nextPageToken.isEmpty());
+      return datasets;
+    } catch (IOException e) {
+      throw new RuntimeIOException(e);
+    }
+  }
+
+  @Override
+  public Table create(Table table) {
+    // Ensure it is an Iceberg table supported by the BigQuery metastore catalog.
+    validateTable(table);
+    // TODO: Ensure table creation is idempotent when handling retries.
+    Table response = null;
+    try {
+      response =
+          BigQueryRetryHelper.runWithRetries(
+              () -> internalCreate(table),
+              bigqueryOptions.getRetrySettings(),
+              BIGQUERY_EXCEPTION_HANDLER,
+              bigqueryOptions.getClock(),
+              DEFAULT_RETRY_CONFIG);
+    } catch (BigQueryRetryHelper.BigQueryRetryHelperException e) {
+      handleBigQueryRetryException(e);
+    }
+
+    return response;
+  }
+
+  @SuppressWarnings("FormatStringAnnotation")
+  private Table internalCreate(Table table) {
+    try {
+      HttpResponse response =
+          client
+              .tables()
+              .insert(
+                  Preconditions.checkNotNull(table.getTableReference()).getProjectId(),
+                  Preconditions.checkNotNull(table.getTableReference()).getDatasetId(),
+                  table)
+              .executeUnparsed();
+      return convertExceptionIfUnsuccessful(response).parseAs(Table.class);
+    } catch (IOException e) {
+      throw new RuntimeIOException(e);
+    } catch (AlreadyExistsException e) {
+      throw new AlreadyExistsException(String.format("Table %s already exists", table), e);
+    }
+  }
+
+  @Override
+  @SuppressWarnings("FormatStringAnnotation")
+  public Table load(TableReference tableReference) {
+    try {
+      HttpResponse response =
+          client
+              .tables()
+              .get(
+                  tableReference.getProjectId(),
+                  tableReference.getDatasetId(),
+                  tableReference.getTableId())
+              .executeUnparsed();
+      if (response.getStatusCode() == HttpStatusCodes.STATUS_CODE_NOT_FOUND) {
+        throw new NoSuchTableException(response.getStatusMessage());
+      }
+
+      return validateTable(convertExceptionIfUnsuccessful(response).parseAs(Table.class));
+    } catch (IOException e) {
+      throw new RuntimeIOException(e);
+    }
+  }
+
+  @Override
+  @SuppressWarnings("FormatStringAnnotation")
+  public Table update(TableReference tableReference, Table table) {
+    // Ensure it is an Iceberg table supported by the BQ metastore catalog.
+    validateTable(table);
+
+    ExternalCatalogTableOptions newExternalCatalogTableOptions =
+        new ExternalCatalogTableOptions()
+            .setStorageDescriptor(table.getExternalCatalogTableOptions().getStorageDescriptor())
+            .setConnectionId(table.getExternalCatalogTableOptions().getConnectionId())
+            .setParameters(table.getExternalCatalogTableOptions().getParameters());
+    Table updatedTable =
+        new Table()
+            .setExternalCatalogTableOptions(newExternalCatalogTableOptions)
+            // Must set the schema as null for using schema auto-detect.
+            .setSchema(Data.nullOf(TableSchema.class));
+
+    Table response = null;
+    try {
+      response =
+          BigQueryRetryHelper.runWithRetries(
+              () -> internalUpdate(tableReference, updatedTable, table.getEtag()),
+              bigqueryOptions.getRetrySettings(),
+              BIGQUERY_EXCEPTION_HANDLER,
+              bigqueryOptions.getClock(),
+              DEFAULT_RETRY_CONFIG);
+    } catch (BigQueryRetryHelper.BigQueryRetryHelperException e) {
+      handleBigQueryRetryException(e);
+    }
+
+    return response;
+  }
+
+  @SuppressWarnings("FormatStringAnnotation")

Review Comment:
   I removed some of them But I can not remove all because in some function i just re-throw exception with wrapping cause.



-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org

For queries about this service, please contact Infrastructure at:
users@infra.apache.org


---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For additional commands, e-mail: issues-help@iceberg.apache.org


Re: [PR] Catalog: Add BigQuery Metastore Catalog Support [iceberg]

Posted by "nastra (via GitHub)" <gi...@apache.org>.
nastra commented on code in PR #12808:
URL: https://github.com/apache/iceberg/pull/12808#discussion_r2063986640


##########
bigquery/src/main/java/org/apache/iceberg/gcp/bigquery/BigQueryMetastoreCatalog.java:
##########
@@ -0,0 +1,394 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.iceberg.gcp.bigquery;
+
+import com.google.api.services.bigquery.model.Dataset;
+import com.google.api.services.bigquery.model.DatasetList.Datasets;
+import com.google.api.services.bigquery.model.DatasetReference;
+import com.google.api.services.bigquery.model.ExternalCatalogDatasetOptions;
+import com.google.api.services.bigquery.model.TableReference;
+import com.google.cloud.ServiceOptions;
+import com.google.cloud.bigquery.BigQueryOptions;
+import java.io.IOException;
+import java.security.GeneralSecurityException;
+import java.util.List;
+import java.util.Locale;
+import java.util.Map;
+import java.util.Objects;
+import java.util.Set;
+import org.apache.hadoop.conf.Configurable;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.iceberg.BaseMetastoreCatalog;
+import org.apache.iceberg.CatalogProperties;
+import org.apache.iceberg.CatalogUtil;
+import org.apache.iceberg.TableMetadata;
+import org.apache.iceberg.TableOperations;
+import org.apache.iceberg.catalog.Namespace;
+import org.apache.iceberg.catalog.SupportsNamespaces;
+import org.apache.iceberg.catalog.TableIdentifier;
+import org.apache.iceberg.exceptions.AlreadyExistsException;
+import org.apache.iceberg.exceptions.NoSuchNamespaceException;
+import org.apache.iceberg.exceptions.NoSuchTableException;
+import org.apache.iceberg.exceptions.ServiceFailureException;
+import org.apache.iceberg.exceptions.ValidationException;
+import org.apache.iceberg.io.FileIO;
+import org.apache.iceberg.relocated.com.google.common.annotations.VisibleForTesting;
+import org.apache.iceberg.relocated.com.google.common.base.Preconditions;
+import org.apache.iceberg.relocated.com.google.common.base.Strings;
+import org.apache.iceberg.relocated.com.google.common.collect.ImmutableList;
+import org.apache.iceberg.relocated.com.google.common.collect.ImmutableMap;
+import org.apache.iceberg.relocated.com.google.common.collect.Maps;
+import org.apache.iceberg.util.LocationUtil;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/** Iceberg Bigquery Metastore Catalog implementation. */
+public class BigQueryMetastoreCatalog extends BaseMetastoreCatalog
+    implements SupportsNamespaces, Configurable {
+
+  // User provided properties.
+  public static final String PROJECT_ID = "gcp-project";
+  public static final String GCP_LOCATION = "gcp-location";
+  public static final String LIST_ALL_TABLES = "list-all-tables";
+
+  public static final String HIVE_METASTORE_WAREHOUSE_DIR = "hive.metastore.warehouse.dir";
+
+  private static final Logger LOG = LoggerFactory.getLogger(BigQueryMetastoreCatalog.class);
+
+  private static final String DEFAULT_GCP_LOCATION = "us";
+
+  private String catalogName;
+  private Map<String, String> catalogProperties;
+  private FileIO fileIO;
+  private Configuration conf;
+  private String projectId;
+  private String location;
+  private BigQueryMetastoreClient client;
+  private boolean listAllTables;
+
+  // Must have a no-arg constructor to be dynamically loaded
+  // initialize(String name, Map<String, String> properties) will be called to complete
+  // initialization
+  public BigQueryMetastoreCatalog() {}
+
+  @Override
+  public void initialize(String inputName, Map<String, String> properties) {
+    Preconditions.checkArgument(
+        properties.containsKey(PROJECT_ID), "GCP project must be specified");
+
+    projectId = properties.get(PROJECT_ID);
+    location = properties.getOrDefault(GCP_LOCATION, DEFAULT_GCP_LOCATION);
+
+    BigQueryOptions options =
+        BigQueryOptions.newBuilder()
+            .setProjectId(projectId)
+            .setLocation(location)
+            .setRetrySettings(ServiceOptions.getDefaultRetrySettings())
+            .build();
+
+    try {
+      client = new BigQueryMetastoreClientImpl(options);
+    } catch (IOException e) {
+      throw new ServiceFailureException(e, "Creating BigQuery client failed");
+    } catch (GeneralSecurityException e) {
+      throw new ValidationException(e, "Creating BigQuery client failed due to a security issue");
+    }
+
+    initialize(inputName, properties, projectId, location, client);
+  }
+
+  @VisibleForTesting
+  void initialize(
+      String inputName,
+      Map<String, String> properties,
+      String initialProjectId,
+      String initialLocation,
+      BigQueryMetastoreClient bigQueryMetaStoreClient) {
+    Preconditions.checkArgument(bigQueryMetaStoreClient != null, "BigQuery client can not be null");
+    this.catalogName = inputName;
+    this.catalogProperties = ImmutableMap.copyOf(properties);
+    this.projectId = initialProjectId;
+    this.location = initialLocation;
+    this.client = bigQueryMetaStoreClient;
+
+    if (this.conf == null) {
+      LOG.warn("No configuration was set, using the default environment Configuration");
+      this.conf = new Configuration();
+    }
+
+    LOG.info("Using BigQuery Metastore Iceberg Catalog: {}", inputName);
+
+    if (properties.containsKey(CatalogProperties.WAREHOUSE_LOCATION)) {
+      this.conf.set(
+          HIVE_METASTORE_WAREHOUSE_DIR,
+          LocationUtil.stripTrailingSlash(properties.get(CatalogProperties.WAREHOUSE_LOCATION)));
+    }
+
+    String fileIoImpl =
+        properties.getOrDefault(
+            CatalogProperties.FILE_IO_IMPL, "org.apache.iceberg.hadoop.HadoopFileIO");
+    this.fileIO = CatalogUtil.loadFileIO(fileIoImpl, properties, conf);
+
+    this.listAllTables = Boolean.parseBoolean(properties.getOrDefault(LIST_ALL_TABLES, "true"));
+  }
+
+  @Override
+  protected TableOperations newTableOps(TableIdentifier identifier) {
+    return new BigQueryTableOperations(client, fileIO, toTableReference(identifier), conf);
+  }
+
+  @Override
+  protected String defaultWarehouseLocation(TableIdentifier identifier) {
+    String locationUri = null;
+    DatasetReference datasetReference = toDatasetReference(identifier.namespace());
+    Dataset dataset = client.load(datasetReference);
+    if (dataset != null && dataset.getExternalCatalogDatasetOptions() != null) {
+      locationUri = dataset.getExternalCatalogDatasetOptions().getDefaultStorageLocationUri();
+    }
+
+    return String.format(
+        "%s/%s",
+        Strings.isNullOrEmpty(locationUri)
+            ? createDefaultStorageLocationUri(datasetReference.getDatasetId())
+            : LocationUtil.stripTrailingSlash(locationUri),
+        identifier.name());
+  }
+
+  @Override
+  public List<TableIdentifier> listTables(Namespace namespace) {
+    validateNamespace(namespace);
+
+    return client.list(toDatasetReference(namespace), listAllTables).stream()
+        .map(
+            table -> TableIdentifier.of(namespace.level(0), table.getTableReference().getTableId()))
+        .collect(ImmutableList.toImmutableList());
+  }
+
+  @Override
+  public boolean dropTable(TableIdentifier identifier, boolean purge) {
+    try {
+      TableOperations ops = newTableOps(identifier);
+      TableMetadata lastMetadata = ops.current();
+
+      client.delete(toTableReference(identifier));
+
+      if (purge && lastMetadata != null) {
+        CatalogUtil.dropTableData(ops.io(), lastMetadata);
+      }
+    } catch (NoSuchTableException e) {
+      return false;
+    }
+
+    return true;
+  }
+
+  @Override
+  public void renameTable(TableIdentifier from, TableIdentifier to) {
+    validateNamespace(from.namespace());
+    validateNamespace(to.namespace());
+
+    if (!namespaceExists(to.namespace())) {
+      throw new NoSuchNamespaceException("Namespace does not exist: %s", to.namespace());
+    }
+
+    if (!namespaceExists(from.namespace())) {
+      throw new NoSuchNamespaceException("Namespace does not exist: %s", from.namespace());
+    }
+
+    // Check if the source table exists
+    if (!tableExists(from)) {
+      throw new NoSuchTableException("Table does not exist: %s", from.name());
+    }
+
+    if (!from.namespace().equals(to.namespace())) {
+      throw new ValidationException("New table name must be in the same namespace");
+    }
+
+    // Check if the destination table already exists
+    if (tableExists(to)) {
+      throw new AlreadyExistsException("Table already exists: %s", to.name());
+    }
+
+    // TODO: Enable once supported by BigQuery API.
+    throw new UnsupportedOperationException("Table rename operation is unsupported.");
+  }
+
+  @Override
+  public void createNamespace(Namespace namespace, Map<String, String> metadata) {
+    Dataset builder = new Dataset();
+    DatasetReference datasetReference = toDatasetReference(namespace);
+    builder.setLocation(this.location);
+    builder.setDatasetReference(datasetReference);
+    builder.setExternalCatalogDatasetOptions(
+        BigQueryMetastoreUtils.createExternalCatalogDatasetOptions(
+            createDefaultStorageLocationUri(datasetReference.getDatasetId()), metadata));
+
+    client.create(builder);
+  }
+
+  @Override
+  public List<Namespace> listNamespaces() {
+    try {
+      return listNamespaces(Namespace.empty());
+    } catch (NoSuchNamespaceException e) {
+      return ImmutableList.of();
+    }
+  }
+
+  /**
+   * Since this catalog only supports one-level namespaces, it always returns an empty list unless
+   * passed an empty namespace to list all namespaces within the catalog.
+   */
+  @Override
+  public List<Namespace> listNamespaces(Namespace namespace) {
+    List<Datasets> allDatasets = client.list(projectId);
+
+    ImmutableList<Namespace> namespaces =
+        allDatasets.stream()
+            .map(BigQueryMetastoreCatalog::toNamespace)
+            .collect(ImmutableList.toImmutableList());
+    if (namespaces.isEmpty()) {
+      throw new NoSuchNamespaceException("Namespace does not exist: %s", namespace);
+    }
+
+    return namespaces;
+  }
+
+  @Override
+  public boolean dropNamespace(Namespace namespace) {
+    try {
+      client.delete(toDatasetReference(namespace));
+      // We don't delete the data folder for safety, which aligns with Hive Metastore's default
+      // behavior.
+      // We can support database or catalog level config controlling file deletion in the future.
+      return true;
+    } catch (NoSuchNamespaceException e) {
+      return false;
+    }
+  }
+
+  @Override
+  public boolean setProperties(Namespace namespace, Map<String, String> properties) {
+    Dataset dataset = client.load(toDatasetReference(namespace));
+
+    ExternalCatalogDatasetOptions existingOptions = dataset.getExternalCatalogDatasetOptions();
+    Map<String, String> existingParameters =
+        existingOptions != null ? existingOptions.getParameters() : null;
+
+    Map<String, String> newParameters = Maps.newHashMap();
+    if (existingParameters != null) {
+      newParameters.putAll(existingParameters);
+    }
+
+    newParameters.putAll(properties);
+
+    if (Objects.equals(existingParameters, newParameters)) {
+      // No change in parameters detected
+      return false;
+    }
+
+    client.setParameters(toDatasetReference(namespace), properties);
+    return true;
+  }
+
+  @Override
+  public boolean removeProperties(Namespace namespace, Set<String> properties) {
+    client.removeParameters(toDatasetReference(namespace), properties);
+    return true;
+  }
+
+  @SuppressWarnings("FormatStringAnnotation")
+  @Override
+  public Map<String, String> loadNamespaceMetadata(Namespace namespace) {
+    try {
+      return toMetadata(client.load(toDatasetReference(namespace)));
+    } catch (IllegalArgumentException e) {
+      throw new NoSuchNamespaceException(e.getMessage());
+    }
+  }
+
+  @Override
+  public String name() {
+    return catalogName;
+  }
+
+  @Override
+  protected Map<String, String> properties() {
+    return catalogProperties == null ? ImmutableMap.of() : catalogProperties;
+  }
+
+  @Override
+  public void setConf(Configuration conf) {
+    this.conf = new Configuration(conf);
+  }
+
+  @Override
+  public Configuration getConf() {
+    return this.conf;
+  }
+
+  private String createDefaultStorageLocationUri(String dbId) {
+    String warehouseLocation = conf.get(HIVE_METASTORE_WAREHOUSE_DIR);
+    Preconditions.checkNotNull(warehouseLocation, "Data warehouse location is not set");
+    return String.format("%s/%s.db", LocationUtil.stripTrailingSlash(warehouseLocation), dbId);
+  }
+
+  private static Namespace toNamespace(Datasets dataset) {

Review Comment:
   this most likely doesn't need to be static



-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org

For queries about this service, please contact Infrastructure at:
users@infra.apache.org


---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For additional commands, e-mail: issues-help@iceberg.apache.org


Re: [PR] Catalog: Add BigQuery Metastore Catalog Support [iceberg]

Posted by "nastra (via GitHub)" <gi...@apache.org>.
nastra commented on code in PR #12808:
URL: https://github.com/apache/iceberg/pull/12808#discussion_r2063969559


##########
bigquery/src/main/java/org/apache/iceberg/gcp/bigquery/BigQueryMetastoreCatalog.java:
##########
@@ -0,0 +1,394 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.iceberg.gcp.bigquery;
+
+import com.google.api.services.bigquery.model.Dataset;
+import com.google.api.services.bigquery.model.DatasetList.Datasets;
+import com.google.api.services.bigquery.model.DatasetReference;
+import com.google.api.services.bigquery.model.ExternalCatalogDatasetOptions;
+import com.google.api.services.bigquery.model.TableReference;
+import com.google.cloud.ServiceOptions;
+import com.google.cloud.bigquery.BigQueryOptions;
+import java.io.IOException;
+import java.security.GeneralSecurityException;
+import java.util.List;
+import java.util.Locale;
+import java.util.Map;
+import java.util.Objects;
+import java.util.Set;
+import org.apache.hadoop.conf.Configurable;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.iceberg.BaseMetastoreCatalog;
+import org.apache.iceberg.CatalogProperties;
+import org.apache.iceberg.CatalogUtil;
+import org.apache.iceberg.TableMetadata;
+import org.apache.iceberg.TableOperations;
+import org.apache.iceberg.catalog.Namespace;
+import org.apache.iceberg.catalog.SupportsNamespaces;
+import org.apache.iceberg.catalog.TableIdentifier;
+import org.apache.iceberg.exceptions.AlreadyExistsException;
+import org.apache.iceberg.exceptions.NoSuchNamespaceException;
+import org.apache.iceberg.exceptions.NoSuchTableException;
+import org.apache.iceberg.exceptions.ServiceFailureException;
+import org.apache.iceberg.exceptions.ValidationException;
+import org.apache.iceberg.io.FileIO;
+import org.apache.iceberg.relocated.com.google.common.annotations.VisibleForTesting;
+import org.apache.iceberg.relocated.com.google.common.base.Preconditions;
+import org.apache.iceberg.relocated.com.google.common.base.Strings;
+import org.apache.iceberg.relocated.com.google.common.collect.ImmutableList;
+import org.apache.iceberg.relocated.com.google.common.collect.ImmutableMap;
+import org.apache.iceberg.relocated.com.google.common.collect.Maps;
+import org.apache.iceberg.util.LocationUtil;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/** Iceberg Bigquery Metastore Catalog implementation. */
+public class BigQueryMetastoreCatalog extends BaseMetastoreCatalog
+    implements SupportsNamespaces, Configurable {
+
+  // User provided properties.
+  public static final String PROJECT_ID = "gcp-project";
+  public static final String GCP_LOCATION = "gcp-location";
+  public static final String LIST_ALL_TABLES = "list-all-tables";
+
+  public static final String HIVE_METASTORE_WAREHOUSE_DIR = "hive.metastore.warehouse.dir";
+
+  private static final Logger LOG = LoggerFactory.getLogger(BigQueryMetastoreCatalog.class);
+
+  private static final String DEFAULT_GCP_LOCATION = "us";
+
+  private String catalogName;
+  private Map<String, String> catalogProperties;
+  private FileIO fileIO;
+  private Configuration conf;
+  private String projectId;
+  private String location;
+  private BigQueryMetastoreClient client;
+  private boolean listAllTables;
+
+  // Must have a no-arg constructor to be dynamically loaded
+  // initialize(String name, Map<String, String> properties) will be called to complete
+  // initialization
+  public BigQueryMetastoreCatalog() {}
+
+  @Override
+  public void initialize(String inputName, Map<String, String> properties) {
+    Preconditions.checkArgument(
+        properties.containsKey(PROJECT_ID), "GCP project must be specified");
+
+    projectId = properties.get(PROJECT_ID);
+    location = properties.getOrDefault(GCP_LOCATION, DEFAULT_GCP_LOCATION);
+
+    BigQueryOptions options =
+        BigQueryOptions.newBuilder()
+            .setProjectId(projectId)
+            .setLocation(location)
+            .setRetrySettings(ServiceOptions.getDefaultRetrySettings())
+            .build();
+
+    try {
+      client = new BigQueryMetastoreClientImpl(options);
+    } catch (IOException e) {
+      throw new ServiceFailureException(e, "Creating BigQuery client failed");
+    } catch (GeneralSecurityException e) {
+      throw new ValidationException(e, "Creating BigQuery client failed due to a security issue");
+    }
+
+    initialize(inputName, properties, projectId, location, client);
+  }
+
+  @VisibleForTesting
+  void initialize(
+      String inputName,
+      Map<String, String> properties,
+      String initialProjectId,
+      String initialLocation,
+      BigQueryMetastoreClient bigQueryMetaStoreClient) {
+    Preconditions.checkArgument(bigQueryMetaStoreClient != null, "BigQuery client can not be null");
+    this.catalogName = inputName;
+    this.catalogProperties = ImmutableMap.copyOf(properties);
+    this.projectId = initialProjectId;
+    this.location = initialLocation;
+    this.client = bigQueryMetaStoreClient;
+
+    if (this.conf == null) {
+      LOG.warn("No configuration was set, using the default environment Configuration");
+      this.conf = new Configuration();
+    }
+
+    LOG.info("Using BigQuery Metastore Iceberg Catalog: {}", inputName);
+
+    if (properties.containsKey(CatalogProperties.WAREHOUSE_LOCATION)) {
+      this.conf.set(
+          HIVE_METASTORE_WAREHOUSE_DIR,
+          LocationUtil.stripTrailingSlash(properties.get(CatalogProperties.WAREHOUSE_LOCATION)));
+    }
+
+    String fileIoImpl =
+        properties.getOrDefault(
+            CatalogProperties.FILE_IO_IMPL, "org.apache.iceberg.hadoop.HadoopFileIO");
+    this.fileIO = CatalogUtil.loadFileIO(fileIoImpl, properties, conf);
+
+    this.listAllTables = Boolean.parseBoolean(properties.getOrDefault(LIST_ALL_TABLES, "true"));
+  }
+
+  @Override
+  protected TableOperations newTableOps(TableIdentifier identifier) {
+    return new BigQueryTableOperations(client, fileIO, toTableReference(identifier), conf);
+  }
+
+  @Override
+  protected String defaultWarehouseLocation(TableIdentifier identifier) {
+    String locationUri = null;
+    DatasetReference datasetReference = toDatasetReference(identifier.namespace());
+    Dataset dataset = client.load(datasetReference);
+    if (dataset != null && dataset.getExternalCatalogDatasetOptions() != null) {
+      locationUri = dataset.getExternalCatalogDatasetOptions().getDefaultStorageLocationUri();
+    }
+
+    return String.format(
+        "%s/%s",
+        Strings.isNullOrEmpty(locationUri)
+            ? createDefaultStorageLocationUri(datasetReference.getDatasetId())
+            : LocationUtil.stripTrailingSlash(locationUri),
+        identifier.name());
+  }
+
+  @Override
+  public List<TableIdentifier> listTables(Namespace namespace) {
+    validateNamespace(namespace);
+
+    return client.list(toDatasetReference(namespace), listAllTables).stream()
+        .map(
+            table -> TableIdentifier.of(namespace.level(0), table.getTableReference().getTableId()))
+        .collect(ImmutableList.toImmutableList());
+  }
+
+  @Override
+  public boolean dropTable(TableIdentifier identifier, boolean purge) {
+    try {
+      TableOperations ops = newTableOps(identifier);
+      TableMetadata lastMetadata = ops.current();
+
+      client.delete(toTableReference(identifier));
+
+      if (purge && lastMetadata != null) {
+        CatalogUtil.dropTableData(ops.io(), lastMetadata);
+      }
+    } catch (NoSuchTableException e) {
+      return false;
+    }
+
+    return true;
+  }
+
+  @Override
+  public void renameTable(TableIdentifier from, TableIdentifier to) {
+    validateNamespace(from.namespace());
+    validateNamespace(to.namespace());
+
+    if (!namespaceExists(to.namespace())) {
+      throw new NoSuchNamespaceException("Namespace does not exist: %s", to.namespace());
+    }
+
+    if (!namespaceExists(from.namespace())) {
+      throw new NoSuchNamespaceException("Namespace does not exist: %s", from.namespace());
+    }
+
+    // Check if the source table exists
+    if (!tableExists(from)) {
+      throw new NoSuchTableException("Table does not exist: %s", from.name());
+    }
+
+    if (!from.namespace().equals(to.namespace())) {
+      throw new ValidationException("New table name must be in the same namespace");
+    }
+
+    // Check if the destination table already exists

Review Comment:
   same as above



-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org

For queries about this service, please contact Infrastructure at:
users@infra.apache.org


---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For additional commands, e-mail: issues-help@iceberg.apache.org


Re: [PR] Catalog: Add BigQuery Metastore Catalog Support [iceberg]

Posted by "talatuyarer (via GitHub)" <gi...@apache.org>.
talatuyarer commented on code in PR #12808:
URL: https://github.com/apache/iceberg/pull/12808#discussion_r2059696767


##########
bigquery/src/main/java/org/apache/iceberg/gcp/bigquery/BigQueryMetastoreClient.java:
##########
@@ -0,0 +1,121 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.iceberg.gcp.bigquery;
+
+import com.google.api.services.bigquery.model.Dataset;
+import com.google.api.services.bigquery.model.DatasetList.Datasets;
+import com.google.api.services.bigquery.model.DatasetReference;
+import com.google.api.services.bigquery.model.Table;
+import com.google.api.services.bigquery.model.TableList.Tables;
+import com.google.api.services.bigquery.model.TableReference;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+
+/**
+ * A client of Google BigQuery Metastore functions over the BigQuery service. Uses the Google
+ * BigQuery API.
+ */
+public interface BigQueryMetastoreClient {
+
+  /**
+   * Creates and returns a new dataset.
+   *
+   * @param dataset the dataset to create
+   */
+  Dataset createDataset(Dataset dataset);
+
+  /**
+   * Returns a dataset.
+   *
+   * @param datasetReference full dataset reference
+   */
+  Dataset getDataset(DatasetReference datasetReference);

Review Comment:
   Done



##########
bigquery/src/main/java/org/apache/iceberg/gcp/bigquery/BigQueryMetastoreClient.java:
##########
@@ -0,0 +1,121 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.iceberg.gcp.bigquery;
+
+import com.google.api.services.bigquery.model.Dataset;
+import com.google.api.services.bigquery.model.DatasetList.Datasets;
+import com.google.api.services.bigquery.model.DatasetReference;
+import com.google.api.services.bigquery.model.Table;
+import com.google.api.services.bigquery.model.TableList.Tables;
+import com.google.api.services.bigquery.model.TableReference;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+
+/**
+ * A client of Google BigQuery Metastore functions over the BigQuery service. Uses the Google
+ * BigQuery API.
+ */
+public interface BigQueryMetastoreClient {
+
+  /**
+   * Creates and returns a new dataset.
+   *
+   * @param dataset the dataset to create
+   */
+  Dataset createDataset(Dataset dataset);
+
+  /**
+   * Returns a dataset.
+   *
+   * @param datasetReference full dataset reference
+   */
+  Dataset getDataset(DatasetReference datasetReference);
+
+  /**
+   * Deletes a dataset.
+   *
+   * @param datasetReference full dataset reference
+   */
+  void deleteDataset(DatasetReference datasetReference);
+
+  /**
+   * Updates parameters of a dataset or adds them if did not exist, leaving already-existing ones
+   * intact.
+   *
+   * @param datasetReference full dataset reference
+   * @param parameters metadata parameters to add
+   * @return dataset after patch
+   */
+  Dataset setDatasetParameters(DatasetReference datasetReference, Map<String, String> parameters);
+
+  /**
+   * Removes given set of keys of parameters of a dataset. Ignores keys that do not exist already.
+   *
+   * @param datasetReference full dataset reference
+   * @param parameters metadata parameter keys to remove
+   * @return dataset after patch
+   */
+  Dataset removeDatasetParameters(DatasetReference datasetReference, Set<String> parameters);
+
+  /**
+   * Lists datasets under a given project
+   *
+   * @param projectId the identifier of the project to list datasets under
+   */
+  List<Datasets> listDatasets(String projectId);
+
+  /**
+   * Creates and returns a new table.
+   *
+   * @param table body of the table to create
+   */
+  Table createTable(Table table);
+
+  /**
+   * Returns a table.
+   *
+   * @param tableReference full table reference
+   */
+  Table getTable(TableReference tableReference);

Review Comment:
   Done



-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org

For queries about this service, please contact Infrastructure at:
users@infra.apache.org


---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For additional commands, e-mail: issues-help@iceberg.apache.org


Re: [PR] Catalog: Add BigQuery Metastore Catalog Support [iceberg]

Posted by "talatuyarer (via GitHub)" <gi...@apache.org>.
talatuyarer commented on code in PR #12808:
URL: https://github.com/apache/iceberg/pull/12808#discussion_r2059769704


##########
bigquery/src/test/java/org/apache/iceberg/gcp/bigquery/TestBigQueryCatalog.java:
##########
@@ -0,0 +1,673 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.iceberg.gcp.bigquery;
+
+import static org.apache.iceberg.CatalogUtil.ICEBERG_CATALOG_TYPE;
+import static org.apache.iceberg.CatalogUtil.ICEBERG_CATALOG_TYPE_BIGQUERY;
+import static org.apache.iceberg.gcp.bigquery.BigQueryMetastoreCatalog.PROJECT_ID;
+import static org.apache.iceberg.gcp.bigquery.BigQueryMetastoreCatalog.TESTING_ENABLED;
+import static org.apache.iceberg.types.Types.NestedField.optional;
+import static org.assertj.core.api.Assertions.assertThat;
+import static org.assertj.core.api.Assertions.assertThatThrownBy;
+
+import java.io.File;
+import java.util.List;
+import java.util.Map;
+import org.apache.iceberg.CatalogProperties;
+import org.apache.iceberg.CatalogUtil;
+import org.apache.iceberg.PartitionSpec;
+import org.apache.iceberg.Schema;
+import org.apache.iceberg.SortOrder;
+import org.apache.iceberg.Table;
+import org.apache.iceberg.TableUtil;
+import org.apache.iceberg.Transaction;
+import org.apache.iceberg.catalog.CatalogTests;
+import org.apache.iceberg.catalog.Namespace;
+import org.apache.iceberg.catalog.TableIdentifier;
+import org.apache.iceberg.exceptions.NoSuchNamespaceException;
+import org.apache.iceberg.exceptions.NoSuchTableException;
+import org.apache.iceberg.expressions.Expressions;
+import org.apache.iceberg.relocated.com.google.common.collect.ImmutableMap;
+import org.apache.iceberg.types.Types;
+import org.junit.jupiter.api.AfterEach;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Disabled;
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.io.TempDir;
+import org.junit.jupiter.params.ParameterizedTest;
+import org.junit.jupiter.params.provider.ValueSource;
+
+public class TestBigQueryCatalog extends CatalogTests<BigQueryMetastoreCatalog> {
+
+  private static final String GCP_PROJECT = "project-id";

Review Comment:
   Done



-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org

For queries about this service, please contact Infrastructure at:
users@infra.apache.org


---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For additional commands, e-mail: issues-help@iceberg.apache.org


Re: [PR] Catalog: Add BigQuery Metastore Catalog Support [iceberg]

Posted by "talatuyarer (via GitHub)" <gi...@apache.org>.
talatuyarer commented on code in PR #12808:
URL: https://github.com/apache/iceberg/pull/12808#discussion_r2059824163


##########
bigquery/src/test/java/org/apache/iceberg/gcp/bigquery/TestBigQueryCatalog.java:
##########
@@ -0,0 +1,673 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.iceberg.gcp.bigquery;
+
+import static org.apache.iceberg.CatalogUtil.ICEBERG_CATALOG_TYPE;
+import static org.apache.iceberg.CatalogUtil.ICEBERG_CATALOG_TYPE_BIGQUERY;
+import static org.apache.iceberg.gcp.bigquery.BigQueryMetastoreCatalog.PROJECT_ID;
+import static org.apache.iceberg.gcp.bigquery.BigQueryMetastoreCatalog.TESTING_ENABLED;
+import static org.apache.iceberg.types.Types.NestedField.optional;
+import static org.assertj.core.api.Assertions.assertThat;
+import static org.assertj.core.api.Assertions.assertThatThrownBy;
+
+import java.io.File;
+import java.util.List;
+import java.util.Map;
+import org.apache.iceberg.CatalogProperties;
+import org.apache.iceberg.CatalogUtil;
+import org.apache.iceberg.PartitionSpec;
+import org.apache.iceberg.Schema;
+import org.apache.iceberg.SortOrder;
+import org.apache.iceberg.Table;
+import org.apache.iceberg.TableUtil;
+import org.apache.iceberg.Transaction;
+import org.apache.iceberg.catalog.CatalogTests;
+import org.apache.iceberg.catalog.Namespace;
+import org.apache.iceberg.catalog.TableIdentifier;
+import org.apache.iceberg.exceptions.NoSuchNamespaceException;
+import org.apache.iceberg.exceptions.NoSuchTableException;
+import org.apache.iceberg.expressions.Expressions;
+import org.apache.iceberg.relocated.com.google.common.collect.ImmutableMap;
+import org.apache.iceberg.types.Types;
+import org.junit.jupiter.api.AfterEach;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Disabled;
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.io.TempDir;
+import org.junit.jupiter.params.ParameterizedTest;
+import org.junit.jupiter.params.provider.ValueSource;
+
+public class TestBigQueryCatalog extends CatalogTests<BigQueryMetastoreCatalog> {
+
+  private static final String GCP_PROJECT = "project-id";
+  private static final String CATALOG_ID = "catalog-name";
+
+  // another schema that is not the same
+  protected static final Schema OTHER_SCHEMA_OPTIONAL =
+      new Schema(optional(1, "some_id", Types.IntegerType.get()));
+
+  // Schema passed to create tables
+  protected static final Schema SCHEMA_OPTIONAL =
+      new Schema(
+          optional(3, "id", Types.IntegerType.get(), "unique ID 🤪"),
+          optional(4, "data", Types.StringType.get()));
+
+  // This is the actual schema for the table, with column IDs reassigned
+  protected static final Schema REPLACE_SCHEMA_OPTIONAL =
+      new Schema(
+          optional(2, "id", Types.IntegerType.get(), "unique ID 🤪"),
+          optional(3, "data", Types.StringType.get()));
+
+  protected static final PartitionSpec REPLACE_SPEC_OPTIONAL =
+      PartitionSpec.builderFor(REPLACE_SCHEMA_OPTIONAL).bucket("id", 16).withSpecId(1).build();
+
+  protected static final SortOrder REPLACE_WRITE_ORDER_OPTIONAL =
+      SortOrder.builderFor(REPLACE_SCHEMA_OPTIONAL)
+          .asc(Expressions.bucket("id", 16))
+          .asc("id")
+          .build();
+
+  @TempDir private File tempFolder;
+  private BigQueryMetastoreCatalog catalog;
+  private BigQueryMetastoreClient fakeBigQueryMetastoreClient;
+  private String warehouseLocation;
+
+  @BeforeEach
+  public void before() throws Exception {
+    catalog = initCatalog(CATALOG_ID, ImmutableMap.of());
+  }
+
+  @AfterEach
+  public void after() throws Exception {
+    // Drop all tables in all datasets
+    List<Namespace> namespaces = catalog().listNamespaces(Namespace.empty());
+    for (Namespace namespace : namespaces) {
+      List<TableIdentifier> tables = catalog().listTables(namespace);
+      for (TableIdentifier table : tables) {
+        try {
+          catalog().dropTable(table, true); // drop with purge
+        } catch (NoSuchTableException e) {
+          // Table might be dropped by a previous iteration, ignore
+        }
+      }
+
+      // Drop all datasets (except any initial ones)
+      try {
+        catalog().dropNamespace(namespace);
+      } catch (NoSuchNamespaceException e) {
+        // Namespace might be dropped by a previous iteration, ignore
+      }
+    }
+  }
+
+  @Override
+  protected boolean requiresNamespaceCreate() {
+    return true;
+  }
+
+  @Override
+  protected boolean supportsNamesWithSlashes() {
+    return false;
+  }
+
+  @Override
+  protected boolean supportsNamesWithDot() {
+    return false;
+  }
+
+  @Override
+  public BigQueryMetastoreCatalog catalog() {
+    return catalog;
+  }
+
+  @Override
+  protected BigQueryMetastoreCatalog initCatalog(
+      String catalogName, Map<String, String> additionalProperties) {
+
+    warehouseLocation = tempFolder.toPath().resolve("hive-warehouse").toString();
+
+    Map<String, String> properties =
+        Map.of(
+            ICEBERG_CATALOG_TYPE,
+            ICEBERG_CATALOG_TYPE_BIGQUERY,
+            PROJECT_ID,
+            GCP_PROJECT,
+            CatalogProperties.WAREHOUSE_LOCATION,
+            warehouseLocation,
+            TESTING_ENABLED,
+            "true",
+            CatalogProperties.TABLE_DEFAULT_PREFIX + "default-key1",
+            "catalog-default-key1",
+            CatalogProperties.TABLE_DEFAULT_PREFIX + "default-key2",
+            "catalog-default-key2",
+            CatalogProperties.TABLE_DEFAULT_PREFIX + "override-key3",
+            "catalog-default-key3",
+            CatalogProperties.TABLE_OVERRIDE_PREFIX + "override-key3",
+            "catalog-override-key3",
+            CatalogProperties.TABLE_OVERRIDE_PREFIX + "override-key4",
+            "catalog-override-key4");
+
+    BigQueryMetastoreCatalog tmpCatalog =
+        (BigQueryMetastoreCatalog)
+            CatalogUtil.buildIcebergCatalog(
+                catalogName,
+                ImmutableMap.<String, String>builder()
+                    .putAll(properties)
+                    .putAll(additionalProperties)
+                    .build(),
+                null);
+
+    return tmpCatalog;
+  }
+
+  @Test
+  public void testRenameTable() {
+    if (requiresNamespaceCreate()) {
+      catalog.createNamespace(NS);
+    }
+
+    assertThat(catalog.tableExists(TABLE))
+        .as("Source table should not exist before create")
+        .isFalse();
+
+    catalog.buildTable(TABLE, SCHEMA).create();
+    assertThat(catalog.tableExists(TABLE)).as("Table should exist after create").isTrue();
+
+    assertThat(catalog.tableExists(RENAMED_TABLE))
+        .as("Destination table should not exist before rename")
+        .isFalse();
+
+    assertThatThrownBy(() -> catalog.renameTable(TABLE, RENAMED_TABLE))
+        .isInstanceOf(UnsupportedOperationException.class)
+        .hasMessageContaining("Table rename operation is unsupported");
+
+    catalog.dropTable(TABLE);
+  }
+
+  @Disabled(
+      "BigQuery does not allow adding a REQUIRED column to an existing table schema. "
+          + "When this operation is attempted, an exception gets thrown.")
+  @Test
+  public void testOverrideTablePropertiesReplaceTransaction() {}
+
+  @Test
+  public void testConcurrentReplaceTransactionSchema() {
+
+    if (requiresNamespaceCreate()) {
+      catalog.createNamespace(NS);
+    }
+
+    Transaction transaction = catalog.buildTable(TABLE, OTHER_SCHEMA_OPTIONAL).createTransaction();
+    transaction.newFastAppend().appendFile(FILE_A).commit();
+    transaction.commitTransaction();
+
+    Table original = catalog.loadTable(TABLE);
+    assertFiles(original, FILE_A);
+
+    Transaction secondReplace =
+        catalog.buildTable(TABLE, OTHER_SCHEMA_OPTIONAL).replaceTransaction();
+    secondReplace.newFastAppend().appendFile(FILE_C).commit();
+
+    Transaction firstReplace = catalog.buildTable(TABLE, SCHEMA_OPTIONAL).replaceTransaction();
+    firstReplace.newFastAppend().appendFile(FILE_B).commit();
+    firstReplace.commitTransaction();
+
+    Table afterFirstReplace = catalog.loadTable(TABLE);
+    assertThat(afterFirstReplace.schema().asStruct())
+        .as("Table schema should match the new schema")
+        .isEqualTo(REPLACE_SCHEMA_OPTIONAL.asStruct());
+    assertUUIDsMatch(original, afterFirstReplace);
+    assertFiles(afterFirstReplace, FILE_B);
+
+    secondReplace.commitTransaction();
+
+    Table afterSecondReplace = catalog.loadTable(TABLE);
+    assertThat(afterSecondReplace.schema().asStruct())
+        .as("Table schema should match the original schema")
+        .isEqualTo(original.schema().asStruct());
+    assertUUIDsMatch(original, afterSecondReplace);
+    assertFiles(afterSecondReplace, FILE_C);
+  }
+
+  @Test
+  public void testCompleteCreateTransaction() {
+    if (requiresNamespaceCreate()) {
+      catalog.createNamespace(NS);
+    }
+
+    String tableLocation = warehouseLocation + "/tmp/ns/table";
+
+    Map<String, String> properties =
+        ImmutableMap.of("user", "someone", "created-at", "2022-02-25T00:38:19");
+    Transaction create =
+        catalog
+            .buildTable(TABLE, SCHEMA)
+            .withLocation(tableLocation)
+            .withPartitionSpec(SPEC)
+            .withSortOrder(WRITE_ORDER)
+            .withProperties(properties)
+            .createTransaction();
+
+    assertThat(catalog.tableExists(TABLE))
+        .as("Table should not exist after createTransaction")
+        .isFalse();
+
+    create.newFastAppend().appendFile(FILE_A).commit();
+
+    assertThat(catalog.tableExists(TABLE))
+        .as("Table should not exist after append commit")
+        .isFalse();
+
+    create.commitTransaction();
+
+    assertThat(catalog.tableExists(TABLE)).as("Table should exist after append commit").isTrue();
+
+    Table table = catalog.loadTable(TABLE);
+    assertThat(table.schema().asStruct())
+        .as("Table schema should match the new schema")
+        .isEqualTo(TABLE_SCHEMA.asStruct());
+    assertThat(table.spec().fields())
+        .as("Table should have create partition spec")
+        .isEqualTo(TABLE_SPEC.fields());
+    assertThat(table.sortOrder())
+        .as("Table should have create sort order")
+        .isEqualTo(TABLE_WRITE_ORDER);
+    assertThat(table.properties().entrySet())
+        .as("Table properties should be a superset of the requested properties")
+        .containsAll(properties.entrySet());
+    if (!overridesRequestedLocation()) {
+      assertThat(table.location())
+          .as("Table location should match requested")
+          .isEqualTo(tableLocation);
+    }
+    assertFiles(table, FILE_A);
+    assertFilesPartitionSpec(table);
+    assertPreviousMetadataFileCount(table, 0);
+  }
+
+  @Disabled(
+      "BigQuery does not allow adding a REQUIRED column to an existing table schema. "
+          + "When this operation is attempted, an exception gets thrown.")
+  @Test
+  public void testCreateOrReplaceReplaceTransactionReplace() {}
+
+  // TODO: Remove this test when BigQuery supports required field change
+  @Test
+  public void testCreateOrReplaceReplaceTransactionReplaceWithOptionalFields() {
+
+    if (requiresNamespaceCreate()) {
+      catalog.createNamespace(NS);
+    }
+
+    Table original = catalog.buildTable(TABLE, OTHER_SCHEMA_OPTIONAL).create();
+
+    assertThat(catalog.tableExists(TABLE))
+        .as("Table should exist before replaceTransaction")
+        .isTrue();
+
+    Transaction createOrReplace =
+        catalog.buildTable(TABLE, SCHEMA_OPTIONAL).createOrReplaceTransaction();
+
+    assertThat(catalog.tableExists(TABLE))
+        .as("Table should still exist after replaceTransaction")
+        .isTrue();
+
+    createOrReplace.newFastAppend().appendFile(FILE_A).commit();
+
+    // validate table has not changed
+    Table table = catalog.loadTable(TABLE);
+
+    assertThat(table.schema().asStruct())
+        .as("Table schema should match concurrent create")
+        .isEqualTo(OTHER_SCHEMA_OPTIONAL.asStruct());
+
+    assertUUIDsMatch(original, table);
+    assertNoFiles(table);
+
+    createOrReplace.commitTransaction();
+
+    // validate the table after replace
+    assertThat(catalog.tableExists(TABLE)).as("Table should exist after append commit").isTrue();
+    table.refresh(); // refresh should work with UUID validation
+
+    Table loaded = catalog.loadTable(TABLE);
+
+    assertThat(loaded.schema().asStruct())
+        .as("Table schema should match the new schema")
+        .isEqualTo(REPLACE_SCHEMA_OPTIONAL.asStruct());
+    assertUUIDsMatch(original, loaded);
+    assertFiles(loaded, FILE_A);
+    assertPreviousMetadataFileCount(loaded, 1);
+  }
+
+  @Disabled(
+      "BigQuery does not allow adding a REQUIRED column to an existing table schema. "
+          + "When this operation is attempted, an exception gets thrown.")
+  @Test
+  public void testConcurrentReplaceTransactionSchema2() {}
+
+  // TODO: Remove this test when BigQuery supports required field change
+  @Test
+  public void testConcurrentReplaceTransactionSchema2WithOptionalFields() {

Review Comment:
   Removed all optionalField tests



-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org

For queries about this service, please contact Infrastructure at:
users@infra.apache.org


---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For additional commands, e-mail: issues-help@iceberg.apache.org


Re: [PR] Catalog: Add BigQuery Metastore Catalog Support [iceberg]

Posted by "talatuyarer (via GitHub)" <gi...@apache.org>.
talatuyarer commented on code in PR #12808:
URL: https://github.com/apache/iceberg/pull/12808#discussion_r2059834363


##########
bigquery/src/test/java/org/apache/iceberg/gcp/bigquery/TestBigQueryCatalog.java:
##########
@@ -0,0 +1,673 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.iceberg.gcp.bigquery;
+
+import static org.apache.iceberg.CatalogUtil.ICEBERG_CATALOG_TYPE;
+import static org.apache.iceberg.CatalogUtil.ICEBERG_CATALOG_TYPE_BIGQUERY;
+import static org.apache.iceberg.gcp.bigquery.BigQueryMetastoreCatalog.PROJECT_ID;
+import static org.apache.iceberg.gcp.bigquery.BigQueryMetastoreCatalog.TESTING_ENABLED;
+import static org.apache.iceberg.types.Types.NestedField.optional;
+import static org.assertj.core.api.Assertions.assertThat;
+import static org.assertj.core.api.Assertions.assertThatThrownBy;
+
+import java.io.File;
+import java.util.List;
+import java.util.Map;
+import org.apache.iceberg.CatalogProperties;
+import org.apache.iceberg.CatalogUtil;
+import org.apache.iceberg.PartitionSpec;
+import org.apache.iceberg.Schema;
+import org.apache.iceberg.SortOrder;
+import org.apache.iceberg.Table;
+import org.apache.iceberg.TableUtil;
+import org.apache.iceberg.Transaction;
+import org.apache.iceberg.catalog.CatalogTests;
+import org.apache.iceberg.catalog.Namespace;
+import org.apache.iceberg.catalog.TableIdentifier;
+import org.apache.iceberg.exceptions.NoSuchNamespaceException;
+import org.apache.iceberg.exceptions.NoSuchTableException;
+import org.apache.iceberg.expressions.Expressions;
+import org.apache.iceberg.relocated.com.google.common.collect.ImmutableMap;
+import org.apache.iceberg.types.Types;
+import org.junit.jupiter.api.AfterEach;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Disabled;
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.io.TempDir;
+import org.junit.jupiter.params.ParameterizedTest;
+import org.junit.jupiter.params.provider.ValueSource;
+
+public class TestBigQueryCatalog extends CatalogTests<BigQueryMetastoreCatalog> {
+
+  private static final String GCP_PROJECT = "project-id";
+  private static final String CATALOG_ID = "catalog-name";
+
+  // another schema that is not the same
+  protected static final Schema OTHER_SCHEMA_OPTIONAL =
+      new Schema(optional(1, "some_id", Types.IntegerType.get()));
+
+  // Schema passed to create tables
+  protected static final Schema SCHEMA_OPTIONAL =
+      new Schema(
+          optional(3, "id", Types.IntegerType.get(), "unique ID 🤪"),
+          optional(4, "data", Types.StringType.get()));
+
+  // This is the actual schema for the table, with column IDs reassigned
+  protected static final Schema REPLACE_SCHEMA_OPTIONAL =
+      new Schema(
+          optional(2, "id", Types.IntegerType.get(), "unique ID 🤪"),
+          optional(3, "data", Types.StringType.get()));
+
+  protected static final PartitionSpec REPLACE_SPEC_OPTIONAL =
+      PartitionSpec.builderFor(REPLACE_SCHEMA_OPTIONAL).bucket("id", 16).withSpecId(1).build();
+
+  protected static final SortOrder REPLACE_WRITE_ORDER_OPTIONAL =
+      SortOrder.builderFor(REPLACE_SCHEMA_OPTIONAL)
+          .asc(Expressions.bucket("id", 16))
+          .asc("id")
+          .build();
+
+  @TempDir private File tempFolder;
+  private BigQueryMetastoreCatalog catalog;
+  private String warehouseLocation;
+
+  @BeforeEach
+  public void before() throws Exception {
+    catalog = initCatalog(CATALOG_ID, ImmutableMap.of());
+  }
+
+  @AfterEach
+  public void after() throws Exception {
+    // Drop all tables in all datasets
+    List<Namespace> namespaces = catalog().listNamespaces(Namespace.empty());
+    for (Namespace namespace : namespaces) {
+      List<TableIdentifier> tables = catalog().listTables(namespace);
+      for (TableIdentifier table : tables) {
+        try {
+          catalog().dropTable(table, true); // drop with purge

Review Comment:
   Done.



-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org

For queries about this service, please contact Infrastructure at:
users@infra.apache.org


---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For additional commands, e-mail: issues-help@iceberg.apache.org


Re: [PR] Catalog: Add BigQuery Metastore Catalog Support [iceberg]

Posted by "gkalra18 (via GitHub)" <gi...@apache.org>.
gkalra18 commented on code in PR #12808:
URL: https://github.com/apache/iceberg/pull/12808#discussion_r2047671107


##########
bigquery/src/main/java/org/apache/iceberg/gcp/bigquery/BigQueryMetaStoreClientImpl.java:
##########
@@ -0,0 +1,620 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.iceberg.gcp.bigquery;
+
+import com.google.api.client.googleapis.javanet.GoogleNetHttpTransport;
+import com.google.api.client.http.HttpHeaders;
+import com.google.api.client.http.HttpResponse;
+import com.google.api.client.http.HttpResponseException;
+import com.google.api.client.http.HttpStatusCodes;
+import com.google.api.client.json.gson.GsonFactory;
+import com.google.api.client.util.Data;
+import com.google.api.services.bigquery.Bigquery;
+import com.google.api.services.bigquery.BigqueryScopes;
+import com.google.api.services.bigquery.model.Dataset;
+import com.google.api.services.bigquery.model.DatasetList;
+import com.google.api.services.bigquery.model.DatasetList.Datasets;
+import com.google.api.services.bigquery.model.DatasetReference;
+import com.google.api.services.bigquery.model.ExternalCatalogDatasetOptions;
+import com.google.api.services.bigquery.model.ExternalCatalogTableOptions;
+import com.google.api.services.bigquery.model.Table;
+import com.google.api.services.bigquery.model.TableList;
+import com.google.api.services.bigquery.model.TableList.Tables;
+import com.google.api.services.bigquery.model.TableReference;
+import com.google.api.services.bigquery.model.TableSchema;
+import com.google.auth.http.HttpCredentialsAdapter;
+import com.google.auth.oauth2.GoogleCredentials;
+import com.google.cloud.BaseServiceException;
+import com.google.cloud.ExceptionHandler;
+import com.google.cloud.bigquery.BigQueryErrorMessages;
+import com.google.cloud.bigquery.BigQueryOptions;
+import com.google.cloud.bigquery.BigQueryRetryConfig;
+import com.google.cloud.bigquery.BigQueryRetryHelper;
+import java.io.IOException;
+import java.nio.charset.StandardCharsets;
+import java.security.GeneralSecurityException;
+import java.util.List;
+import java.util.Locale;
+import java.util.Map;
+import java.util.Set;
+import java.util.concurrent.Callable;
+import java.util.stream.Collectors;
+import java.util.stream.Stream;
+import org.apache.iceberg.BaseMetastoreTableOperations;
+import org.apache.iceberg.exceptions.AlreadyExistsException;
+import org.apache.iceberg.exceptions.BadRequestException;
+import org.apache.iceberg.exceptions.ForbiddenException;
+import org.apache.iceberg.exceptions.NoSuchIcebergTableException;
+import org.apache.iceberg.exceptions.NoSuchNamespaceException;
+import org.apache.iceberg.exceptions.NoSuchTableException;
+import org.apache.iceberg.exceptions.NotAuthorizedException;
+import org.apache.iceberg.exceptions.RuntimeIOException;
+import org.apache.iceberg.exceptions.ServiceFailureException;
+import org.apache.iceberg.exceptions.ServiceUnavailableException;
+import org.apache.iceberg.exceptions.ValidationException;
+import org.apache.iceberg.relocated.com.google.common.base.Preconditions;
+import org.apache.iceberg.relocated.com.google.common.collect.Lists;
+import org.apache.iceberg.relocated.com.google.common.collect.Maps;
+
+/** A client of Google Bigquery Metastore functions over the BigQuery service. */
+public final class BigQueryMetaStoreClientImpl implements BigQueryMetaStoreClient {
+
+  public static final ExceptionHandler.Interceptor EXCEPTION_HANDLER_INTERCEPTOR =
+      new ExceptionHandler.Interceptor() {
+
+        private static final long serialVersionUID = -8429573486870467828L;
+
+        @Override
+        public RetryResult afterEval(Exception exception, RetryResult retryResult) {
+          return ExceptionHandler.Interceptor.RetryResult.CONTINUE_EVALUATION;
+        }
+
+        @Override
+        public RetryResult beforeEval(Exception exception) {
+          if (exception instanceof BaseServiceException) {
+            boolean retriable = ((BaseServiceException) exception).isRetryable();
+            return retriable
+                ? ExceptionHandler.Interceptor.RetryResult.RETRY
+                : ExceptionHandler.Interceptor.RetryResult.CONTINUE_EVALUATION;
+          }
+          return ExceptionHandler.Interceptor.RetryResult.CONTINUE_EVALUATION;
+        }
+      };
+
+  private final Bigquery client;
+  private final BigQueryOptions bigqueryOptions;
+
+  // Retry config with error messages and regex for rate limit exceeded errors.
+  private static final BigQueryRetryConfig DEFAULT_RETRY_CONFIG =
+      BigQueryRetryConfig.newBuilder()
+          .retryOnMessage(BigQueryErrorMessages.RATE_LIMIT_EXCEEDED_MSG)
+          .retryOnMessage(BigQueryErrorMessages.JOB_RATE_LIMIT_EXCEEDED_MSG)
+          .retryOnRegEx(BigQueryErrorMessages.RetryRegExPatterns.RATE_LIMIT_EXCEEDED_REGEX)
+          .build();
+
+  public static final ExceptionHandler BIGQUERY_EXCEPTION_HANDLER =
+      ExceptionHandler.newBuilder()
+          .abortOn(RuntimeException.class)
+          .retryOn(java.net.ConnectException.class) // retry on Connection Exception
+          .retryOn(java.net.UnknownHostException.class) // retry on UnknownHostException
+          .addInterceptors(EXCEPTION_HANDLER_INTERCEPTOR)
+          .build();
+
+  /** Constructs a client of the Google BigQuery service. */
+  public BigQueryMetaStoreClientImpl(BigQueryOptions options)
+      throws IOException, GeneralSecurityException {
+    // Initialize client that will be used to send requests. This client only needs to be created
+    // once, and can be reused for multiple requests
+    HttpCredentialsAdapter httpCredentialsAdapter =
+        new HttpCredentialsAdapter(
+            GoogleCredentials.getApplicationDefault().createScoped(BigqueryScopes.all()));
+    this.client =
+        new Bigquery.Builder(
+                GoogleNetHttpTransport.newTrustedTransport(),
+                GsonFactory.getDefaultInstance(),
+                httpRequest -> {
+                  httpCredentialsAdapter.initialize(httpRequest);
+                  // Instead of throwing exceptions, analyze the HttpResponse object and inspect its
+                  // status code. This will allow BigQuery API errors to be converted into Iceberg
+                  // exceptions.
+                  httpRequest.setThrowExceptionOnExecuteError(false);
+                })
+            .setApplicationName("BigQuery Iceberg Catalog Plugin")
+            .build();
+    this.bigqueryOptions = options;
+  }
+
+  @Override
+  public Dataset createDataset(Dataset dataset) {
+    // TODO (b/399885863): Ensure Dataset creation is idempotent when handling retries.
+    Dataset response = null;
+    try {
+      response =
+          BigQueryRetryHelper.runWithRetries(
+              new Callable<Dataset>() {
+                @Override
+                public Dataset call() {
+                  return create(dataset);
+                }
+              },
+              bigqueryOptions.getRetrySettings(),
+              BIGQUERY_EXCEPTION_HANDLER,
+              bigqueryOptions.getClock(),
+              DEFAULT_RETRY_CONFIG);
+    } catch (BigQueryRetryHelper.BigQueryRetryHelperException e) {
+      handleBigQueryRetryException(e);
+    }
+    return response;
+  }
+
+  @SuppressWarnings("FormatStringAnnotation")
+  private Dataset create(Dataset dataset) {
+    try {
+      HttpResponse response =
+          client
+              .datasets()
+              .insert(dataset.getDatasetReference().getProjectId(), dataset)
+              .executeUnparsed();
+      return convertExceptionIfUnsuccessful(response).parseAs(Dataset.class);
+    } catch (IOException e) {
+      throw new RuntimeIOException(e);
+    } catch (AlreadyExistsException e) {
+      throw new AlreadyExistsException("Namespace already exists: " + dataset.getId());
+    }
+  }
+
+  @Override
+  @SuppressWarnings("FormatStringAnnotation")
+  public Dataset getDataset(DatasetReference datasetReference) {
+    try {
+      HttpResponse response =
+          client
+              .datasets()
+              .get(datasetReference.getProjectId(), datasetReference.getDatasetId())
+              .executeUnparsed();
+      if (response.getStatusCode() == HttpStatusCodes.STATUS_CODE_NOT_FOUND) {
+        throw new NoSuchNamespaceException(
+            "Namespace does not exist: " + datasetReference.getDatasetId());
+      }
+      return convertExceptionIfUnsuccessful(response).parseAs(Dataset.class);
+    } catch (IOException e) {
+      throw new RuntimeIOException(e);
+    }
+  }
+
+  @Override
+  @SuppressWarnings("FormatStringAnnotation")
+  public void deleteDataset(DatasetReference datasetReference) {
+    try {
+      HttpResponse response =
+          client
+              .datasets()
+              .delete(datasetReference.getProjectId(), datasetReference.getDatasetId())
+              .executeUnparsed();
+      if (response.getStatusCode() == HttpStatusCodes.STATUS_CODE_NOT_FOUND) {
+        throw new NoSuchNamespaceException(
+            "Namespace does not exist: " + datasetReference.getDatasetId());
+      }
+      convertExceptionIfUnsuccessful(response);
+    } catch (IOException e) {
+      throw new RuntimeIOException(e);
+    }
+  }
+
+  @Override
+  public Dataset setDatasetParameters(
+      DatasetReference datasetReference, Map<String, String> parameters) {
+    Dataset dataset = getDataset(datasetReference);
+    ExternalCatalogDatasetOptions externalCatalogDatasetOptions =
+        dataset.getExternalCatalogDatasetOptions() == null
+            ? new ExternalCatalogDatasetOptions()
+            : dataset.getExternalCatalogDatasetOptions();
+    Map<String, String> finalParameters =
+        externalCatalogDatasetOptions.getParameters() == null
+            ? Maps.newHashMap()
+            : externalCatalogDatasetOptions.getParameters();
+    finalParameters.putAll(parameters);
+
+    dataset.setExternalCatalogDatasetOptions(
+        externalCatalogDatasetOptions.setParameters(finalParameters));
+
+    return updateDataset(dataset);
+  }
+
+  @Override
+  public Dataset removeDatasetParameters(
+      DatasetReference datasetReference, Set<String> parameters) {
+    Dataset dataset = getDataset(datasetReference);
+    ExternalCatalogDatasetOptions externalCatalogDatasetOptions =
+        dataset.getExternalCatalogDatasetOptions() == null
+            ? new ExternalCatalogDatasetOptions()
+            : dataset.getExternalCatalogDatasetOptions();
+    Map<String, String> finalParameters =
+        externalCatalogDatasetOptions.getParameters() == null
+            ? Maps.newHashMap()
+            : externalCatalogDatasetOptions.getParameters();
+    parameters.forEach(finalParameters::remove);
+
+    dataset.setExternalCatalogDatasetOptions(
+        externalCatalogDatasetOptions.setParameters(finalParameters));
+
+    return updateDataset(dataset);
+  }
+
+  @Override
+  public List<Datasets> listDatasets(String projectId) {
+    try {
+      String nextPageToken = null;
+      List<Datasets> datasets = Lists.newArrayList();
+      do {
+        HttpResponse pageResponse =
+            client.datasets().list(projectId).setPageToken(nextPageToken).executeUnparsed();
+        DatasetList result =
+            convertExceptionIfUnsuccessful(pageResponse).parseAs(DatasetList.class);
+        nextPageToken = result.getNextPageToken();
+        if (result.getDatasets() != null) {
+          datasets.addAll(result.getDatasets());
+        }
+      } while (nextPageToken != null && !nextPageToken.isEmpty());
+      return datasets;
+    } catch (IOException e) {
+      throw new RuntimeIOException(e);
+    }
+  }
+
+  @Override
+  public Table createTable(Table table) {
+    // Ensure it is an Iceberg table supported by the BQ metastore catalog.
+    validateTable(table);
+    // TODO (b/399885863): Ensure table creation is idempotent when handling retries.
+    Table response = null;
+    try {
+      response =
+          BigQueryRetryHelper.runWithRetries(
+              new Callable<Table>() {
+                @Override
+                public Table call() {
+                  return create(table);
+                }
+              },
+              bigqueryOptions.getRetrySettings(),
+              BIGQUERY_EXCEPTION_HANDLER,
+              bigqueryOptions.getClock(),
+              DEFAULT_RETRY_CONFIG);
+    } catch (BigQueryRetryHelper.BigQueryRetryHelperException e) {
+      handleBigQueryRetryException(e);
+    }
+    return response;
+  }
+
+  @SuppressWarnings("FormatStringAnnotation")
+  private Table create(Table table) {
+    try {
+      HttpResponse response =
+          client
+              .tables()
+              .insert(
+                  Preconditions.checkNotNull(table.getTableReference()).getProjectId(),
+                  Preconditions.checkNotNull(table.getTableReference()).getDatasetId(),
+                  table)
+              .executeUnparsed();
+      return convertExceptionIfUnsuccessful(response).parseAs(Table.class);
+    } catch (IOException e) {
+      throw new RuntimeIOException(e);
+    } catch (AlreadyExistsException e) {
+      throw new AlreadyExistsException("Table already exists", e);
+    }
+  }
+
+  @Override
+  @SuppressWarnings("FormatStringAnnotation")
+  public Table getTable(TableReference tableReference) {
+    try {
+      HttpResponse response =
+          client
+              .tables()
+              .get(
+                  tableReference.getProjectId(),
+                  tableReference.getDatasetId(),
+                  tableReference.getTableId())
+              .executeUnparsed();
+      if (response.getStatusCode() == HttpStatusCodes.STATUS_CODE_NOT_FOUND) {
+        throw new NoSuchTableException(response.getStatusMessage());
+      }
+      return validateTable(convertExceptionIfUnsuccessful(response).parseAs(Table.class));
+    } catch (IOException e) {
+      throw new RuntimeIOException(e);
+    }
+  }
+
+  @Override
+  @SuppressWarnings("FormatStringAnnotation")
+  public Table patchTable(TableReference tableReference, Table table) {
+    // Ensure it is an Iceberg table supported by the BQ metastore catalog.
+    validateTable(table);
+
+    ExternalCatalogTableOptions newExternalCatalogTableOptions =
+        new ExternalCatalogTableOptions()
+            .setStorageDescriptor(table.getExternalCatalogTableOptions().getStorageDescriptor())
+            .setConnectionId(table.getExternalCatalogTableOptions().getConnectionId())
+            .setParameters(table.getExternalCatalogTableOptions().getParameters());
+    Table updatedTable =
+        new Table()
+            .setExternalCatalogTableOptions(newExternalCatalogTableOptions)
+            // Must set the schema as null for using schema auto-detect.
+            .setSchema(Data.nullOf(TableSchema.class));
+
+    Table response = null;
+    try {
+      response =
+          BigQueryRetryHelper.runWithRetries(
+              new Callable<Table>() {
+                @Override
+                public Table call() {
+                  return patch(tableReference, updatedTable, table.getEtag());
+                }
+              },
+              bigqueryOptions.getRetrySettings(),
+              BIGQUERY_EXCEPTION_HANDLER,
+              bigqueryOptions.getClock(),
+              DEFAULT_RETRY_CONFIG);
+    } catch (BigQueryRetryHelper.BigQueryRetryHelperException e) {
+      handleBigQueryRetryException(e);
+    }
+    return response;
+  }
+
+  @SuppressWarnings("FormatStringAnnotation")
+  private Table patch(TableReference tableReference, Table table, String etag) {
+    try {
+      HttpResponse response =
+          client
+              .tables()
+              .patch(
+                  tableReference.getProjectId(),
+                  tableReference.getDatasetId(),
+                  tableReference.getTableId(),
+                  table)
+              .setRequestHeaders(new HttpHeaders().setIfMatch(etag))
+              .executeUnparsed();
+
+      if (response.getStatusCode() == HttpStatusCodes.STATUS_CODE_NOT_FOUND) {
+        String responseString = response.parseAsString();
+        if (responseString.toLowerCase(Locale.ROOT).contains("not found: connection")) {
+          throw new BadRequestException(responseString);
+        }
+        throw new NoSuchTableException(response.getStatusMessage());
+      }
+      return convertExceptionIfUnsuccessful(response).parseAs(Table.class);
+    } catch (IOException e) {
+      throw new RuntimeIOException(e);
+    }
+  }
+
+  @Override
+  @SuppressWarnings("FormatStringAnnotation")
+  public Table renameTable(TableReference tableToRename, String newTableId) {
+    Table table = getTable(tableToRename); // Verify table first
+    Table patch =
+        new Table()
+            .setTableReference(
+                new TableReference()
+                    .setProjectId(table.getTableReference().getProjectId())
+                    .setDatasetId(table.getTableReference().getDatasetId())
+                    .setTableId(newTableId));
+
+    try {
+      HttpResponse response =
+          client
+              .tables()
+              .patch(
+                  tableToRename.getProjectId(),
+                  tableToRename.getDatasetId(),
+                  tableToRename.getTableId(),
+                  patch)
+              .setRequestHeaders(new HttpHeaders().setIfMatch(table.getEtag()))
+              .executeUnparsed();
+
+      if (response.getStatusCode() == HttpStatusCodes.STATUS_CODE_NOT_FOUND) {
+        throw new NoSuchTableException(response.getStatusMessage());
+      }
+      return convertExceptionIfUnsuccessful(response).parseAs(Table.class);
+    } catch (IOException e) {
+      throw new RuntimeIOException(e);
+    }
+  }
+
+  @Override
+  @SuppressWarnings("FormatStringAnnotation")
+  public void deleteTable(TableReference tableReference) {
+    try {
+      getTable(tableReference); // Fetching it to validate it is a BigQuery Metastore table first
+
+      HttpResponse response =
+          client
+              .tables()
+              .delete(
+                  tableReference.getProjectId(),
+                  tableReference.getDatasetId(),
+                  tableReference.getTableId())
+              .executeUnparsed();
+
+      if (response.getStatusCode() == HttpStatusCodes.STATUS_CODE_NOT_FOUND) {
+        throw new NoSuchTableException(response.getStatusMessage());
+      }
+      convertExceptionIfUnsuccessful(response);
+    } catch (IOException e) {
+      throw new RuntimeIOException(e);
+    }
+  }
+
+  @Override
+  @SuppressWarnings("FormatStringAnnotation")
+  public List<Tables> listTables(
+      DatasetReference datasetReference, boolean filterUnsupportedTables) {
+    try {
+      String nextPageToken = null;
+      Stream<Tables> tablesStream = Stream.empty();
+      do {
+        HttpResponse pageResponse =
+            client
+                .tables()
+                .list(datasetReference.getProjectId(), datasetReference.getDatasetId())
+                .setPageToken(nextPageToken)
+                .executeUnparsed();
+        if (pageResponse.getStatusCode() == HttpStatusCodes.STATUS_CODE_NOT_FOUND) {
+          throw new NoSuchNamespaceException(pageResponse.getStatusMessage());
+        }
+        TableList result = convertExceptionIfUnsuccessful(pageResponse).parseAs(TableList.class);
+        nextPageToken = result.getNextPageToken();
+        List<Tables> tablesPage = result.getTables();
+        Stream<Tables> tablesPageStream =
+            tablesPage == null ? Stream.empty() : result.getTables().stream();
+        tablesStream = Stream.concat(tablesStream, tablesPageStream);
+      } while (nextPageToken != null && !nextPageToken.isEmpty());
+
+      // TODO(b/345839927): The server should return more metadata here to distinguish Iceberg
+      // BQMS tables for us to filter out those results since invoking `getTable` on them would
+      // correctly raise a `NoSuchIcebergTableException` for being inoperable by this plugin.
+      if (filterUnsupportedTables) {
+        tablesStream =
+            tablesStream
+                .parallel()
+                .filter(
+                    table -> {
+                      try {
+                        getTable(table.getTableReference());
+                      } catch (NoSuchTableException e) {
+                        return false;
+                      }
+                      return true;
+                    });
+      }
+
+      return tablesStream.collect(Collectors.toList());
+    } catch (IOException e) {
+      throw new RuntimeIOException(e);
+    }
+  }
+
+  @SuppressWarnings("FormatStringAnnotation")
+  private Dataset updateDataset(Dataset dataset) {
+    try {
+      HttpResponse response =
+          client
+              .datasets()
+              .update(
+                  Preconditions.checkNotNull(dataset.getDatasetReference()).getProjectId(),
+                  Preconditions.checkNotNull(dataset.getDatasetReference().getDatasetId()),
+                  dataset)
+              .setRequestHeaders(new HttpHeaders().setIfMatch(dataset.getEtag()))
+              .executeUnparsed();
+      if (response.getStatusCode() == HttpStatusCodes.STATUS_CODE_NOT_FOUND) {
+        throw new NoSuchNamespaceException(response.getStatusMessage());
+      }
+      return convertExceptionIfUnsuccessful(response).parseAs(Dataset.class);
+    } catch (IOException e) {
+      throw new RuntimeIOException(e);
+    }
+  }
+
+  /**
+   * Returns true when it is a BigQuery Metastore Iceberg table, defined by having the
+   * ExternalCatalogTableOptions object and a parameter of METADATA_LOCATION_PROP as part of its
+   * parameters map.
+   *
+   * @param table to check
+   */
+  private boolean isValidIcebergTable(Table table) {
+    return table.getExternalCatalogTableOptions() != null
+        && !table.getExternalCatalogTableOptions().isEmpty()
+        && table.getExternalCatalogTableOptions().getParameters() != null
+        && table
+            .getExternalCatalogTableOptions()
+            .getParameters()
+            .containsKey(BaseMetastoreTableOperations.METADATA_LOCATION_PROP)
+        && BaseMetastoreTableOperations.ICEBERG_TABLE_TYPE_VALUE.equalsIgnoreCase(
+            table
+                .getExternalCatalogTableOptions()
+                .getParameters()
+                .get(BaseMetastoreTableOperations.TABLE_TYPE_PROP));

Review Comment:
   check this key is present in the map?



-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org

For queries about this service, please contact Infrastructure at:
users@infra.apache.org


---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For additional commands, e-mail: issues-help@iceberg.apache.org


Re: [PR] Catalog: Add BigQuery Metastore Catalog Support [iceberg]

Posted by "talatuyarer (via GitHub)" <gi...@apache.org>.
talatuyarer commented on code in PR #12808:
URL: https://github.com/apache/iceberg/pull/12808#discussion_r2084990687


##########
bigquery/src/main/java/org/apache/iceberg/gcp/bigquery/BigQueryMetastoreCatalog.java:
##########
@@ -0,0 +1,366 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.iceberg.gcp.bigquery;
+
+import com.google.api.services.bigquery.model.Dataset;
+import com.google.api.services.bigquery.model.DatasetList.Datasets;
+import com.google.api.services.bigquery.model.DatasetReference;
+import com.google.api.services.bigquery.model.ExternalCatalogDatasetOptions;
+import com.google.api.services.bigquery.model.TableReference;
+import com.google.cloud.ServiceOptions;
+import com.google.cloud.bigquery.BigQueryOptions;
+import java.io.IOException;
+import java.io.UncheckedIOException;
+import java.security.GeneralSecurityException;
+import java.util.List;
+import java.util.Locale;
+import java.util.Map;
+import java.util.Objects;
+import java.util.Set;
+import org.apache.hadoop.conf.Configurable;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.iceberg.BaseMetastoreCatalog;
+import org.apache.iceberg.CatalogProperties;
+import org.apache.iceberg.CatalogUtil;
+import org.apache.iceberg.TableMetadata;
+import org.apache.iceberg.TableOperations;
+import org.apache.iceberg.catalog.Namespace;
+import org.apache.iceberg.catalog.SupportsNamespaces;
+import org.apache.iceberg.catalog.TableIdentifier;
+import org.apache.iceberg.exceptions.NoSuchNamespaceException;
+import org.apache.iceberg.exceptions.NoSuchTableException;
+import org.apache.iceberg.io.FileIO;
+import org.apache.iceberg.relocated.com.google.common.annotations.VisibleForTesting;
+import org.apache.iceberg.relocated.com.google.common.base.Preconditions;
+import org.apache.iceberg.relocated.com.google.common.base.Strings;
+import org.apache.iceberg.relocated.com.google.common.collect.ImmutableList;
+import org.apache.iceberg.relocated.com.google.common.collect.ImmutableMap;
+import org.apache.iceberg.relocated.com.google.common.collect.Maps;
+import org.apache.iceberg.util.LocationUtil;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/** Iceberg Bigquery Metastore Catalog implementation. */
+public class BigQueryMetastoreCatalog extends BaseMetastoreCatalog
+    implements SupportsNamespaces, Configurable {
+
+  // User provided properties.
+  public static final String PROJECT_ID = "gcp.bigquery.project-id";
+  public static final String GCP_LOCATION = "gcp.bigquery.location";
+  public static final String LIST_ALL_TABLES = "gcp.bigquery.list-all-tables";
+
+  private static final Logger LOG = LoggerFactory.getLogger(BigQueryMetastoreCatalog.class);
+
+  private static final String DEFAULT_GCP_LOCATION = "us";
+
+  private String catalogName;
+  private Map<String, String> catalogProperties;
+  private FileIO fileIO;
+  private Configuration conf;
+  private String projectId;
+  private String projectLocation;
+  private BigQueryMetastoreClient client;
+  private boolean listAllTables;
+  private String warehouseLocation;
+
+  public BigQueryMetastoreCatalog() {}
+
+  @Override
+  public void initialize(String name, Map<String, String> properties) {
+    Preconditions.checkArgument(
+        properties.containsKey(PROJECT_ID),
+        "Invalid GCP project: %s must be specified",
+        PROJECT_ID);
+
+    this.projectId = properties.get(PROJECT_ID);
+    this.projectLocation = properties.getOrDefault(GCP_LOCATION, DEFAULT_GCP_LOCATION);
+
+    BigQueryOptions options =
+        BigQueryOptions.newBuilder()
+            .setProjectId(projectId)
+            .setLocation(projectLocation)
+            .setRetrySettings(ServiceOptions.getDefaultRetrySettings())
+            .build();
+
+    try {
+      client = new BigQueryMetastoreClientImpl(options);
+    } catch (IOException e) {
+      throw new UncheckedIOException("Creating BigQuery client failed", e);
+    } catch (GeneralSecurityException e) {
+      throw new RuntimeException("Creating BigQuery client failed due to a security issue", e);
+    }
+
+    initialize(name, properties, projectId, projectLocation, client);
+  }
+
+  @VisibleForTesting
+  void initialize(
+      String name,
+      Map<String, String> properties,
+      String initialProjectId,
+      String initialLocation,
+      BigQueryMetastoreClient bigQueryMetaStoreClient) {
+    Preconditions.checkArgument(bigQueryMetaStoreClient != null, "Invalid BigQuery client: null");
+    this.catalogName = name;
+    this.catalogProperties = ImmutableMap.copyOf(properties);
+    this.projectId = initialProjectId;
+    this.projectLocation = initialLocation;
+    this.client = bigQueryMetaStoreClient;
+
+    if (this.conf == null) {
+      LOG.warn("No configuration was set, using the default environment Configuration");
+      this.conf = new Configuration();
+    }
+
+    LOG.info("Using BigQuery Metastore Iceberg Catalog: {}", name);
+
+    if (properties.containsKey(CatalogProperties.WAREHOUSE_LOCATION)) {
+      this.warehouseLocation =
+          LocationUtil.stripTrailingSlash(properties.get(CatalogProperties.WAREHOUSE_LOCATION));
+    }
+
+    this.fileIO =
+        CatalogUtil.loadFileIO(
+            properties.getOrDefault(
+                CatalogProperties.FILE_IO_IMPL, "org.apache.iceberg.io.ResolvingFileIO"),
+            properties,
+            conf);
+
+    this.listAllTables = Boolean.parseBoolean(properties.getOrDefault(LIST_ALL_TABLES, "true"));
+  }
+
+  @Override
+  protected TableOperations newTableOps(TableIdentifier identifier) {
+    return new BigQueryTableOperations(client, fileIO, toTableReference(identifier), conf);
+  }
+
+  @Override
+  protected String defaultWarehouseLocation(TableIdentifier identifier) {
+    String locationUri = null;
+    DatasetReference datasetReference = toDatasetReference(identifier.namespace());
+    Dataset dataset = client.load(datasetReference);
+    if (dataset != null && dataset.getExternalCatalogDatasetOptions() != null) {
+      locationUri = dataset.getExternalCatalogDatasetOptions().getDefaultStorageLocationUri();
+    }
+
+    return String.format(
+        "%s/%s",
+        Strings.isNullOrEmpty(locationUri)
+            ? createDefaultStorageLocationUri(datasetReference.getDatasetId())
+            : LocationUtil.stripTrailingSlash(locationUri),
+        identifier.name());
+  }
+
+  @Override
+  public List<TableIdentifier> listTables(Namespace namespace) {
+    validateNamespace(namespace);
+
+    return client.list(toDatasetReference(namespace), listAllTables).stream()
+        .map(
+            table -> TableIdentifier.of(namespace.level(0), table.getTableReference().getTableId()))
+        .collect(ImmutableList.toImmutableList());
+  }
+
+  @Override
+  public boolean dropTable(TableIdentifier identifier, boolean purge) {
+    try {
+      TableOperations ops = newTableOps(identifier);
+      TableMetadata lastMetadata = ops.current();
+
+      client.delete(toTableReference(identifier));
+
+      if (purge && lastMetadata != null) {
+        CatalogUtil.dropTableData(ops.io(), lastMetadata);
+      }
+    } catch (NoSuchTableException e) {
+      return false;
+    }
+
+    return true;
+  }
+
+  @Override
+  public void renameTable(TableIdentifier from, TableIdentifier to) {
+    // TODO: Enable once supported by BigQuery API.
+    throw new UnsupportedOperationException("Table rename operation is unsupported.");
+  }
+
+  @Override
+  public void createNamespace(Namespace namespace, Map<String, String> metadata) {
+    Dataset builder = new Dataset();
+    DatasetReference datasetReference = toDatasetReference(namespace);
+    builder.setLocation(this.projectLocation);
+    builder.setDatasetReference(datasetReference);
+    builder.setExternalCatalogDatasetOptions(
+        BigQueryMetastoreUtils.createExternalCatalogDatasetOptions(
+            createDefaultStorageLocationUri(datasetReference.getDatasetId()), metadata));
+
+    client.create(builder);
+  }
+
+  @Override
+  public List<Namespace> listNamespaces() {
+    try {
+      return listNamespaces(Namespace.empty());
+    } catch (NoSuchNamespaceException e) {
+      return ImmutableList.of();
+    }
+  }
+
+  /**
+   * Since this catalog only supports one-level namespaces, it always returns an empty list unless
+   * passed an empty namespace to list all namespaces within the catalog.
+   */
+  @Override
+  public List<Namespace> listNamespaces(Namespace namespace) {
+    List<Datasets> allDatasets = client.list(projectId);
+
+    ImmutableList<Namespace> namespaces =
+        allDatasets.stream().map(this::toNamespace).collect(ImmutableList.toImmutableList());
+    if (namespaces.isEmpty()) {
+      throw new NoSuchNamespaceException("Namespace does not exist: %s", namespace);
+    }
+
+    return namespaces;
+  }
+
+  @Override
+  public boolean dropNamespace(Namespace namespace) {
+    try {
+      client.delete(toDatasetReference(namespace));
+      // We don't delete the data folder for safety, which aligns with Hive Metastore's default
+      // behavior.
+      // We can support database or catalog level config controlling file deletion in the future.
+      return true;
+    } catch (NoSuchNamespaceException e) {
+      return false;
+    }
+  }
+
+  @Override
+  public boolean setProperties(Namespace namespace, Map<String, String> properties) {
+    Dataset dataset = client.load(toDatasetReference(namespace));
+
+    ExternalCatalogDatasetOptions existingOptions = dataset.getExternalCatalogDatasetOptions();
+    Map<String, String> existingParameters =
+        existingOptions != null ? existingOptions.getParameters() : null;
+
+    Map<String, String> newParameters = Maps.newHashMap();
+    if (existingParameters != null) {
+      newParameters.putAll(existingParameters);
+    }
+
+    newParameters.putAll(properties);
+
+    if (Objects.equals(existingParameters, newParameters)) {
+      // No change in parameters detected
+      return false;
+    }
+
+    client.setParameters(toDatasetReference(namespace), properties);
+    return true;
+  }
+
+  @Override
+  public boolean removeProperties(Namespace namespace, Set<String> properties) {
+    client.removeParameters(toDatasetReference(namespace), properties);
+    return true;

Review Comment:
   Done The code follows JDBC's behaviour now. because i did not touch this line. The comment does not shown as outdated. 



-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org

For queries about this service, please contact Infrastructure at:
users@infra.apache.org


---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For additional commands, e-mail: issues-help@iceberg.apache.org


Re: [PR] Catalog: Add BigQuery Metastore Catalog Support [iceberg]

Posted by "talatuyarer (via GitHub)" <gi...@apache.org>.
talatuyarer commented on code in PR #12808:
URL: https://github.com/apache/iceberg/pull/12808#discussion_r2081046612


##########
bigquery/src/main/java/org/apache/iceberg/gcp/bigquery/BigQueryMetastoreCatalog.java:
##########
@@ -0,0 +1,385 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.iceberg.gcp.bigquery;
+
+import com.google.api.services.bigquery.model.Dataset;
+import com.google.api.services.bigquery.model.DatasetList.Datasets;
+import com.google.api.services.bigquery.model.DatasetReference;
+import com.google.api.services.bigquery.model.ExternalCatalogDatasetOptions;
+import com.google.api.services.bigquery.model.TableReference;
+import com.google.cloud.ServiceOptions;
+import com.google.cloud.bigquery.BigQueryOptions;
+import java.io.IOException;
+import java.io.UncheckedIOException;
+import java.security.GeneralSecurityException;
+import java.util.List;
+import java.util.Locale;
+import java.util.Map;
+import java.util.Objects;
+import java.util.Set;
+import org.apache.hadoop.conf.Configurable;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.iceberg.BaseMetastoreCatalog;
+import org.apache.iceberg.CatalogProperties;
+import org.apache.iceberg.CatalogUtil;
+import org.apache.iceberg.TableMetadata;
+import org.apache.iceberg.TableOperations;
+import org.apache.iceberg.catalog.Namespace;
+import org.apache.iceberg.catalog.SupportsNamespaces;
+import org.apache.iceberg.catalog.TableIdentifier;
+import org.apache.iceberg.exceptions.NoSuchNamespaceException;
+import org.apache.iceberg.exceptions.NoSuchTableException;
+import org.apache.iceberg.io.FileIO;
+import org.apache.iceberg.relocated.com.google.common.annotations.VisibleForTesting;
+import org.apache.iceberg.relocated.com.google.common.base.Preconditions;
+import org.apache.iceberg.relocated.com.google.common.base.Strings;
+import org.apache.iceberg.relocated.com.google.common.collect.ImmutableList;
+import org.apache.iceberg.relocated.com.google.common.collect.ImmutableMap;
+import org.apache.iceberg.relocated.com.google.common.collect.Maps;
+import org.apache.iceberg.util.LocationUtil;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/** Iceberg Bigquery Metastore Catalog implementation. */
+public class BigQueryMetastoreCatalog extends BaseMetastoreCatalog
+    implements SupportsNamespaces, Configurable {
+
+  // User provided properties.
+  public static final String PROJECT_ID = "gcp.bigquery.project-id";
+  public static final String GCP_LOCATION = "gcp.bigquery.location";
+  public static final String LIST_ALL_TABLES = "gcp.bigquery.list-all-tables";
+
+  private static final Logger LOG = LoggerFactory.getLogger(BigQueryMetastoreCatalog.class);
+
+  private static final String DEFAULT_GCP_LOCATION = "us";
+
+  private String catalogName;
+  private Map<String, String> catalogProperties;
+  private FileIO fileIO;
+  private Configuration conf;
+  private String projectId;
+  private String projectLocation;
+  private BigQueryMetastoreClient client;
+  private boolean listAllTables;
+  private String warehouseLocation;
+
+  public BigQueryMetastoreCatalog() {}
+
+  @Override
+  public void initialize(String name, Map<String, String> properties) {
+    Preconditions.checkArgument(
+        properties.containsKey(PROJECT_ID),
+        "Invalid GCP project: %s must be specified",
+        PROJECT_ID);

Review Comment:
   You can find my comment under your other comment. 



##########
bigquery/src/main/java/org/apache/iceberg/gcp/bigquery/BigQueryMetastoreClientImpl.java:
##########
@@ -0,0 +1,579 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.iceberg.gcp.bigquery;
+
+import com.google.api.client.googleapis.javanet.GoogleNetHttpTransport;
+import com.google.api.client.http.HttpHeaders;
+import com.google.api.client.http.HttpResponse;
+import com.google.api.client.http.HttpResponseException;
+import com.google.api.client.http.HttpStatusCodes;
+import com.google.api.client.json.gson.GsonFactory;
+import com.google.api.client.util.Data;
+import com.google.api.services.bigquery.Bigquery;
+import com.google.api.services.bigquery.BigqueryScopes;
+import com.google.api.services.bigquery.model.Dataset;
+import com.google.api.services.bigquery.model.DatasetList;
+import com.google.api.services.bigquery.model.DatasetList.Datasets;
+import com.google.api.services.bigquery.model.DatasetReference;
+import com.google.api.services.bigquery.model.ExternalCatalogDatasetOptions;
+import com.google.api.services.bigquery.model.ExternalCatalogTableOptions;
+import com.google.api.services.bigquery.model.Table;
+import com.google.api.services.bigquery.model.TableList;
+import com.google.api.services.bigquery.model.TableList.Tables;
+import com.google.api.services.bigquery.model.TableReference;
+import com.google.api.services.bigquery.model.TableSchema;
+import com.google.auth.http.HttpCredentialsAdapter;
+import com.google.auth.oauth2.GoogleCredentials;
+import com.google.cloud.BaseServiceException;
+import com.google.cloud.ExceptionHandler;
+import com.google.cloud.bigquery.BigQueryErrorMessages;
+import com.google.cloud.bigquery.BigQueryOptions;
+import com.google.cloud.bigquery.BigQueryRetryConfig;
+import com.google.cloud.bigquery.BigQueryRetryHelper;
+import java.io.IOException;
+import java.nio.charset.StandardCharsets;
+import java.security.GeneralSecurityException;
+import java.util.List;
+import java.util.Locale;
+import java.util.Map;
+import java.util.Set;
+import java.util.stream.Collectors;
+import java.util.stream.Stream;
+import org.apache.iceberg.BaseMetastoreTableOperations;
+import org.apache.iceberg.exceptions.AlreadyExistsException;
+import org.apache.iceberg.exceptions.BadRequestException;
+import org.apache.iceberg.exceptions.ForbiddenException;
+import org.apache.iceberg.exceptions.NoSuchIcebergTableException;
+import org.apache.iceberg.exceptions.NoSuchNamespaceException;
+import org.apache.iceberg.exceptions.NoSuchTableException;
+import org.apache.iceberg.exceptions.NotAuthorizedException;
+import org.apache.iceberg.exceptions.RuntimeIOException;
+import org.apache.iceberg.exceptions.ServiceFailureException;
+import org.apache.iceberg.exceptions.ServiceUnavailableException;
+import org.apache.iceberg.exceptions.ValidationException;
+import org.apache.iceberg.relocated.com.google.common.base.Preconditions;
+import org.apache.iceberg.relocated.com.google.common.collect.Lists;
+import org.apache.iceberg.relocated.com.google.common.collect.Maps;
+
+/** A client of Google Bigquery Metastore functions over the BigQuery service. */
+public final class BigQueryMetastoreClientImpl implements BigQueryMetastoreClient {
+
+  private final Bigquery client;
+  private final BigQueryOptions bigqueryOptions;
+
+  public static final ExceptionHandler.Interceptor EXCEPTION_HANDLER_INTERCEPTOR =
+      new ExceptionHandler.Interceptor() {
+
+        @Override
+        public RetryResult afterEval(Exception exception, RetryResult retryResult) {
+          return ExceptionHandler.Interceptor.RetryResult.CONTINUE_EVALUATION;
+        }
+
+        @Override
+        public RetryResult beforeEval(Exception exception) {
+          if (exception instanceof BaseServiceException) {
+            boolean retriable = ((BaseServiceException) exception).isRetryable();
+            return retriable
+                ? ExceptionHandler.Interceptor.RetryResult.RETRY
+                : ExceptionHandler.Interceptor.RetryResult.CONTINUE_EVALUATION;
+          }
+
+          return ExceptionHandler.Interceptor.RetryResult.CONTINUE_EVALUATION;
+        }
+      };
+
+  // Retry config with error messages and regex for rate limit exceeded errors.
+  private static final BigQueryRetryConfig DEFAULT_RETRY_CONFIG =
+      BigQueryRetryConfig.newBuilder()
+          .retryOnMessage(BigQueryErrorMessages.RATE_LIMIT_EXCEEDED_MSG)
+          .retryOnMessage(BigQueryErrorMessages.JOB_RATE_LIMIT_EXCEEDED_MSG)
+          .retryOnRegEx(BigQueryErrorMessages.RetryRegExPatterns.RATE_LIMIT_EXCEEDED_REGEX)
+          .build();
+
+  public static final ExceptionHandler BIGQUERY_EXCEPTION_HANDLER =
+      ExceptionHandler.newBuilder()
+          .abortOn(RuntimeException.class)
+          // Retry on connection failures due to transient network issues.
+          .retryOn(java.net.ConnectException.class)
+          // Retry to recover from temporary DNS resolution failures.
+          .retryOn(java.net.UnknownHostException.class)
+          .addInterceptors(EXCEPTION_HANDLER_INTERCEPTOR)
+          .build();
+
+  /** Constructs a client of the Google BigQuery service. */
+  public BigQueryMetastoreClientImpl(BigQueryOptions options)
+      throws IOException, GeneralSecurityException {
+    // Initialize client that will be used to send requests. This client only needs to be created
+    // once, and can be reused for multiple requests
+    HttpCredentialsAdapter httpCredentialsAdapter =
+        new HttpCredentialsAdapter(
+            GoogleCredentials.getApplicationDefault().createScoped(BigqueryScopes.all()));

Review Comment:
   Thank you @ebyhr Good catch! If it is not very important for you. I would like to address that support after merging this. I already received so much comments on this PR. I want to merge this first. This PR will cover most of use cases and also provide main basic functionality for GCP users.



##########
bigquery/src/main/java/org/apache/iceberg/gcp/bigquery/BigQueryMetastoreCatalog.java:
##########
@@ -0,0 +1,366 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.iceberg.gcp.bigquery;
+
+import com.google.api.services.bigquery.model.Dataset;
+import com.google.api.services.bigquery.model.DatasetList.Datasets;
+import com.google.api.services.bigquery.model.DatasetReference;
+import com.google.api.services.bigquery.model.ExternalCatalogDatasetOptions;
+import com.google.api.services.bigquery.model.TableReference;
+import com.google.cloud.ServiceOptions;
+import com.google.cloud.bigquery.BigQueryOptions;
+import java.io.IOException;
+import java.io.UncheckedIOException;
+import java.security.GeneralSecurityException;
+import java.util.List;
+import java.util.Locale;
+import java.util.Map;
+import java.util.Objects;
+import java.util.Set;
+import org.apache.hadoop.conf.Configurable;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.iceberg.BaseMetastoreCatalog;
+import org.apache.iceberg.CatalogProperties;
+import org.apache.iceberg.CatalogUtil;
+import org.apache.iceberg.TableMetadata;
+import org.apache.iceberg.TableOperations;
+import org.apache.iceberg.catalog.Namespace;
+import org.apache.iceberg.catalog.SupportsNamespaces;
+import org.apache.iceberg.catalog.TableIdentifier;
+import org.apache.iceberg.exceptions.NoSuchNamespaceException;
+import org.apache.iceberg.exceptions.NoSuchTableException;
+import org.apache.iceberg.io.FileIO;
+import org.apache.iceberg.relocated.com.google.common.annotations.VisibleForTesting;
+import org.apache.iceberg.relocated.com.google.common.base.Preconditions;
+import org.apache.iceberg.relocated.com.google.common.base.Strings;
+import org.apache.iceberg.relocated.com.google.common.collect.ImmutableList;
+import org.apache.iceberg.relocated.com.google.common.collect.ImmutableMap;
+import org.apache.iceberg.relocated.com.google.common.collect.Maps;
+import org.apache.iceberg.util.LocationUtil;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/** Iceberg Bigquery Metastore Catalog implementation. */
+public class BigQueryMetastoreCatalog extends BaseMetastoreCatalog
+    implements SupportsNamespaces, Configurable {
+
+  // User provided properties.
+  public static final String PROJECT_ID = "gcp.bigquery.project-id";
+  public static final String GCP_LOCATION = "gcp.bigquery.location";
+  public static final String LIST_ALL_TABLES = "gcp.bigquery.list-all-tables";
+
+  private static final Logger LOG = LoggerFactory.getLogger(BigQueryMetastoreCatalog.class);
+
+  private static final String DEFAULT_GCP_LOCATION = "us";
+
+  private String catalogName;
+  private Map<String, String> catalogProperties;
+  private FileIO fileIO;
+  private Configuration conf;
+  private String projectId;
+  private String projectLocation;
+  private BigQueryMetastoreClient client;
+  private boolean listAllTables;
+  private String warehouseLocation;
+
+  public BigQueryMetastoreCatalog() {}
+
+  @Override
+  public void initialize(String name, Map<String, String> properties) {
+    Preconditions.checkArgument(
+        properties.containsKey(PROJECT_ID),
+        "Invalid GCP project: %s must be specified",
+        PROJECT_ID);
+
+    this.projectId = properties.get(PROJECT_ID);
+    this.projectLocation = properties.getOrDefault(GCP_LOCATION, DEFAULT_GCP_LOCATION);
+
+    BigQueryOptions options =
+        BigQueryOptions.newBuilder()
+            .setProjectId(projectId)
+            .setLocation(projectLocation)
+            .setRetrySettings(ServiceOptions.getDefaultRetrySettings())
+            .build();
+
+    try {
+      client = new BigQueryMetastoreClientImpl(options);
+    } catch (IOException e) {
+      throw new UncheckedIOException("Creating BigQuery client failed", e);
+    } catch (GeneralSecurityException e) {
+      throw new RuntimeException("Creating BigQuery client failed due to a security issue", e);
+    }
+
+    initialize(name, properties, projectId, projectLocation, client);
+  }
+
+  @VisibleForTesting
+  void initialize(
+      String name,
+      Map<String, String> properties,
+      String initialProjectId,
+      String initialLocation,
+      BigQueryMetastoreClient bigQueryMetaStoreClient) {
+    Preconditions.checkArgument(bigQueryMetaStoreClient != null, "Invalid BigQuery client: null");
+    this.catalogName = name;
+    this.catalogProperties = ImmutableMap.copyOf(properties);
+    this.projectId = initialProjectId;
+    this.projectLocation = initialLocation;
+    this.client = bigQueryMetaStoreClient;
+
+    if (this.conf == null) {
+      LOG.warn("No configuration was set, using the default environment Configuration");
+      this.conf = new Configuration();
+    }
+
+    LOG.info("Using BigQuery Metastore Iceberg Catalog: {}", name);
+
+    if (properties.containsKey(CatalogProperties.WAREHOUSE_LOCATION)) {
+      this.warehouseLocation =
+          LocationUtil.stripTrailingSlash(properties.get(CatalogProperties.WAREHOUSE_LOCATION));
+    }
+
+    this.fileIO =
+        CatalogUtil.loadFileIO(
+            properties.getOrDefault(
+                CatalogProperties.FILE_IO_IMPL, "org.apache.iceberg.io.ResolvingFileIO"),
+            properties,
+            conf);
+
+    this.listAllTables = Boolean.parseBoolean(properties.getOrDefault(LIST_ALL_TABLES, "true"));
+  }
+
+  @Override
+  protected TableOperations newTableOps(TableIdentifier identifier) {
+    return new BigQueryTableOperations(client, fileIO, toTableReference(identifier), conf);
+  }
+
+  @Override
+  protected String defaultWarehouseLocation(TableIdentifier identifier) {
+    String locationUri = null;
+    DatasetReference datasetReference = toDatasetReference(identifier.namespace());
+    Dataset dataset = client.load(datasetReference);
+    if (dataset != null && dataset.getExternalCatalogDatasetOptions() != null) {
+      locationUri = dataset.getExternalCatalogDatasetOptions().getDefaultStorageLocationUri();
+    }
+
+    return String.format(
+        "%s/%s",
+        Strings.isNullOrEmpty(locationUri)
+            ? createDefaultStorageLocationUri(datasetReference.getDatasetId())
+            : LocationUtil.stripTrailingSlash(locationUri),
+        identifier.name());
+  }
+
+  @Override
+  public List<TableIdentifier> listTables(Namespace namespace) {
+    validateNamespace(namespace);
+
+    return client.list(toDatasetReference(namespace), listAllTables).stream()
+        .map(
+            table -> TableIdentifier.of(namespace.level(0), table.getTableReference().getTableId()))
+        .collect(ImmutableList.toImmutableList());
+  }
+
+  @Override
+  public boolean dropTable(TableIdentifier identifier, boolean purge) {
+    try {
+      TableOperations ops = newTableOps(identifier);
+      TableMetadata lastMetadata = ops.current();
+
+      client.delete(toTableReference(identifier));
+
+      if (purge && lastMetadata != null) {
+        CatalogUtil.dropTableData(ops.io(), lastMetadata);
+      }
+    } catch (NoSuchTableException e) {
+      return false;
+    }
+
+    return true;
+  }
+
+  @Override
+  public void renameTable(TableIdentifier from, TableIdentifier to) {
+    // TODO: Enable once supported by BigQuery API.
+    throw new UnsupportedOperationException("Table rename operation is unsupported.");
+  }
+
+  @Override
+  public void createNamespace(Namespace namespace, Map<String, String> metadata) {
+    Dataset builder = new Dataset();
+    DatasetReference datasetReference = toDatasetReference(namespace);
+    builder.setLocation(this.projectLocation);
+    builder.setDatasetReference(datasetReference);
+    builder.setExternalCatalogDatasetOptions(
+        BigQueryMetastoreUtils.createExternalCatalogDatasetOptions(
+            createDefaultStorageLocationUri(datasetReference.getDatasetId()), metadata));
+
+    client.create(builder);
+  }
+
+  @Override
+  public List<Namespace> listNamespaces() {
+    try {
+      return listNamespaces(Namespace.empty());
+    } catch (NoSuchNamespaceException e) {
+      return ImmutableList.of();
+    }
+  }
+
+  /**
+   * Since this catalog only supports one-level namespaces, it always returns an empty list unless
+   * passed an empty namespace to list all namespaces within the catalog.
+   */
+  @Override
+  public List<Namespace> listNamespaces(Namespace namespace) {
+    List<Datasets> allDatasets = client.list(projectId);
+
+    ImmutableList<Namespace> namespaces =
+        allDatasets.stream().map(this::toNamespace).collect(ImmutableList.toImmutableList());
+    if (namespaces.isEmpty()) {
+      throw new NoSuchNamespaceException("Namespace does not exist: %s", namespace);
+    }
+
+    return namespaces;
+  }
+
+  @Override
+  public boolean dropNamespace(Namespace namespace) {
+    try {
+      client.delete(toDatasetReference(namespace));

Review Comment:
   I agree but I am testing against BigQueryProduction using real client too. Otherwise there is no point to have this code in Iceberg Repo :) 



-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org

For queries about this service, please contact Infrastructure at:
users@infra.apache.org


---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For additional commands, e-mail: issues-help@iceberg.apache.org


Re: [PR] Catalog: Add BigQuery Metastore Catalog Support [iceberg]

Posted by "talatuyarer (via GitHub)" <gi...@apache.org>.
talatuyarer commented on code in PR #12808:
URL: https://github.com/apache/iceberg/pull/12808#discussion_r2081050614


##########
bigquery/src/main/java/org/apache/iceberg/gcp/bigquery/BigQueryTableOperations.java:
##########
@@ -0,0 +1,296 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.iceberg.gcp.bigquery;
+
+import com.google.api.services.bigquery.model.ExternalCatalogTableOptions;
+import com.google.api.services.bigquery.model.Table;
+import com.google.api.services.bigquery.model.TableReference;
+import java.util.Locale;
+import java.util.Map;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hive.common.StatsSetupConst;
+import org.apache.iceberg.BaseMetastoreOperations;
+import org.apache.iceberg.BaseMetastoreTableOperations;
+import org.apache.iceberg.SnapshotSummary;
+import org.apache.iceberg.TableMetadata;
+import org.apache.iceberg.TableProperties;
+import org.apache.iceberg.exceptions.AlreadyExistsException;
+import org.apache.iceberg.exceptions.CommitFailedException;
+import org.apache.iceberg.exceptions.CommitStateUnknownException;
+import org.apache.iceberg.exceptions.NoSuchTableException;
+import org.apache.iceberg.exceptions.ValidationException;
+import org.apache.iceberg.hadoop.ConfigProperties;
+import org.apache.iceberg.io.FileIO;
+import org.apache.iceberg.relocated.com.google.common.collect.Maps;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/** Handles BigQuery metastore table operations. */
+public final class BigQueryTableOperations extends BaseMetastoreTableOperations {
+
+  private static final Logger LOG = LoggerFactory.getLogger(BigQueryTableOperations.class);
+
+  private static final String TABLE_PROPERTIES_BQ_CONNECTION = "bq_connection";
+
+  private final BigQueryMetastoreClient client;
+  private final FileIO fileIO;
+  private final TableReference tableReference;
+  private final Configuration conf;
+
+  BigQueryTableOperations(
+      BigQueryMetastoreClient client,
+      FileIO fileIO,
+      TableReference tableReference,
+      Configuration conf) {
+    this.client = client;
+    this.fileIO = fileIO;
+    this.tableReference = tableReference;
+    this.conf = conf;
+  }
+
+  // The doRefresh method should provide implementation on how to get the metadata location.
+  @Override
+  public void doRefresh() {
+    // Must default to null.
+    String metadataLocation = null;
+    try {
+      metadataLocation =
+          loadMetadataLocationOrThrow(
+              client.load(this.tableReference).getExternalCatalogTableOptions());
+    } catch (NoSuchTableException e) {
+      if (currentMetadataLocation() != null) {
+        // Re-throws the exception because the table must exist in this case.
+        throw e;
+      }
+    }
+    refreshFromMetadataLocation(metadataLocation);
+  }
+
+  // The doCommit method should provide implementation on how to update with metadata location
+  // atomically
+  @Override
+  public void doCommit(TableMetadata base, TableMetadata metadata) {
+    String newMetadataLocation =
+        base == null && metadata.metadataFileLocation() != null
+            ? metadata.metadataFileLocation()
+            : writeNewMetadata(metadata, currentVersion() + 1);
+    BaseMetastoreOperations.CommitStatus commitStatus =
+        BaseMetastoreOperations.CommitStatus.FAILURE;
+    try {
+      if (base == null) {
+        createTable(newMetadataLocation, metadata);
+      } else {
+        updateTable(base.metadataFileLocation(), newMetadataLocation, metadata);
+      }
+      commitStatus = BaseMetastoreOperations.CommitStatus.SUCCESS;
+    } catch (CommitFailedException | CommitStateUnknownException e) {
+      throw e;
+    } catch (Throwable e) {
+      LOG.error("Exception thrown on commit: ", e);
+      if (e instanceof AlreadyExistsException) {
+        throw e;
+      }
+      commitStatus =
+          BaseMetastoreOperations.CommitStatus.valueOf(
+              checkCommitStatus(newMetadataLocation, metadata).name());
+      if (commitStatus == BaseMetastoreOperations.CommitStatus.FAILURE) {
+        throw new CommitFailedException(e, "Failed to commit");
+      }
+      if (commitStatus == BaseMetastoreOperations.CommitStatus.UNKNOWN) {
+        throw new CommitStateUnknownException(e);
+      }
+    } finally {
+      try {
+        if (commitStatus == BaseMetastoreOperations.CommitStatus.FAILURE) {
+          LOG.warn("Failed to commit updates to table {}", tableName());
+          io().deleteFile(newMetadataLocation);
+        }
+      } catch (RuntimeException e) {
+        LOG.error(
+            "Failed to cleanup metadata file at {} for table {}",
+            newMetadataLocation,
+            tableName(),
+            e);
+      }
+    }
+  }
+
+  @Override
+  public String tableName() {
+    return String.format("%s.%s", tableReference.getDatasetId(), tableReference.getTableId());
+  }
+
+  @Override
+  public FileIO io() {
+    return fileIO;
+  }
+
+  private void createTable(String newMetadataLocation, TableMetadata metadata) {
+    LOG.debug("Creating a new Iceberg table: {}", tableName());
+    Table tableBuilder = makeNewTable(metadata, newMetadataLocation);
+    tableBuilder.setTableReference(this.tableReference);
+    addConnectionIfProvided(tableBuilder, metadata.properties());
+
+    client.create(tableBuilder);
+  }
+
+  private void addConnectionIfProvided(Table tableBuilder, Map<String, String> metadataProperties) {
+    if (metadataProperties.containsKey(TABLE_PROPERTIES_BQ_CONNECTION)) {
+      tableBuilder
+          .getExternalCatalogTableOptions()
+          .setConnectionId(metadataProperties.get(TABLE_PROPERTIES_BQ_CONNECTION));
+    }
+  }
+
+  /** Update table properties with concurrent update detection using etag. */
+  private void updateTable(
+      String oldMetadataLocation, String newMetadataLocation, TableMetadata metadata) {
+    Table table = client.load(this.tableReference);
+    if (table.getEtag().isEmpty()) {
+      throw new ValidationException(
+          "Etag of legacy table %s is empty, manually update the table via the BigQuery API or"
+              + " recreate and retry",
+          tableName());
+    }
+    ExternalCatalogTableOptions options = table.getExternalCatalogTableOptions();
+    addConnectionIfProvided(table, metadata.properties());
+
+    // If `metadataLocationFromMetastore` is different from metadata location of base, it means
+    // someone has updated metadata location in metastore, which is a conflict update.
+    String metadataLocationFromMetastore =
+        options.getParameters().getOrDefault(METADATA_LOCATION_PROP, "");
+    if (!metadataLocationFromMetastore.isEmpty()
+        && !metadataLocationFromMetastore.equals(oldMetadataLocation)) {
+      throw new CommitFailedException(
+          "Cannot commit base metadata location '%s' is not same as the current table metadata location '%s' for"
+              + " %s.%s",
+          oldMetadataLocation,
+          metadataLocationFromMetastore,
+          tableReference.getDatasetId(),
+          tableReference.getTableId());
+    }
+
+    options.setParameters(buildTableParameters(newMetadataLocation, metadata));
+    try {
+      client.update(tableReference, table);
+    } catch (ValidationException e) {
+      if (e.getMessage().toLowerCase(Locale.ENGLISH).contains("etag mismatch")) {
+        throw new CommitFailedException(
+            "Updating table failed due to conflict updates (etag mismatch). Retry the update");
+      }
+
+      throw e;
+    }
+  }
+
+  // To make the table queryable from Hive, the user would likely be setting the HIVE_ENGINE_ENABLED
+  // parameter.
+  //
+  // TODO: We need to make a decision on how to make the table queryable from Hive.
+  // (could be a server side change or a client side change - that's TBD).
+  private Table makeNewTable(TableMetadata metadata, String metadataFileLocation) {
+    return new Table()
+        .setExternalCatalogTableOptions(
+            BigQueryMetastoreUtils.createExternalCatalogTableOptions(
+                metadata.location(),
+                buildTableParameters(metadataFileLocation, metadata),
+                hiveEngineEnabled(metadata)));

Review Comment:
   Thank you for comment. I removed hive support from the catalog.



-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org

For queries about this service, please contact Infrastructure at:
users@infra.apache.org


---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For additional commands, e-mail: issues-help@iceberg.apache.org


Re: [PR] Catalog: Add BigQuery Metastore Catalog Support [iceberg]

Posted by "talatuyarer (via GitHub)" <gi...@apache.org>.
talatuyarer commented on code in PR #12808:
URL: https://github.com/apache/iceberg/pull/12808#discussion_r2081048008


##########
build.gradle:
##########
@@ -657,6 +657,46 @@ project(':iceberg-delta-lake') {
   }
 }
 
+project(':iceberg-bigquery') {
+  test {
+    useJUnitPlatform()
+  }
+
+  dependencies {
+    api project(':iceberg-api')
+    implementation project(':iceberg-common')
+    implementation project(':iceberg-core')
+
+    implementation project(path: ':iceberg-bundled-guava', configuration: 'shadow')
+
+    implementation platform(libs.google.libraries.bom)
+    compileOnly "com.google.cloud:google-cloud-storage"
+    implementation "com.google.cloud:google-cloud-bigquery"
+    implementation "com.google.cloud:google-cloud-core"
+
+    compileOnly libs.hadoop3.common

Review Comment:
   Trino is important. After mergining this. I would like to give a support Trino too. I removed all hadoop and hive dependency except test. Is that OK @ebyhr ?



-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org

For queries about this service, please contact Infrastructure at:
users@infra.apache.org


---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For additional commands, e-mail: issues-help@iceberg.apache.org


Re: [PR] Catalog: Add BigQuery Metastore Catalog Support [iceberg]

Posted by "talatuyarer (via GitHub)" <gi...@apache.org>.
talatuyarer commented on code in PR #12808:
URL: https://github.com/apache/iceberg/pull/12808#discussion_r2074296337


##########
bigquery/src/main/java/org/apache/iceberg/gcp/bigquery/BigQueryMetastoreCatalog.java:
##########
@@ -0,0 +1,366 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.iceberg.gcp.bigquery;
+
+import com.google.api.services.bigquery.model.Dataset;
+import com.google.api.services.bigquery.model.DatasetList.Datasets;
+import com.google.api.services.bigquery.model.DatasetReference;
+import com.google.api.services.bigquery.model.ExternalCatalogDatasetOptions;
+import com.google.api.services.bigquery.model.TableReference;
+import com.google.cloud.ServiceOptions;
+import com.google.cloud.bigquery.BigQueryOptions;
+import java.io.IOException;
+import java.io.UncheckedIOException;
+import java.security.GeneralSecurityException;
+import java.util.List;
+import java.util.Locale;
+import java.util.Map;
+import java.util.Objects;
+import java.util.Set;
+import org.apache.hadoop.conf.Configurable;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.iceberg.BaseMetastoreCatalog;
+import org.apache.iceberg.CatalogProperties;
+import org.apache.iceberg.CatalogUtil;
+import org.apache.iceberg.TableMetadata;
+import org.apache.iceberg.TableOperations;
+import org.apache.iceberg.catalog.Namespace;
+import org.apache.iceberg.catalog.SupportsNamespaces;
+import org.apache.iceberg.catalog.TableIdentifier;
+import org.apache.iceberg.exceptions.NoSuchNamespaceException;
+import org.apache.iceberg.exceptions.NoSuchTableException;
+import org.apache.iceberg.io.FileIO;
+import org.apache.iceberg.relocated.com.google.common.annotations.VisibleForTesting;
+import org.apache.iceberg.relocated.com.google.common.base.Preconditions;
+import org.apache.iceberg.relocated.com.google.common.base.Strings;
+import org.apache.iceberg.relocated.com.google.common.collect.ImmutableList;
+import org.apache.iceberg.relocated.com.google.common.collect.ImmutableMap;
+import org.apache.iceberg.relocated.com.google.common.collect.Maps;
+import org.apache.iceberg.util.LocationUtil;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/** Iceberg Bigquery Metastore Catalog implementation. */
+public class BigQueryMetastoreCatalog extends BaseMetastoreCatalog
+    implements SupportsNamespaces, Configurable {
+
+  // User provided properties.
+  public static final String PROJECT_ID = "gcp.bigquery.project-id";
+  public static final String GCP_LOCATION = "gcp.bigquery.location";
+  public static final String LIST_ALL_TABLES = "gcp.bigquery.list-all-tables";
+
+  private static final Logger LOG = LoggerFactory.getLogger(BigQueryMetastoreCatalog.class);
+
+  private static final String DEFAULT_GCP_LOCATION = "us";
+
+  private String catalogName;
+  private Map<String, String> catalogProperties;
+  private FileIO fileIO;
+  private Configuration conf;
+  private String projectId;
+  private String projectLocation;
+  private BigQueryMetastoreClient client;
+  private boolean listAllTables;
+  private String warehouseLocation;
+
+  public BigQueryMetastoreCatalog() {}
+
+  @Override
+  public void initialize(String name, Map<String, String> properties) {
+    Preconditions.checkArgument(
+        properties.containsKey(PROJECT_ID),
+        "Invalid GCP project: %s must be specified",
+        PROJECT_ID);
+
+    this.projectId = properties.get(PROJECT_ID);
+    this.projectLocation = properties.getOrDefault(GCP_LOCATION, DEFAULT_GCP_LOCATION);
+
+    BigQueryOptions options =
+        BigQueryOptions.newBuilder()
+            .setProjectId(projectId)
+            .setLocation(projectLocation)
+            .setRetrySettings(ServiceOptions.getDefaultRetrySettings())
+            .build();
+
+    try {
+      client = new BigQueryMetastoreClientImpl(options);
+    } catch (IOException e) {
+      throw new UncheckedIOException("Creating BigQuery client failed", e);
+    } catch (GeneralSecurityException e) {
+      throw new RuntimeException("Creating BigQuery client failed due to a security issue", e);
+    }
+
+    initialize(name, properties, projectId, projectLocation, client);
+  }
+
+  @VisibleForTesting
+  void initialize(
+      String name,
+      Map<String, String> properties,
+      String initialProjectId,
+      String initialLocation,
+      BigQueryMetastoreClient bigQueryMetaStoreClient) {
+    Preconditions.checkArgument(bigQueryMetaStoreClient != null, "Invalid BigQuery client: null");
+    this.catalogName = name;
+    this.catalogProperties = ImmutableMap.copyOf(properties);
+    this.projectId = initialProjectId;
+    this.projectLocation = initialLocation;
+    this.client = bigQueryMetaStoreClient;
+
+    if (this.conf == null) {
+      LOG.warn("No configuration was set, using the default environment Configuration");
+      this.conf = new Configuration();
+    }
+
+    LOG.info("Using BigQuery Metastore Iceberg Catalog: {}", name);
+
+    if (properties.containsKey(CatalogProperties.WAREHOUSE_LOCATION)) {
+      this.warehouseLocation =
+          LocationUtil.stripTrailingSlash(properties.get(CatalogProperties.WAREHOUSE_LOCATION));
+    }
+
+    this.fileIO =
+        CatalogUtil.loadFileIO(
+            properties.getOrDefault(
+                CatalogProperties.FILE_IO_IMPL, "org.apache.iceberg.io.ResolvingFileIO"),
+            properties,
+            conf);
+
+    this.listAllTables = Boolean.parseBoolean(properties.getOrDefault(LIST_ALL_TABLES, "true"));
+  }
+
+  @Override
+  protected TableOperations newTableOps(TableIdentifier identifier) {
+    return new BigQueryTableOperations(client, fileIO, toTableReference(identifier), conf);
+  }
+
+  @Override
+  protected String defaultWarehouseLocation(TableIdentifier identifier) {
+    String locationUri = null;
+    DatasetReference datasetReference = toDatasetReference(identifier.namespace());
+    Dataset dataset = client.load(datasetReference);
+    if (dataset != null && dataset.getExternalCatalogDatasetOptions() != null) {
+      locationUri = dataset.getExternalCatalogDatasetOptions().getDefaultStorageLocationUri();
+    }
+
+    return String.format(
+        "%s/%s",
+        Strings.isNullOrEmpty(locationUri)
+            ? createDefaultStorageLocationUri(datasetReference.getDatasetId())
+            : LocationUtil.stripTrailingSlash(locationUri),
+        identifier.name());
+  }
+
+  @Override
+  public List<TableIdentifier> listTables(Namespace namespace) {
+    validateNamespace(namespace);
+
+    return client.list(toDatasetReference(namespace), listAllTables).stream()
+        .map(
+            table -> TableIdentifier.of(namespace.level(0), table.getTableReference().getTableId()))
+        .collect(ImmutableList.toImmutableList());
+  }
+
+  @Override
+  public boolean dropTable(TableIdentifier identifier, boolean purge) {
+    try {
+      TableOperations ops = newTableOps(identifier);
+      TableMetadata lastMetadata = ops.current();
+
+      client.delete(toTableReference(identifier));
+
+      if (purge && lastMetadata != null) {
+        CatalogUtil.dropTableData(ops.io(), lastMetadata);
+      }
+    } catch (NoSuchTableException e) {
+      return false;
+    }
+
+    return true;
+  }
+
+  @Override
+  public void renameTable(TableIdentifier from, TableIdentifier to) {
+    // TODO: Enable once supported by BigQuery API.
+    throw new UnsupportedOperationException("Table rename operation is unsupported.");
+  }
+
+  @Override
+  public void createNamespace(Namespace namespace, Map<String, String> metadata) {
+    Dataset builder = new Dataset();
+    DatasetReference datasetReference = toDatasetReference(namespace);
+    builder.setLocation(this.projectLocation);
+    builder.setDatasetReference(datasetReference);
+    builder.setExternalCatalogDatasetOptions(
+        BigQueryMetastoreUtils.createExternalCatalogDatasetOptions(
+            createDefaultStorageLocationUri(datasetReference.getDatasetId()), metadata));
+
+    client.create(builder);
+  }
+
+  @Override
+  public List<Namespace> listNamespaces() {
+    try {
+      return listNamespaces(Namespace.empty());
+    } catch (NoSuchNamespaceException e) {
+      return ImmutableList.of();
+    }
+  }
+
+  /**
+   * Since this catalog only supports one-level namespaces, it always returns an empty list unless
+   * passed an empty namespace to list all namespaces within the catalog.
+   */
+  @Override
+  public List<Namespace> listNamespaces(Namespace namespace) {
+    List<Datasets> allDatasets = client.list(projectId);
+
+    ImmutableList<Namespace> namespaces =

Review Comment:
   I believe I addressed that concern by adding namespace is not empty check below



-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org

For queries about this service, please contact Infrastructure at:
users@infra.apache.org


---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For additional commands, e-mail: issues-help@iceberg.apache.org


Re: [PR] Catalog: Add BigQuery Metastore Catalog Support [iceberg]

Posted by "ebyhr (via GitHub)" <gi...@apache.org>.
ebyhr commented on code in PR #12808:
URL: https://github.com/apache/iceberg/pull/12808#discussion_r2074830051


##########
build.gradle:
##########
@@ -657,6 +657,46 @@ project(':iceberg-delta-lake') {
   }
 }
 
+project(':iceberg-bigquery') {
+  test {
+    useJUnitPlatform()
+  }
+
+  dependencies {
+    api project(':iceberg-api')
+    implementation project(':iceberg-common')
+    implementation project(':iceberg-core')
+
+    implementation project(path: ':iceberg-bundled-guava', configuration: 'shadow')
+
+    implementation platform(libs.google.libraries.bom)
+    compileOnly "com.google.cloud:google-cloud-storage"
+    implementation "com.google.cloud:google-cloud-bigquery"
+    implementation "com.google.cloud:google-cloud-core"
+
+    compileOnly libs.hadoop3.common

Review Comment:
   This Hadoop dependency will probably prevent Trino from adopting BigQuery metastore catalog since we banned those libraries in https://github.com/trinodb/trino/issues/15921. 



-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org

For queries about this service, please contact Infrastructure at:
users@infra.apache.org


---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For additional commands, e-mail: issues-help@iceberg.apache.org


Re: [PR] Catalog: Add BigQuery Metastore Catalog Support [iceberg]

Posted by "danielcweeks (via GitHub)" <gi...@apache.org>.
danielcweeks commented on code in PR #12808:
URL: https://github.com/apache/iceberg/pull/12808#discussion_r2072144358


##########
bigquery/src/main/java/org/apache/iceberg/gcp/bigquery/BigQueryMetastoreCatalog.java:
##########
@@ -0,0 +1,366 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.iceberg.gcp.bigquery;
+
+import com.google.api.services.bigquery.model.Dataset;
+import com.google.api.services.bigquery.model.DatasetList.Datasets;
+import com.google.api.services.bigquery.model.DatasetReference;
+import com.google.api.services.bigquery.model.ExternalCatalogDatasetOptions;
+import com.google.api.services.bigquery.model.TableReference;
+import com.google.cloud.ServiceOptions;
+import com.google.cloud.bigquery.BigQueryOptions;
+import java.io.IOException;
+import java.io.UncheckedIOException;
+import java.security.GeneralSecurityException;
+import java.util.List;
+import java.util.Locale;
+import java.util.Map;
+import java.util.Objects;
+import java.util.Set;
+import org.apache.hadoop.conf.Configurable;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.iceberg.BaseMetastoreCatalog;
+import org.apache.iceberg.CatalogProperties;
+import org.apache.iceberg.CatalogUtil;
+import org.apache.iceberg.TableMetadata;
+import org.apache.iceberg.TableOperations;
+import org.apache.iceberg.catalog.Namespace;
+import org.apache.iceberg.catalog.SupportsNamespaces;
+import org.apache.iceberg.catalog.TableIdentifier;
+import org.apache.iceberg.exceptions.NoSuchNamespaceException;
+import org.apache.iceberg.exceptions.NoSuchTableException;
+import org.apache.iceberg.io.FileIO;
+import org.apache.iceberg.relocated.com.google.common.annotations.VisibleForTesting;
+import org.apache.iceberg.relocated.com.google.common.base.Preconditions;
+import org.apache.iceberg.relocated.com.google.common.base.Strings;
+import org.apache.iceberg.relocated.com.google.common.collect.ImmutableList;
+import org.apache.iceberg.relocated.com.google.common.collect.ImmutableMap;
+import org.apache.iceberg.relocated.com.google.common.collect.Maps;
+import org.apache.iceberg.util.LocationUtil;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/** Iceberg Bigquery Metastore Catalog implementation. */
+public class BigQueryMetastoreCatalog extends BaseMetastoreCatalog
+    implements SupportsNamespaces, Configurable {
+
+  // User provided properties.
+  public static final String PROJECT_ID = "gcp.bigquery.project-id";
+  public static final String GCP_LOCATION = "gcp.bigquery.location";
+  public static final String LIST_ALL_TABLES = "gcp.bigquery.list-all-tables";
+
+  private static final Logger LOG = LoggerFactory.getLogger(BigQueryMetastoreCatalog.class);
+
+  private static final String DEFAULT_GCP_LOCATION = "us";
+
+  private String catalogName;
+  private Map<String, String> catalogProperties;
+  private FileIO fileIO;
+  private Configuration conf;
+  private String projectId;
+  private String projectLocation;
+  private BigQueryMetastoreClient client;
+  private boolean listAllTables;
+  private String warehouseLocation;
+
+  public BigQueryMetastoreCatalog() {}
+
+  @Override
+  public void initialize(String name, Map<String, String> properties) {
+    Preconditions.checkArgument(
+        properties.containsKey(PROJECT_ID),
+        "Invalid GCP project: %s must be specified",
+        PROJECT_ID);
+
+    this.projectId = properties.get(PROJECT_ID);
+    this.projectLocation = properties.getOrDefault(GCP_LOCATION, DEFAULT_GCP_LOCATION);
+
+    BigQueryOptions options =
+        BigQueryOptions.newBuilder()
+            .setProjectId(projectId)
+            .setLocation(projectLocation)
+            .setRetrySettings(ServiceOptions.getDefaultRetrySettings())
+            .build();
+
+    try {
+      client = new BigQueryMetastoreClientImpl(options);
+    } catch (IOException e) {
+      throw new UncheckedIOException("Creating BigQuery client failed", e);
+    } catch (GeneralSecurityException e) {
+      throw new RuntimeException("Creating BigQuery client failed due to a security issue", e);
+    }
+
+    initialize(name, properties, projectId, projectLocation, client);
+  }
+
+  @VisibleForTesting
+  void initialize(
+      String name,
+      Map<String, String> properties,
+      String initialProjectId,
+      String initialLocation,
+      BigQueryMetastoreClient bigQueryMetaStoreClient) {
+    Preconditions.checkArgument(bigQueryMetaStoreClient != null, "Invalid BigQuery client: null");
+    this.catalogName = name;
+    this.catalogProperties = ImmutableMap.copyOf(properties);
+    this.projectId = initialProjectId;
+    this.projectLocation = initialLocation;
+    this.client = bigQueryMetaStoreClient;
+
+    if (this.conf == null) {
+      LOG.warn("No configuration was set, using the default environment Configuration");
+      this.conf = new Configuration();
+    }
+
+    LOG.info("Using BigQuery Metastore Iceberg Catalog: {}", name);
+
+    if (properties.containsKey(CatalogProperties.WAREHOUSE_LOCATION)) {
+      this.warehouseLocation =
+          LocationUtil.stripTrailingSlash(properties.get(CatalogProperties.WAREHOUSE_LOCATION));
+    }
+
+    this.fileIO =
+        CatalogUtil.loadFileIO(
+            properties.getOrDefault(
+                CatalogProperties.FILE_IO_IMPL, "org.apache.iceberg.io.ResolvingFileIO"),
+            properties,
+            conf);
+
+    this.listAllTables = Boolean.parseBoolean(properties.getOrDefault(LIST_ALL_TABLES, "true"));
+  }
+
+  @Override
+  protected TableOperations newTableOps(TableIdentifier identifier) {
+    return new BigQueryTableOperations(client, fileIO, toTableReference(identifier), conf);
+  }
+
+  @Override
+  protected String defaultWarehouseLocation(TableIdentifier identifier) {
+    String locationUri = null;
+    DatasetReference datasetReference = toDatasetReference(identifier.namespace());
+    Dataset dataset = client.load(datasetReference);
+    if (dataset != null && dataset.getExternalCatalogDatasetOptions() != null) {
+      locationUri = dataset.getExternalCatalogDatasetOptions().getDefaultStorageLocationUri();
+    }
+
+    return String.format(
+        "%s/%s",
+        Strings.isNullOrEmpty(locationUri)
+            ? createDefaultStorageLocationUri(datasetReference.getDatasetId())
+            : LocationUtil.stripTrailingSlash(locationUri),
+        identifier.name());
+  }
+
+  @Override
+  public List<TableIdentifier> listTables(Namespace namespace) {
+    validateNamespace(namespace);
+
+    return client.list(toDatasetReference(namespace), listAllTables).stream()
+        .map(
+            table -> TableIdentifier.of(namespace.level(0), table.getTableReference().getTableId()))
+        .collect(ImmutableList.toImmutableList());
+  }
+
+  @Override
+  public boolean dropTable(TableIdentifier identifier, boolean purge) {
+    try {
+      TableOperations ops = newTableOps(identifier);
+      TableMetadata lastMetadata = ops.current();
+
+      client.delete(toTableReference(identifier));
+
+      if (purge && lastMetadata != null) {
+        CatalogUtil.dropTableData(ops.io(), lastMetadata);
+      }
+    } catch (NoSuchTableException e) {
+      return false;
+    }
+
+    return true;
+  }
+
+  @Override
+  public void renameTable(TableIdentifier from, TableIdentifier to) {
+    // TODO: Enable once supported by BigQuery API.
+    throw new UnsupportedOperationException("Table rename operation is unsupported.");
+  }
+
+  @Override
+  public void createNamespace(Namespace namespace, Map<String, String> metadata) {
+    Dataset builder = new Dataset();
+    DatasetReference datasetReference = toDatasetReference(namespace);
+    builder.setLocation(this.projectLocation);
+    builder.setDatasetReference(datasetReference);
+    builder.setExternalCatalogDatasetOptions(
+        BigQueryMetastoreUtils.createExternalCatalogDatasetOptions(
+            createDefaultStorageLocationUri(datasetReference.getDatasetId()), metadata));
+
+    client.create(builder);
+  }
+
+  @Override
+  public List<Namespace> listNamespaces() {
+    try {
+      return listNamespaces(Namespace.empty());
+    } catch (NoSuchNamespaceException e) {
+      return ImmutableList.of();
+    }
+  }
+
+  /**
+   * Since this catalog only supports one-level namespaces, it always returns an empty list unless
+   * passed an empty namespace to list all namespaces within the catalog.
+   */
+  @Override
+  public List<Namespace> listNamespaces(Namespace namespace) {
+    List<Datasets> allDatasets = client.list(projectId);
+
+    ImmutableList<Namespace> namespaces =
+        allDatasets.stream().map(this::toNamespace).collect(ImmutableList.toImmutableList());
+    if (namespaces.isEmpty()) {
+      throw new NoSuchNamespaceException("Namespace does not exist: %s", namespace);
+    }
+
+    return namespaces;
+  }
+
+  @Override
+  public boolean dropNamespace(Namespace namespace) {
+    try {
+      client.delete(toDatasetReference(namespace));
+      // We don't delete the data folder for safety, which aligns with Hive Metastore's default
+      // behavior.
+      // We can support database or catalog level config controlling file deletion in the future.
+      return true;
+    } catch (NoSuchNamespaceException e) {
+      return false;
+    }
+  }
+
+  @Override
+  public boolean setProperties(Namespace namespace, Map<String, String> properties) {
+    Dataset dataset = client.load(toDatasetReference(namespace));
+
+    ExternalCatalogDatasetOptions existingOptions = dataset.getExternalCatalogDatasetOptions();
+    Map<String, String> existingParameters =
+        existingOptions != null ? existingOptions.getParameters() : null;
+
+    Map<String, String> newParameters = Maps.newHashMap();
+    if (existingParameters != null) {
+      newParameters.putAll(existingParameters);
+    }
+
+    newParameters.putAll(properties);
+
+    if (Objects.equals(existingParameters, newParameters)) {
+      // No change in parameters detected
+      return false;
+    }
+
+    client.setParameters(toDatasetReference(namespace), properties);

Review Comment:
   Wait, it appears that we're reloading and apply the changes again in `client.setProperties`? do we need to do that?



-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org

For queries about this service, please contact Infrastructure at:
users@infra.apache.org


---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For additional commands, e-mail: issues-help@iceberg.apache.org


Re: [PR] Catalog: Add BigQuery Metastore Catalog Support [iceberg]

Posted by "danielcweeks (via GitHub)" <gi...@apache.org>.
danielcweeks commented on code in PR #12808:
URL: https://github.com/apache/iceberg/pull/12808#discussion_r2072198456


##########
bigquery/src/main/java/org/apache/iceberg/gcp/bigquery/BigQueryTableOperations.java:
##########
@@ -0,0 +1,296 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.iceberg.gcp.bigquery;
+
+import com.google.api.services.bigquery.model.ExternalCatalogTableOptions;
+import com.google.api.services.bigquery.model.Table;
+import com.google.api.services.bigquery.model.TableReference;
+import java.util.Locale;
+import java.util.Map;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hive.common.StatsSetupConst;
+import org.apache.iceberg.BaseMetastoreOperations;
+import org.apache.iceberg.BaseMetastoreTableOperations;
+import org.apache.iceberg.SnapshotSummary;
+import org.apache.iceberg.TableMetadata;
+import org.apache.iceberg.TableProperties;
+import org.apache.iceberg.exceptions.AlreadyExistsException;
+import org.apache.iceberg.exceptions.CommitFailedException;
+import org.apache.iceberg.exceptions.CommitStateUnknownException;
+import org.apache.iceberg.exceptions.NoSuchTableException;
+import org.apache.iceberg.exceptions.ValidationException;
+import org.apache.iceberg.hadoop.ConfigProperties;
+import org.apache.iceberg.io.FileIO;
+import org.apache.iceberg.relocated.com.google.common.collect.Maps;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/** Handles BigQuery metastore table operations. */
+public final class BigQueryTableOperations extends BaseMetastoreTableOperations {
+
+  private static final Logger LOG = LoggerFactory.getLogger(BigQueryTableOperations.class);
+
+  private static final String TABLE_PROPERTIES_BQ_CONNECTION = "bq_connection";
+
+  private final BigQueryMetastoreClient client;
+  private final FileIO fileIO;
+  private final TableReference tableReference;
+  private final Configuration conf;
+
+  BigQueryTableOperations(
+      BigQueryMetastoreClient client,
+      FileIO fileIO,
+      TableReference tableReference,
+      Configuration conf) {
+    this.client = client;
+    this.fileIO = fileIO;
+    this.tableReference = tableReference;
+    this.conf = conf;
+  }
+
+  // The doRefresh method should provide implementation on how to get the metadata location.
+  @Override
+  public void doRefresh() {
+    // Must default to null.
+    String metadataLocation = null;
+    try {
+      metadataLocation =
+          loadMetadataLocationOrThrow(
+              client.load(this.tableReference).getExternalCatalogTableOptions());

Review Comment:
   remote `this` reference



-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org

For queries about this service, please contact Infrastructure at:
users@infra.apache.org


---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For additional commands, e-mail: issues-help@iceberg.apache.org


Re: [PR] Catalog: Add BigQuery Metastore Catalog Support [iceberg]

Posted by "nastra (via GitHub)" <gi...@apache.org>.
nastra commented on code in PR #12808:
URL: https://github.com/apache/iceberg/pull/12808#discussion_r2046286635


##########
.github/labeler.yml:
##########
@@ -148,7 +148,9 @@ GCP:
   - changed-files:
     - any-glob-to-any-file: [
       'gcp/**/*',
-      'gcp-bundle/**/*'
+      'gcp-bundle/**/*',
+      'bigquery/**/*',
+      'bigquery-bundle/**/*'

Review Comment:
   there isn't a bigquery-bundle, so I believe this can be removed



##########
build.gradle:
##########
@@ -657,6 +657,49 @@ project(':iceberg-delta-lake') {
   }
 }
 
+project(':iceberg-bigquery') {
+  test {
+    useJUnitPlatform()
+  }
+
+  dependencies {
+    implementation project(path: ':iceberg-bundled-guava', configuration: 'shadow')
+    implementation project(':iceberg-gcp')
+
+    implementation platform(libs.google.libraries.bom)
+    compileOnly "com.google.cloud:google-cloud-storage"
+    implementation "com.google.cloud:google-cloud-bigquery"
+    implementation "com.google.cloud:google-cloud-core"
+
+    api project(':iceberg-api')
+    implementation project(':iceberg-common')
+    implementation project(':iceberg-core')
+
+    compileOnly("org.apache.hadoop:hadoop-common:3.4.1")

Review Comment:
   this should be referencing the version defined in `libs.versions.toml` via `libs.versions.hadoop3.common`



##########
build.gradle:
##########
@@ -657,6 +657,49 @@ project(':iceberg-delta-lake') {
   }
 }
 
+project(':iceberg-bigquery') {
+  test {
+    useJUnitPlatform()
+  }
+
+  dependencies {
+    implementation project(path: ':iceberg-bundled-guava', configuration: 'shadow')
+    implementation project(':iceberg-gcp')
+
+    implementation platform(libs.google.libraries.bom)
+    compileOnly "com.google.cloud:google-cloud-storage"
+    implementation "com.google.cloud:google-cloud-bigquery"
+    implementation "com.google.cloud:google-cloud-core"
+
+    api project(':iceberg-api')
+    implementation project(':iceberg-common')
+    implementation project(':iceberg-core')
+
+    compileOnly("org.apache.hadoop:hadoop-common:3.4.1")
+    compileOnly("org.apache.hive:hive-metastore:4.0.1") {
+      exclude group: 'org.apache.avro', module: 'avro'
+      exclude group: 'org.slf4j', module: 'slf4j-log4j12'
+      exclude group: 'org.pentaho' // missing dependency
+      exclude group: 'org.apache.hbase'
+      exclude group: 'org.apache.logging.log4j'
+      exclude group: 'co.cask.tephra'
+      exclude group: 'com.google.code.findbugs', module: 'jsr305'
+      exclude group: 'org.eclipse.jetty.aggregate', module: 'jetty-all'
+      exclude group: 'org.eclipse.jetty.orbit', module: 'javax.servlet'
+      exclude group: 'org.apache.parquet', module: 'parquet-hadoop-bundle'
+      exclude group: 'com.tdunning', module: 'json'
+      exclude group: 'javax.transaction', module: 'transaction-api'
+      exclude group: 'com.zaxxer', module: 'HikariCP'
+      exclude group: 'com.google.code.gson', module: 'gson'
+    }
+
+    testImplementation project(path: ':iceberg-core', configuration: 'testArtifacts')
+    testImplementation project(path: ':iceberg-api', configuration: 'testArtifacts')
+
+    testImplementation 'org.mockito:mockito-core:5.12.0'

Review Comment:
   this dependency should already be included via https://github.com/apache/iceberg/blob/main/build.gradle#L205, so can be removed here



-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org

For queries about this service, please contact Infrastructure at:
users@infra.apache.org


---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For additional commands, e-mail: issues-help@iceberg.apache.org


Re: [PR] Catalog: Add BigQuery Metastore Catalog Support [iceberg]

Posted by "nastra (via GitHub)" <gi...@apache.org>.
nastra commented on code in PR #12808:
URL: https://github.com/apache/iceberg/pull/12808#discussion_r2066093734


##########
bigquery/src/test/java/org/apache/iceberg/gcp/bigquery/BigQueryTableOperationsTest.java:
##########
@@ -0,0 +1,274 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.iceberg.gcp.bigquery;
+
+import static org.apache.iceberg.gcp.bigquery.BigQueryMetastoreCatalog.PROJECT_ID;
+import static org.apache.iceberg.types.Types.NestedField.required;
+import static org.assertj.core.api.Assertions.assertThat;
+import static org.assertj.core.api.Assertions.assertThatThrownBy;
+import static org.mockito.Mockito.any;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.reset;
+import static org.mockito.Mockito.times;
+import static org.mockito.Mockito.verify;
+import static org.mockito.Mockito.when;
+
+import com.google.api.services.bigquery.model.Dataset;
+import com.google.api.services.bigquery.model.DatasetReference;
+import com.google.api.services.bigquery.model.ExternalCatalogDatasetOptions;
+import com.google.api.services.bigquery.model.ExternalCatalogTableOptions;
+import com.google.api.services.bigquery.model.StorageDescriptor;
+import com.google.api.services.bigquery.model.Table;
+import com.google.api.services.bigquery.model.TableReference;
+import java.io.File;
+import java.io.IOException;
+import java.util.Collections;
+import java.util.Optional;
+import org.apache.commons.io.FileUtils;
+import org.apache.commons.io.filefilter.TrueFileFilter;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.iceberg.CatalogProperties;
+import org.apache.iceberg.PartitionSpec;
+import org.apache.iceberg.Schema;
+import org.apache.iceberg.catalog.TableIdentifier;
+import org.apache.iceberg.exceptions.CommitFailedException;
+import org.apache.iceberg.exceptions.NoSuchTableException;
+import org.apache.iceberg.exceptions.ValidationException;
+import org.apache.iceberg.relocated.com.google.common.collect.ImmutableMap;
+import org.apache.iceberg.types.Types;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.io.TempDir;
+import org.mockito.ArgumentCaptor;
+
+public class BigQueryTableOperationsTest {
+
+  @TempDir private File tempFolder;
+  public static final String METADATA_LOCATION_PROP = "metadata_location";
+  private static final String GCP_PROJECT = "my-project";
+  private static final String GCP_REGION = "us";
+  private static final String DATASET_ID = "db";
+  private static final String TABLE_ID = "tbl";
+  private static final TableIdentifier SPARK_TABLE_ID = TableIdentifier.of(DATASET_ID, TABLE_ID);
+
+  private static final TableReference TABLE_REFERENCE =
+      new TableReference().setProjectId(GCP_PROJECT).setDatasetId(DATASET_ID).setTableId(TABLE_ID);
+
+  private final BigQueryMetastoreClient bigQueryMetaStoreClient =
+      mock(BigQueryMetastoreClient.class);
+
+  private BigQueryMetastoreCatalog bigQueryMetastoreCatalog;
+  private BigQueryTableOperations tableOps;
+
+  @BeforeEach
+  public void before() {
+    this.bigQueryMetastoreCatalog = new BigQueryMetastoreCatalog();
+    this.bigQueryMetastoreCatalog.setConf(new Configuration());
+    String warehouseLocation = tempFolder.toPath().resolve("hive-warehouse").toString();
+
+    bigQueryMetastoreCatalog.initialize(
+        "CATALOG_ID",
+        ImmutableMap.of(
+            PROJECT_ID,
+            GCP_PROJECT,
+            CatalogProperties.WAREHOUSE_LOCATION,
+            warehouseLocation,
+            CatalogProperties.FILE_IO_IMPL,
+            "org.apache.iceberg.hadoop.HadoopFileIO"),
+        GCP_PROJECT,
+        GCP_REGION,
+        bigQueryMetaStoreClient);
+    this.tableOps = (BigQueryTableOperations) bigQueryMetastoreCatalog.newTableOps(SPARK_TABLE_ID);
+  }
+
+  @Test
+  public void testFetchLatestMetadataFromBigQuery() throws Exception {

Review Comment:
   existing tests in the codebase use `test` as a prefix in test methods but for new test classes we typically try to avoid using `test` in the prefix as it's a legacy thing from old JUnit days that isn't needed anymore



-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org

For queries about this service, please contact Infrastructure at:
users@infra.apache.org


---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For additional commands, e-mail: issues-help@iceberg.apache.org


Re: [PR] Catalog: Add BigQuery Metastore Catalog Support [iceberg]

Posted by "nastra (via GitHub)" <gi...@apache.org>.
nastra commented on code in PR #12808:
URL: https://github.com/apache/iceberg/pull/12808#discussion_r2066079076


##########
bigquery/src/test/java/org/apache/iceberg/gcp/bigquery/BigQueryTableOperationsTest.java:
##########
@@ -0,0 +1,274 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.iceberg.gcp.bigquery;
+
+import static org.apache.iceberg.gcp.bigquery.BigQueryMetastoreCatalog.PROJECT_ID;
+import static org.apache.iceberg.types.Types.NestedField.required;
+import static org.assertj.core.api.Assertions.assertThat;
+import static org.assertj.core.api.Assertions.assertThatThrownBy;
+import static org.mockito.Mockito.any;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.reset;
+import static org.mockito.Mockito.times;
+import static org.mockito.Mockito.verify;
+import static org.mockito.Mockito.when;
+
+import com.google.api.services.bigquery.model.Dataset;
+import com.google.api.services.bigquery.model.DatasetReference;
+import com.google.api.services.bigquery.model.ExternalCatalogDatasetOptions;
+import com.google.api.services.bigquery.model.ExternalCatalogTableOptions;
+import com.google.api.services.bigquery.model.StorageDescriptor;
+import com.google.api.services.bigquery.model.Table;
+import com.google.api.services.bigquery.model.TableReference;
+import java.io.File;
+import java.io.IOException;
+import java.util.Collections;
+import java.util.Optional;
+import org.apache.commons.io.FileUtils;
+import org.apache.commons.io.filefilter.TrueFileFilter;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.iceberg.CatalogProperties;
+import org.apache.iceberg.PartitionSpec;
+import org.apache.iceberg.Schema;
+import org.apache.iceberg.catalog.TableIdentifier;
+import org.apache.iceberg.exceptions.CommitFailedException;
+import org.apache.iceberg.exceptions.NoSuchTableException;
+import org.apache.iceberg.exceptions.ValidationException;
+import org.apache.iceberg.relocated.com.google.common.collect.ImmutableMap;
+import org.apache.iceberg.types.Types;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.io.TempDir;
+import org.mockito.ArgumentCaptor;
+
+public class BigQueryTableOperationsTest {
+
+  @TempDir private File tempFolder;
+  public static final String METADATA_LOCATION_PROP = "metadata_location";
+  private static final String GCP_PROJECT = "my-project";
+  private static final String GCP_REGION = "us";
+  private static final String DATASET_ID = "db";
+  private static final String TABLE_ID = "tbl";
+  private static final TableIdentifier SPARK_TABLE_ID = TableIdentifier.of(DATASET_ID, TABLE_ID);
+
+  private static final TableReference TABLE_REFERENCE =
+      new TableReference().setProjectId(GCP_PROJECT).setDatasetId(DATASET_ID).setTableId(TABLE_ID);
+
+  private final BigQueryMetastoreClient bigQueryMetaStoreClient =
+      mock(BigQueryMetastoreClient.class);
+
+  private BigQueryMetastoreCatalog bigQueryMetastoreCatalog;

Review Comment:
   ```suggestion
     private BigQueryMetastoreCatalog catalog;
   ```



-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org

For queries about this service, please contact Infrastructure at:
users@infra.apache.org


---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For additional commands, e-mail: issues-help@iceberg.apache.org


Re: [PR] Catalog: Add BigQuery Metastore Catalog Support [iceberg]

Posted by "danielcweeks (via GitHub)" <gi...@apache.org>.
danielcweeks commented on code in PR #12808:
URL: https://github.com/apache/iceberg/pull/12808#discussion_r2067325844


##########
bigquery/src/main/java/org/apache/iceberg/gcp/bigquery/BigQueryMetastoreCatalog.java:
##########
@@ -0,0 +1,405 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.iceberg.gcp.bigquery;
+
+import com.google.api.services.bigquery.model.Dataset;
+import com.google.api.services.bigquery.model.DatasetList.Datasets;
+import com.google.api.services.bigquery.model.DatasetReference;
+import com.google.api.services.bigquery.model.ExternalCatalogDatasetOptions;
+import com.google.api.services.bigquery.model.TableReference;
+import com.google.cloud.ServiceOptions;
+import com.google.cloud.bigquery.BigQueryOptions;
+import java.io.IOException;
+import java.security.GeneralSecurityException;
+import java.util.List;
+import java.util.Locale;
+import java.util.Map;
+import java.util.Objects;
+import java.util.Set;
+import org.apache.hadoop.conf.Configurable;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.iceberg.BaseMetastoreCatalog;
+import org.apache.iceberg.CatalogProperties;
+import org.apache.iceberg.CatalogUtil;
+import org.apache.iceberg.TableMetadata;
+import org.apache.iceberg.TableOperations;
+import org.apache.iceberg.catalog.Namespace;
+import org.apache.iceberg.catalog.SupportsNamespaces;
+import org.apache.iceberg.catalog.TableIdentifier;
+import org.apache.iceberg.exceptions.AlreadyExistsException;
+import org.apache.iceberg.exceptions.NoSuchNamespaceException;
+import org.apache.iceberg.exceptions.NoSuchTableException;
+import org.apache.iceberg.exceptions.ServiceFailureException;
+import org.apache.iceberg.exceptions.ValidationException;
+import org.apache.iceberg.io.FileIO;
+import org.apache.iceberg.relocated.com.google.common.annotations.VisibleForTesting;
+import org.apache.iceberg.relocated.com.google.common.base.Preconditions;
+import org.apache.iceberg.relocated.com.google.common.base.Strings;
+import org.apache.iceberg.relocated.com.google.common.collect.ImmutableList;
+import org.apache.iceberg.relocated.com.google.common.collect.ImmutableMap;
+import org.apache.iceberg.relocated.com.google.common.collect.Maps;
+import org.apache.iceberg.util.LocationUtil;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/** Iceberg Bigquery Metastore Catalog implementation. */
+public class BigQueryMetastoreCatalog extends BaseMetastoreCatalog
+    implements SupportsNamespaces, Configurable {
+
+  // User provided properties.
+  public static final String PROJECT_ID = "gcp-project";
+  public static final String GCP_LOCATION = "gcp-location";
+  public static final String LIST_ALL_TABLES = "list-all-tables";
+
+  public static final String HIVE_METASTORE_WAREHOUSE_DIR = "hive.metastore.warehouse.dir";
+
+  private static final Logger LOG = LoggerFactory.getLogger(BigQueryMetastoreCatalog.class);
+
+  private static final String DEFAULT_GCP_LOCATION = "us";
+
+  private String catalogName;
+  private Map<String, String> catalogProperties;
+  private FileIO fileIO;
+  private Configuration conf;
+  private String projectId;
+  private String location;
+  private BigQueryMetastoreClient client;
+  private boolean listAllTables;
+
+  // Must have a no-arg constructor to be dynamically loaded
+  // initialize(String name, Map<String, String> properties) will be called to complete
+  // initialization
+  public BigQueryMetastoreCatalog() {}
+
+  @Override
+  public void initialize(String inputName, Map<String, String> properties) {
+    Preconditions.checkArgument(
+        properties.containsKey(PROJECT_ID),
+        "Invalid GCP project: %s must be specified",
+        PROJECT_ID);
+
+    projectId = properties.get(PROJECT_ID);
+    location = properties.getOrDefault(GCP_LOCATION, DEFAULT_GCP_LOCATION);
+
+    BigQueryOptions options =
+        BigQueryOptions.newBuilder()
+            .setProjectId(projectId)
+            .setLocation(location)
+            .setRetrySettings(ServiceOptions.getDefaultRetrySettings())
+            .build();
+
+    try {
+      client = new BigQueryMetastoreClientImpl(options);
+    } catch (IOException e) {
+      throw new ServiceFailureException(e, "Creating BigQuery client failed");
+    } catch (GeneralSecurityException e) {
+      throw new ValidationException(e, "Creating BigQuery client failed due to a security issue");
+    }
+
+    initialize(inputName, properties, projectId, location, client);
+  }
+
+  @VisibleForTesting
+  void initialize(
+      String inputName,
+      Map<String, String> properties,
+      String initialProjectId,
+      String initialLocation,
+      BigQueryMetastoreClient bigQueryMetaStoreClient) {
+    Preconditions.checkArgument(bigQueryMetaStoreClient != null, "Invalid BigQuery client: null");
+    this.catalogName = inputName;
+    this.catalogProperties = ImmutableMap.copyOf(properties);
+    this.projectId = initialProjectId;
+    this.location = initialLocation;
+    this.client = bigQueryMetaStoreClient;
+
+    if (this.conf == null) {
+      LOG.warn("No configuration was set, using the default environment Configuration");
+      this.conf = new Configuration();
+    }
+
+    LOG.info("Using BigQuery Metastore Iceberg Catalog: {}", inputName);
+
+    if (properties.containsKey(CatalogProperties.WAREHOUSE_LOCATION)) {
+      this.conf.set(
+          HIVE_METASTORE_WAREHOUSE_DIR,
+          LocationUtil.stripTrailingSlash(properties.get(CatalogProperties.WAREHOUSE_LOCATION)));
+    }
+
+    String fileIoImpl =
+        properties.getOrDefault(

Review Comment:
   I think we want to use ResolvingFileIO as opposed to hadoop.



-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org

For queries about this service, please contact Infrastructure at:
users@infra.apache.org


---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For additional commands, e-mail: issues-help@iceberg.apache.org


Re: [PR] Catalog: Add BigQuery Metastore Catalog Support [iceberg]

Posted by "danielcweeks (via GitHub)" <gi...@apache.org>.
danielcweeks commented on code in PR #12808:
URL: https://github.com/apache/iceberg/pull/12808#discussion_r2067530383


##########
bigquery/src/main/java/org/apache/iceberg/gcp/bigquery/BigQueryMetastoreCatalog.java:
##########
@@ -0,0 +1,405 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.iceberg.gcp.bigquery;
+
+import com.google.api.services.bigquery.model.Dataset;
+import com.google.api.services.bigquery.model.DatasetList.Datasets;
+import com.google.api.services.bigquery.model.DatasetReference;
+import com.google.api.services.bigquery.model.ExternalCatalogDatasetOptions;
+import com.google.api.services.bigquery.model.TableReference;
+import com.google.cloud.ServiceOptions;
+import com.google.cloud.bigquery.BigQueryOptions;
+import java.io.IOException;
+import java.security.GeneralSecurityException;
+import java.util.List;
+import java.util.Locale;
+import java.util.Map;
+import java.util.Objects;
+import java.util.Set;
+import org.apache.hadoop.conf.Configurable;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.iceberg.BaseMetastoreCatalog;
+import org.apache.iceberg.CatalogProperties;
+import org.apache.iceberg.CatalogUtil;
+import org.apache.iceberg.TableMetadata;
+import org.apache.iceberg.TableOperations;
+import org.apache.iceberg.catalog.Namespace;
+import org.apache.iceberg.catalog.SupportsNamespaces;
+import org.apache.iceberg.catalog.TableIdentifier;
+import org.apache.iceberg.exceptions.AlreadyExistsException;
+import org.apache.iceberg.exceptions.NoSuchNamespaceException;
+import org.apache.iceberg.exceptions.NoSuchTableException;
+import org.apache.iceberg.exceptions.ServiceFailureException;
+import org.apache.iceberg.exceptions.ValidationException;
+import org.apache.iceberg.io.FileIO;
+import org.apache.iceberg.relocated.com.google.common.annotations.VisibleForTesting;
+import org.apache.iceberg.relocated.com.google.common.base.Preconditions;
+import org.apache.iceberg.relocated.com.google.common.base.Strings;
+import org.apache.iceberg.relocated.com.google.common.collect.ImmutableList;
+import org.apache.iceberg.relocated.com.google.common.collect.ImmutableMap;
+import org.apache.iceberg.relocated.com.google.common.collect.Maps;
+import org.apache.iceberg.util.LocationUtil;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/** Iceberg Bigquery Metastore Catalog implementation. */
+public class BigQueryMetastoreCatalog extends BaseMetastoreCatalog
+    implements SupportsNamespaces, Configurable {
+
+  // User provided properties.
+  public static final String PROJECT_ID = "gcp-project";
+  public static final String GCP_LOCATION = "gcp-location";
+  public static final String LIST_ALL_TABLES = "list-all-tables";
+
+  public static final String HIVE_METASTORE_WAREHOUSE_DIR = "hive.metastore.warehouse.dir";

Review Comment:
   I would agree that we `CatalogProperties.WAREHOUSE_LOCATION` seems a better fit.  I don't feel like we want make any associations with Hive (properties/configs/otherwise) since this is intentionally a BQ specific catalog.



-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org

For queries about this service, please contact Infrastructure at:
users@infra.apache.org


---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For additional commands, e-mail: issues-help@iceberg.apache.org


Re: [PR] Catalog: Add BigQuery Metastore Catalog Support [iceberg]

Posted by "talatuyarer (via GitHub)" <gi...@apache.org>.
talatuyarer commented on code in PR #12808:
URL: https://github.com/apache/iceberg/pull/12808#discussion_r2067701639


##########
bigquery/src/test/java/org/apache/iceberg/gcp/bigquery/BigQueryTableOperationsTest.java:
##########
@@ -0,0 +1,274 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.iceberg.gcp.bigquery;
+
+import static org.apache.iceberg.gcp.bigquery.BigQueryMetastoreCatalog.PROJECT_ID;
+import static org.apache.iceberg.types.Types.NestedField.required;
+import static org.assertj.core.api.Assertions.assertThat;
+import static org.assertj.core.api.Assertions.assertThatThrownBy;
+import static org.mockito.Mockito.any;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.reset;
+import static org.mockito.Mockito.times;
+import static org.mockito.Mockito.verify;
+import static org.mockito.Mockito.when;
+
+import com.google.api.services.bigquery.model.Dataset;
+import com.google.api.services.bigquery.model.DatasetReference;
+import com.google.api.services.bigquery.model.ExternalCatalogDatasetOptions;
+import com.google.api.services.bigquery.model.ExternalCatalogTableOptions;
+import com.google.api.services.bigquery.model.StorageDescriptor;
+import com.google.api.services.bigquery.model.Table;
+import com.google.api.services.bigquery.model.TableReference;
+import java.io.File;
+import java.io.IOException;
+import java.util.Collections;
+import java.util.Optional;
+import org.apache.commons.io.FileUtils;
+import org.apache.commons.io.filefilter.TrueFileFilter;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.iceberg.CatalogProperties;
+import org.apache.iceberg.PartitionSpec;
+import org.apache.iceberg.Schema;
+import org.apache.iceberg.catalog.TableIdentifier;
+import org.apache.iceberg.exceptions.CommitFailedException;
+import org.apache.iceberg.exceptions.NoSuchTableException;
+import org.apache.iceberg.exceptions.ValidationException;
+import org.apache.iceberg.relocated.com.google.common.collect.ImmutableMap;
+import org.apache.iceberg.types.Types;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.io.TempDir;
+import org.mockito.ArgumentCaptor;
+
+public class BigQueryTableOperationsTest {
+
+  @TempDir private File tempFolder;
+  public static final String METADATA_LOCATION_PROP = "metadata_location";
+  private static final String GCP_PROJECT = "my-project";
+  private static final String GCP_REGION = "us";
+  private static final String DATASET_ID = "db";
+  private static final String TABLE_ID = "tbl";
+  private static final TableIdentifier SPARK_TABLE_ID = TableIdentifier.of(DATASET_ID, TABLE_ID);
+
+  private static final TableReference TABLE_REFERENCE =
+      new TableReference().setProjectId(GCP_PROJECT).setDatasetId(DATASET_ID).setTableId(TABLE_ID);
+
+  private final BigQueryMetastoreClient bigQueryMetaStoreClient =
+      mock(BigQueryMetastoreClient.class);
+
+  private BigQueryMetastoreCatalog bigQueryMetastoreCatalog;
+  private BigQueryTableOperations tableOps;
+
+  @BeforeEach
+  public void before() {
+    this.bigQueryMetastoreCatalog = new BigQueryMetastoreCatalog();
+    this.bigQueryMetastoreCatalog.setConf(new Configuration());
+    String warehouseLocation = tempFolder.toPath().resolve("hive-warehouse").toString();
+
+    bigQueryMetastoreCatalog.initialize(
+        "CATALOG_ID",
+        ImmutableMap.of(
+            PROJECT_ID,
+            GCP_PROJECT,
+            CatalogProperties.WAREHOUSE_LOCATION,
+            warehouseLocation,
+            CatalogProperties.FILE_IO_IMPL,
+            "org.apache.iceberg.hadoop.HadoopFileIO"),
+        GCP_PROJECT,
+        GCP_REGION,
+        bigQueryMetaStoreClient);
+    this.tableOps = (BigQueryTableOperations) bigQueryMetastoreCatalog.newTableOps(SPARK_TABLE_ID);
+  }
+
+  @Test
+  public void testFetchLatestMetadataFromBigQuery() throws Exception {
+    Table createdTable = createTestTable();
+    reset(bigQueryMetaStoreClient);
+    when(bigQueryMetaStoreClient.load(TABLE_REFERENCE)).thenReturn(createdTable);
+
+    tableOps.refresh();
+    assertThat(
+            createdTable
+                .getExternalCatalogTableOptions()
+                .getParameters()
+                .getOrDefault(METADATA_LOCATION_PROP, ""))
+        .isEqualTo(tableOps.currentMetadataLocation());
+
+    reset(bigQueryMetaStoreClient);
+    when(bigQueryMetaStoreClient.load(TABLE_REFERENCE))
+        .thenThrow(new NoSuchTableException("error message getTable"));
+    // Refresh fails when table is not found but metadata already presents.
+    assertThatThrownBy(() -> tableOps.refresh())
+        .isInstanceOf(NoSuchTableException.class)
+        .hasMessageContaining("error message getTable");
+  }
+
+  @Test
+  public void testFailForNonIcebergTable() {
+    when(bigQueryMetaStoreClient.load(TABLE_REFERENCE))
+        .thenReturn(new Table().setTableReference(TABLE_REFERENCE));
+
+    assertThatThrownBy(() -> tableOps.refresh())
+        .isInstanceOf(ValidationException.class)
+        .hasMessageContaining("metadata location not found");
+  }
+
+  @Test
+  public void testNoOpWhenMetadataAndTableNotFound() {
+    when(bigQueryMetaStoreClient.load(TABLE_REFERENCE))
+        .thenThrow(new NoSuchTableException("error message getTable"));
+    // Table not found won't cause errors when the metadata is null.
+    assertThat(tableOps.currentMetadataLocation()).isNull();
+    tableOps.refresh();
+  }
+
+  @Test
+  public void testTableNameAsExpected() {
+    assertThat(tableOps.tableName()).isEqualTo("db.tbl");
+  }
+
+  @Test
+  public void testUseEtagForUpdateTable() throws Exception {
+    Table tableWithEtag = createTestTable().setEtag("etag");
+    reset(bigQueryMetaStoreClient);
+    when(bigQueryMetaStoreClient.load(TABLE_REFERENCE)).thenReturn(tableWithEtag, tableWithEtag);
+
+    org.apache.iceberg.Table loadedTable = bigQueryMetastoreCatalog.loadTable(SPARK_TABLE_ID);
+
+    when(bigQueryMetaStoreClient.update(any(), any())).thenReturn(tableWithEtag);
+    loadedTable.updateSchema().addColumn("n", Types.IntegerType.get()).commit();
+
+    ArgumentCaptor<TableReference> tableReferenceArgumentCaptor =
+        ArgumentCaptor.forClass(TableReference.class);
+    ArgumentCaptor<Table> tableArgumentCaptor = ArgumentCaptor.forClass(Table.class);
+    verify(bigQueryMetaStoreClient, times(1))
+        .update(tableReferenceArgumentCaptor.capture(), tableArgumentCaptor.capture());
+    assertThat(tableReferenceArgumentCaptor.getValue()).isEqualTo(TABLE_REFERENCE);
+    assertThat(tableArgumentCaptor.getValue().getEtag()).isEqualTo("etag");
+  }
+
+  @Test
+  public void testFailWhenEtagMismatch() throws Exception {
+    Table tableWithEtag = createTestTable().setEtag("etag");
+    reset(bigQueryMetaStoreClient);
+    when(bigQueryMetaStoreClient.load(TABLE_REFERENCE)).thenReturn(tableWithEtag, tableWithEtag);
+
+    org.apache.iceberg.Table loadedTable = bigQueryMetastoreCatalog.loadTable(SPARK_TABLE_ID);
+
+    when(bigQueryMetaStoreClient.update(any(), any()))
+        .thenThrow(new ValidationException("error message etag mismatch"));
+    assertThatThrownBy(
+            () -> loadedTable.updateSchema().addColumn("n", Types.IntegerType.get()).commit())
+        .isInstanceOf(CommitFailedException.class)
+        .hasMessageContaining(
+            "Updating table failed due to conflict updates (etag mismatch). Retry the update");
+  }
+
+  @Test
+  public void testFailWhenMetadataLocationDiff() throws Exception {
+    Table tableWithEtag = createTestTable().setEtag("etag");
+    Table tableWithNewMetadata =
+        new Table()
+            .setEtag("etag")
+            .setExternalCatalogTableOptions(
+                new ExternalCatalogTableOptions()
+                    .setParameters(ImmutableMap.of(METADATA_LOCATION_PROP, "a/new/location")));
+
+    reset(bigQueryMetaStoreClient);
+    // Two invocations, for loadTable and commit.
+    when(bigQueryMetaStoreClient.load(TABLE_REFERENCE))
+        .thenReturn(tableWithEtag, tableWithNewMetadata);
+
+    org.apache.iceberg.Table loadedTable = bigQueryMetastoreCatalog.loadTable(SPARK_TABLE_ID);
+
+    when(bigQueryMetaStoreClient.update(any(), any())).thenReturn(tableWithEtag);
+    assertThatThrownBy(
+            () -> loadedTable.updateSchema().addColumn("n", Types.IntegerType.get()).commit())
+        .isInstanceOf(CommitFailedException.class)
+        .hasMessageContaining("is not same as the current table metadata location");
+  }
+
+  @Test
+  public void testCreateTableCommitSucceeds() throws Exception {
+    var testTable = createTestTable();
+    when(bigQueryMetaStoreClient.create(any(Table.class))).thenReturn(testTable);
+    when(bigQueryMetaStoreClient.load(
+            new DatasetReference().setProjectId(GCP_PROJECT).setDatasetId(DATASET_ID)))
+        .thenReturn(
+            new Dataset()
+                .setExternalCatalogDatasetOptions(
+                    new ExternalCatalogDatasetOptions()
+                        .setDefaultStorageLocationUri("build/db_folder")));
+
+    Schema schema = testSchema();
+    bigQueryMetastoreCatalog.createTable(SPARK_TABLE_ID, schema, PartitionSpec.unpartitioned());
+  }
+
+  /** Creates a test table to have Iceberg metadata files in place. */
+  private Table createTestTable() throws Exception {
+    when(bigQueryMetaStoreClient.load(TABLE_REFERENCE))
+        .thenThrow(new NoSuchTableException("error message getTable"));
+    return createTestTable(tempFolder, bigQueryMetastoreCatalog, TABLE_REFERENCE);
+  }
+
+  public static Table createTestTable(
+      File tempFolder,
+      BigQueryMetastoreCatalog bigQueryMetastoreCatalog,
+      TableReference tableReference)
+      throws IOException {
+    Schema schema = testSchema();
+    TableIdentifier tableIdentifier =
+        TableIdentifier.of(tableReference.getDatasetId(), tableReference.getTableId());
+    String tableDir = tempFolder.toPath().resolve(tableReference.getTableId()).toString();
+
+    bigQueryMetastoreCatalog
+        .buildTable(tableIdentifier, schema)
+        .withLocation(tableDir)
+        .createTransaction()
+        .commitTransaction();
+
+    Optional<String> metadataLocation = metadataFilePath(tableDir);
+    assertThat(metadataLocation).isPresent();
+    return new Table()
+        .setTableReference(tableReference)
+        .setExternalCatalogTableOptions(
+            new ExternalCatalogTableOptions()
+                .setStorageDescriptor(new StorageDescriptor().setLocationUri(tableDir))
+                .setParameters(
+                    Collections.singletonMap(METADATA_LOCATION_PROP, metadataLocation.get())));
+  }
+
+  private static Schema testSchema() {

Review Comment:
   Done



-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org

For queries about this service, please contact Infrastructure at:
users@infra.apache.org


---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For additional commands, e-mail: issues-help@iceberg.apache.org


Re: [PR] Catalog: Add BigQuery Metastore Catalog Support [iceberg]

Posted by "talatuyarer (via GitHub)" <gi...@apache.org>.
talatuyarer commented on code in PR #12808:
URL: https://github.com/apache/iceberg/pull/12808#discussion_r2067702487


##########
bigquery/src/test/java/org/apache/iceberg/gcp/bigquery/BigQueryTableOperationsTest.java:
##########
@@ -0,0 +1,274 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.iceberg.gcp.bigquery;
+
+import static org.apache.iceberg.gcp.bigquery.BigQueryMetastoreCatalog.PROJECT_ID;
+import static org.apache.iceberg.types.Types.NestedField.required;
+import static org.assertj.core.api.Assertions.assertThat;
+import static org.assertj.core.api.Assertions.assertThatThrownBy;
+import static org.mockito.Mockito.any;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.reset;
+import static org.mockito.Mockito.times;
+import static org.mockito.Mockito.verify;
+import static org.mockito.Mockito.when;
+
+import com.google.api.services.bigquery.model.Dataset;
+import com.google.api.services.bigquery.model.DatasetReference;
+import com.google.api.services.bigquery.model.ExternalCatalogDatasetOptions;
+import com.google.api.services.bigquery.model.ExternalCatalogTableOptions;
+import com.google.api.services.bigquery.model.StorageDescriptor;
+import com.google.api.services.bigquery.model.Table;
+import com.google.api.services.bigquery.model.TableReference;
+import java.io.File;
+import java.io.IOException;
+import java.util.Collections;
+import java.util.Optional;
+import org.apache.commons.io.FileUtils;
+import org.apache.commons.io.filefilter.TrueFileFilter;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.iceberg.CatalogProperties;
+import org.apache.iceberg.PartitionSpec;
+import org.apache.iceberg.Schema;
+import org.apache.iceberg.catalog.TableIdentifier;
+import org.apache.iceberg.exceptions.CommitFailedException;
+import org.apache.iceberg.exceptions.NoSuchTableException;
+import org.apache.iceberg.exceptions.ValidationException;
+import org.apache.iceberg.relocated.com.google.common.collect.ImmutableMap;
+import org.apache.iceberg.types.Types;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.io.TempDir;
+import org.mockito.ArgumentCaptor;
+
+public class BigQueryTableOperationsTest {
+
+  @TempDir private File tempFolder;
+  public static final String METADATA_LOCATION_PROP = "metadata_location";
+  private static final String GCP_PROJECT = "my-project";
+  private static final String GCP_REGION = "us";
+  private static final String DATASET_ID = "db";
+  private static final String TABLE_ID = "tbl";
+  private static final TableIdentifier SPARK_TABLE_ID = TableIdentifier.of(DATASET_ID, TABLE_ID);
+
+  private static final TableReference TABLE_REFERENCE =
+      new TableReference().setProjectId(GCP_PROJECT).setDatasetId(DATASET_ID).setTableId(TABLE_ID);
+
+  private final BigQueryMetastoreClient bigQueryMetaStoreClient =
+      mock(BigQueryMetastoreClient.class);
+
+  private BigQueryMetastoreCatalog bigQueryMetastoreCatalog;
+  private BigQueryTableOperations tableOps;
+
+  @BeforeEach
+  public void before() {
+    this.bigQueryMetastoreCatalog = new BigQueryMetastoreCatalog();
+    this.bigQueryMetastoreCatalog.setConf(new Configuration());
+    String warehouseLocation = tempFolder.toPath().resolve("hive-warehouse").toString();
+
+    bigQueryMetastoreCatalog.initialize(
+        "CATALOG_ID",
+        ImmutableMap.of(
+            PROJECT_ID,
+            GCP_PROJECT,
+            CatalogProperties.WAREHOUSE_LOCATION,
+            warehouseLocation,
+            CatalogProperties.FILE_IO_IMPL,
+            "org.apache.iceberg.hadoop.HadoopFileIO"),
+        GCP_PROJECT,
+        GCP_REGION,
+        bigQueryMetaStoreClient);
+    this.tableOps = (BigQueryTableOperations) bigQueryMetastoreCatalog.newTableOps(SPARK_TABLE_ID);
+  }
+
+  @Test
+  public void testFetchLatestMetadataFromBigQuery() throws Exception {

Review Comment:
   Catalogtest also have same issue. Do you think I should create a a seperate PR to fix naming on CatalogTests ? 



-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org

For queries about this service, please contact Infrastructure at:
users@infra.apache.org


---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For additional commands, e-mail: issues-help@iceberg.apache.org


Re: [PR] Catalog: Add BigQuery Metastore Catalog Support [iceberg]

Posted by "talatuyarer (via GitHub)" <gi...@apache.org>.
talatuyarer commented on code in PR #12808:
URL: https://github.com/apache/iceberg/pull/12808#discussion_r2067701133


##########
gradle/libs.versions.toml:
##########
@@ -51,6 +51,7 @@ guava = "33.4.7-jre"
 hadoop3 = "3.4.1"
 httpcomponents-httpclient5 = "5.4.3"
 hive2 = { strictly = "2.3.10"} # see rich version usage explanation above
+hive4 = { strictly = "4.0.1"}

Review Comment:
   No There is not. We use constant looks like it is also available on hive2. I removed dependency.



-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org

For queries about this service, please contact Infrastructure at:
users@infra.apache.org


---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For additional commands, e-mail: issues-help@iceberg.apache.org


Re: [PR] Catalog: Add BigQuery Metastore Catalog Support [iceberg]

Posted by "danielcweeks (via GitHub)" <gi...@apache.org>.
danielcweeks commented on code in PR #12808:
URL: https://github.com/apache/iceberg/pull/12808#discussion_r2085606448


##########
bigquery/src/main/java/org/apache/iceberg/gcp/bigquery/BigQueryMetastoreClientImpl.java:
##########
@@ -0,0 +1,660 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.iceberg.gcp.bigquery;
+
+import com.google.api.client.googleapis.javanet.GoogleNetHttpTransport;
+import com.google.api.client.googleapis.json.GoogleJsonError;
+import com.google.api.client.googleapis.json.GoogleJsonResponseException;
+import com.google.api.client.http.HttpHeaders;
+import com.google.api.client.http.HttpResponse;
+import com.google.api.client.http.HttpResponseException;
+import com.google.api.client.http.HttpStatusCodes;
+import com.google.api.client.json.gson.GsonFactory;
+import com.google.api.client.util.Data;
+import com.google.api.services.bigquery.Bigquery;
+import com.google.api.services.bigquery.BigqueryScopes;
+import com.google.api.services.bigquery.model.Dataset;
+import com.google.api.services.bigquery.model.DatasetList;
+import com.google.api.services.bigquery.model.DatasetList.Datasets;
+import com.google.api.services.bigquery.model.DatasetReference;
+import com.google.api.services.bigquery.model.ExternalCatalogDatasetOptions;
+import com.google.api.services.bigquery.model.ExternalCatalogTableOptions;
+import com.google.api.services.bigquery.model.Table;
+import com.google.api.services.bigquery.model.TableList;
+import com.google.api.services.bigquery.model.TableList.Tables;
+import com.google.api.services.bigquery.model.TableReference;
+import com.google.api.services.bigquery.model.TableSchema;
+import com.google.auth.http.HttpCredentialsAdapter;
+import com.google.auth.oauth2.GoogleCredentials;
+import com.google.cloud.BaseServiceException;
+import com.google.cloud.ExceptionHandler;
+import com.google.cloud.bigquery.BigQueryErrorMessages;
+import com.google.cloud.bigquery.BigQueryOptions;
+import com.google.cloud.bigquery.BigQueryRetryConfig;
+import com.google.cloud.bigquery.BigQueryRetryHelper;
+import java.io.IOException;
+import java.security.GeneralSecurityException;
+import java.util.List;
+import java.util.Locale;
+import java.util.Map;
+import java.util.Objects;
+import java.util.Set;
+import java.util.stream.Collectors;
+import java.util.stream.Stream;
+import org.apache.iceberg.BaseMetastoreTableOperations;
+import org.apache.iceberg.exceptions.AlreadyExistsException;
+import org.apache.iceberg.exceptions.BadRequestException;
+import org.apache.iceberg.exceptions.ForbiddenException;
+import org.apache.iceberg.exceptions.NamespaceNotEmptyException;
+import org.apache.iceberg.exceptions.NoSuchIcebergTableException;
+import org.apache.iceberg.exceptions.NoSuchNamespaceException;
+import org.apache.iceberg.exceptions.NoSuchTableException;
+import org.apache.iceberg.exceptions.NotAuthorizedException;
+import org.apache.iceberg.exceptions.RuntimeIOException;
+import org.apache.iceberg.exceptions.ServiceFailureException;
+import org.apache.iceberg.exceptions.ServiceUnavailableException;
+import org.apache.iceberg.exceptions.ValidationException;
+import org.apache.iceberg.relocated.com.google.common.base.Preconditions;
+import org.apache.iceberg.relocated.com.google.common.collect.Lists;
+import org.apache.iceberg.relocated.com.google.common.collect.Maps;
+
+/** A client of Google Bigquery Metastore functions over the BigQuery service. */
+public final class BigQueryMetastoreClientImpl implements BigQueryMetastoreClient {
+
+  private final Bigquery client;
+  private final BigQueryOptions bigqueryOptions;
+
+  public static final ExceptionHandler.Interceptor EXCEPTION_HANDLER_INTERCEPTOR =
+      new ExceptionHandler.Interceptor() {
+
+        @Override
+        public RetryResult afterEval(Exception exception, RetryResult retryResult) {
+          return ExceptionHandler.Interceptor.RetryResult.CONTINUE_EVALUATION;
+        }
+
+        @Override
+        public RetryResult beforeEval(Exception exception) {
+          if (exception instanceof BaseServiceException) {
+            boolean retriable = ((BaseServiceException) exception).isRetryable();
+            return retriable
+                ? ExceptionHandler.Interceptor.RetryResult.RETRY
+                : ExceptionHandler.Interceptor.RetryResult.CONTINUE_EVALUATION;
+          }
+
+          return ExceptionHandler.Interceptor.RetryResult.CONTINUE_EVALUATION;
+        }
+      };
+
+  // Retry config with error messages and regex for rate limit exceeded errors.
+  private static final BigQueryRetryConfig DEFAULT_RETRY_CONFIG =
+      BigQueryRetryConfig.newBuilder()
+          .retryOnMessage(BigQueryErrorMessages.RATE_LIMIT_EXCEEDED_MSG)
+          .retryOnMessage(BigQueryErrorMessages.JOB_RATE_LIMIT_EXCEEDED_MSG)
+          .retryOnRegEx(BigQueryErrorMessages.RetryRegExPatterns.RATE_LIMIT_EXCEEDED_REGEX)
+          .build();
+
+  public static final ExceptionHandler BIGQUERY_EXCEPTION_HANDLER =
+      ExceptionHandler.newBuilder()
+          .abortOn(RuntimeException.class)
+          // Retry on connection failures due to transient network issues.
+          .retryOn(java.net.ConnectException.class)
+          // Retry to recover from temporary DNS resolution failures.
+          .retryOn(java.net.UnknownHostException.class)
+          .addInterceptors(EXCEPTION_HANDLER_INTERCEPTOR)
+          .build();
+
+  /** Constructs a client of the Google BigQuery service. */
+  public BigQueryMetastoreClientImpl(BigQueryOptions options)
+      throws IOException, GeneralSecurityException {
+    // Initialize client that will be used to send requests. This client only needs to be created
+    // once, and can be reused for multiple requests
+    HttpCredentialsAdapter httpCredentialsAdapter =
+        new HttpCredentialsAdapter(
+            GoogleCredentials.getApplicationDefault().createScoped(BigqueryScopes.all()));
+    this.client =
+        new Bigquery.Builder(
+                GoogleNetHttpTransport.newTrustedTransport(),
+                GsonFactory.getDefaultInstance(),
+                httpRequest -> {
+                  httpCredentialsAdapter.initialize(httpRequest);
+                  // Instead of throwing exceptions, analyze the HttpResponse object and inspect its
+                  // status code. This will allow BigQuery API errors to be converted into Iceberg
+                  // exceptions.
+                  httpRequest.setThrowExceptionOnExecuteError(false);
+                })
+            .setApplicationName("BigQuery Metastore Iceberg Catalog Plugin")
+            .build();
+    this.bigqueryOptions = options;
+  }
+
+  @Override
+  public Dataset create(Dataset dataset) {
+    Dataset response = null;
+    try {
+      response =
+          BigQueryRetryHelper.runWithRetries(
+              () -> internalCreate(dataset),
+              bigqueryOptions.getRetrySettings(),
+              BIGQUERY_EXCEPTION_HANDLER,
+              bigqueryOptions.getClock(),
+              DEFAULT_RETRY_CONFIG);
+    } catch (BigQueryRetryHelper.BigQueryRetryHelperException e) {
+      handleBigQueryRetryException(e);
+    }
+    return response;
+  }
+
+  private Dataset internalCreate(Dataset dataset) {
+    try {
+      HttpResponse response =
+          client
+              .datasets()
+              .insert(dataset.getDatasetReference().getProjectId(), dataset)
+              .executeUnparsed();
+      return convertExceptionIfUnsuccessful(response).parseAs(Dataset.class);
+    } catch (IOException e) {
+      throw new RuntimeIOException(e);
+    } catch (AlreadyExistsException e) {
+      throw new AlreadyExistsException("Namespace already exists: %s", dataset.getId());
+    }
+  }
+
+  @Override
+  public Dataset load(DatasetReference datasetReference) {
+    try {
+      HttpResponse response =
+          client
+              .datasets()
+              .get(datasetReference.getProjectId(), datasetReference.getDatasetId())
+              .executeUnparsed();
+      if (response.getStatusCode() == HttpStatusCodes.STATUS_CODE_NOT_FOUND) {
+        throw new NoSuchNamespaceException(
+            "Namespace does not exist: %s", datasetReference.getDatasetId());
+      }
+
+      return convertExceptionIfUnsuccessful(response).parseAs(Dataset.class);
+    } catch (IOException e) {
+      throw new RuntimeIOException(e);
+    }
+  }
+
+  @Override
+  public void delete(DatasetReference datasetReference) {
+    try {
+      HttpResponse response =
+          client
+              .datasets()
+              .delete(datasetReference.getProjectId(), datasetReference.getDatasetId())
+              .executeUnparsed();
+      if (response.getStatusCode() == HttpStatusCodes.STATUS_CODE_NOT_FOUND) {
+        throw new NoSuchNamespaceException(
+            "Namespace does not exist: %s", datasetReference.getDatasetId());
+      }
+
+      convertExceptionIfUnsuccessful(response);
+    } catch (IOException e) {
+      throw new RuntimeIOException(e);
+    } catch (NamespaceNotEmptyException e) {
+      throw new NamespaceNotEmptyException(
+          "%s is not empty: %s", datasetReference.getDatasetId(), e.getMessage());
+    }
+  }
+
+  @Override
+  public boolean setParameters(DatasetReference datasetReference, Map<String, String> parameters) {
+    Dataset dataset = load(datasetReference);
+    ExternalCatalogDatasetOptions existingOptions = dataset.getExternalCatalogDatasetOptions();
+
+    Map<String, String> existingParameters =
+        (existingOptions == null || existingOptions.getParameters() == null)
+            ? Maps.newHashMap() // Use HashMap to allow modification below
+            : Maps.newHashMap(existingOptions.getParameters()); // Copy to compare later
+
+    // Calculate what the new parameters would be.
+    Map<String, String> newParameters = Maps.newHashMap(existingParameters);
+    newParameters.putAll(parameters);
+
+    if (Objects.equals(existingParameters, newParameters)) {
+      // No change in parameters detected
+      return false;
+    }
+
+    ExternalCatalogDatasetOptions optionsToUpdate =
+        existingOptions == null ? new ExternalCatalogDatasetOptions() : existingOptions;
+
+    dataset.setExternalCatalogDatasetOptions(optionsToUpdate.setParameters(newParameters));
+
+    try {
+      BigQueryRetryHelper.runWithRetries(
+          () -> internalUpdate(dataset),
+          bigqueryOptions.getRetrySettings(),
+          BIGQUERY_EXCEPTION_HANDLER,
+          bigqueryOptions.getClock(),
+          DEFAULT_RETRY_CONFIG);
+    } catch (BigQueryRetryHelper.BigQueryRetryHelperException e) {
+      handleBigQueryRetryException(e);
+    }
+
+    return true;
+  }
+
+  @Override
+  public boolean removeParameters(DatasetReference datasetReference, Set<String> parameters) {
+    Dataset dataset = load(datasetReference); // Throws NoSuchNamespaceException if not found.
+
+    ExternalCatalogDatasetOptions existingOptions = dataset.getExternalCatalogDatasetOptions();
+
+    if (existingOptions == null
+        || existingOptions.getParameters() == null
+        || existingOptions.getParameters().isEmpty()) {
+      return false;
+    }
+
+    Map<String, String> existingParameters = Maps.newHashMap(existingOptions.getParameters());
+
+    // Calculate the new parameters map.
+    Map<String, String> newParameters = Maps.newHashMap(existingParameters);
+    parameters.forEach(newParameters::remove);
+
+    if (Objects.equals(existingParameters, newParameters)) {
+      // No change in parameters detected (e.g., keys to remove didn't exist)
+      return false;
+    }
+
+    dataset.setExternalCatalogDatasetOptions(existingOptions.setParameters(newParameters));
+
+    try {
+      BigQueryRetryHelper.runWithRetries(
+          () -> internalUpdate(dataset),
+          bigqueryOptions.getRetrySettings(),
+          BIGQUERY_EXCEPTION_HANDLER,
+          bigqueryOptions.getClock(),
+          DEFAULT_RETRY_CONFIG);
+    } catch (BigQueryRetryHelper.BigQueryRetryHelperException e) {
+      handleBigQueryRetryException(e);
+    }
+    return true;
+  }
+
+  @Override
+  public List<Datasets> list(String projectId) {
+    try {
+      String nextPageToken = null;
+      List<Datasets> datasets = Lists.newArrayList();
+      do {
+        HttpResponse pageResponse =
+            client.datasets().list(projectId).setPageToken(nextPageToken).executeUnparsed();
+        DatasetList result =
+            convertExceptionIfUnsuccessful(pageResponse).parseAs(DatasetList.class);
+        nextPageToken = result.getNextPageToken();
+        if (result.getDatasets() != null) {
+          datasets.addAll(result.getDatasets());
+        }
+
+      } while (nextPageToken != null && !nextPageToken.isEmpty());
+      return datasets;
+    } catch (IOException e) {
+      throw new RuntimeIOException(e);
+    }
+  }
+
+  @Override
+  public Table create(Table table) {
+    // Ensure it is an Iceberg table supported by the BigQuery metastore catalog.
+    validateTable(table);
+    // TODO: Ensure table creation is idempotent when handling retries.
+    Table response = null;
+    try {
+      response =
+          BigQueryRetryHelper.runWithRetries(
+              () -> internalCreate(table),
+              bigqueryOptions.getRetrySettings(),
+              BIGQUERY_EXCEPTION_HANDLER,
+              bigqueryOptions.getClock(),
+              DEFAULT_RETRY_CONFIG);
+    } catch (BigQueryRetryHelper.BigQueryRetryHelperException e) {
+      handleBigQueryRetryException(e);
+    }
+
+    return response;
+  }
+
+  private Table internalCreate(Table table) {
+    try {
+      HttpResponse response =
+          client
+              .tables()
+              .insert(
+                  Preconditions.checkNotNull(table.getTableReference()).getProjectId(),
+                  Preconditions.checkNotNull(table.getTableReference()).getDatasetId(),
+                  table)
+              .executeUnparsed();
+      return convertExceptionIfUnsuccessful(response).parseAs(Table.class);
+    } catch (IOException e) {
+      throw new RuntimeIOException("%s", e);
+    } catch (AlreadyExistsException e) {
+      throw new AlreadyExistsException(e, "Table already exists: %s", table);
+    }
+  }
+
+  @Override
+  public Table load(TableReference tableReference) {
+    try {
+      HttpResponse response =
+          client
+              .tables()
+              .get(
+                  tableReference.getProjectId(),
+                  tableReference.getDatasetId(),
+                  tableReference.getTableId())
+              .executeUnparsed();
+      if (response.getStatusCode() == HttpStatusCodes.STATUS_CODE_NOT_FOUND) {
+        throw new NoSuchTableException("%s", response.getStatusMessage());
+      }
+
+      return validateTable(convertExceptionIfUnsuccessful(response).parseAs(Table.class));
+    } catch (IOException e) {
+      throw new RuntimeIOException("%s", e);
+    }
+  }
+
+  @Override
+  public Table update(TableReference tableReference, Table table) {
+    // Ensure it is an Iceberg table supported by the BQ metastore catalog.
+    validateTable(table);
+
+    ExternalCatalogTableOptions newExternalCatalogTableOptions =
+        new ExternalCatalogTableOptions()
+            .setStorageDescriptor(table.getExternalCatalogTableOptions().getStorageDescriptor())
+            .setConnectionId(table.getExternalCatalogTableOptions().getConnectionId())
+            .setParameters(table.getExternalCatalogTableOptions().getParameters());
+    Table updatedTable =
+        new Table()
+            .setExternalCatalogTableOptions(newExternalCatalogTableOptions)
+            // Must set the schema as null for using schema auto-detect.
+            .setSchema(Data.nullOf(TableSchema.class));
+
+    Table response = null;
+    try {
+      response =
+          BigQueryRetryHelper.runWithRetries(
+              () -> internalUpdate(tableReference, updatedTable, table.getEtag()),
+              bigqueryOptions.getRetrySettings(),
+              BIGQUERY_EXCEPTION_HANDLER,
+              bigqueryOptions.getClock(),
+              DEFAULT_RETRY_CONFIG);
+    } catch (BigQueryRetryHelper.BigQueryRetryHelperException e) {
+      handleBigQueryRetryException(e);
+    }
+
+    return response;
+  }
+
+  private Table internalUpdate(TableReference tableReference, Table table, String etag) {
+    try {
+      HttpResponse response =
+          client
+              .tables()
+              .patch(
+                  tableReference.getProjectId(),
+                  tableReference.getDatasetId(),
+                  tableReference.getTableId(),
+                  table)
+              .setRequestHeaders(new HttpHeaders().setIfMatch(etag))
+              .executeUnparsed();
+
+      if (response.getStatusCode() == HttpStatusCodes.STATUS_CODE_NOT_FOUND) {
+        String responseString = response.parseAsString();
+        if (responseString.toLowerCase(Locale.ENGLISH).contains("not found: connection")) {
+          throw new BadRequestException("%s", responseString);
+        }
+
+        throw new NoSuchTableException("%s", response.getStatusMessage());
+      }
+
+      return convertExceptionIfUnsuccessful(response).parseAs(Table.class);
+    } catch (IOException e) {
+      throw new RuntimeIOException("%s", e);
+    }
+  }
+
+  @Override
+  public void delete(TableReference tableReference) {
+    try {
+      load(tableReference); // Fetching it to validate it is a BigQuery Metastore table first
+
+      HttpResponse response =
+          client
+              .tables()
+              .delete(
+                  tableReference.getProjectId(),
+                  tableReference.getDatasetId(),
+                  tableReference.getTableId())
+              .executeUnparsed();
+
+      if (response.getStatusCode() == HttpStatusCodes.STATUS_CODE_NOT_FOUND) {
+        throw new NoSuchTableException("%s", response.getStatusMessage());
+      }
+
+      convertExceptionIfUnsuccessful(response);
+    } catch (IOException e) {
+      throw new RuntimeIOException("%s", e);
+    }
+  }
+
+  @Override
+  public List<Tables> list(DatasetReference datasetReference, boolean filterUnsupportedTables) {

Review Comment:
   This flag, `filterUnsupportedTables`, is inverted from the behavior of the `list-all-tables` property.  So, if you set list all, it actually filters the tables.



-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org

For queries about this service, please contact Infrastructure at:
users@infra.apache.org


---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For additional commands, e-mail: issues-help@iceberg.apache.org


Re: [PR] Catalog: Add BigQuery Metastore Catalog Support [iceberg]

Posted by "juldrixx (via GitHub)" <gi...@apache.org>.
juldrixx commented on PR #12808:
URL: https://github.com/apache/iceberg/pull/12808#issuecomment-2873067133

   Is there any thought to make this catalog implementation available in the kafka connect runtime? I guess by adding it [here](https://github.com/apache/iceberg/blob/main/kafka-connect/build.gradle#L128) (not sure if it's the only thing to do)


-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org

For queries about this service, please contact Infrastructure at:
users@infra.apache.org


---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For additional commands, e-mail: issues-help@iceberg.apache.org


Re: [PR] Catalog: Add BigQuery Metastore Catalog Support [iceberg]

Posted by "nastra (via GitHub)" <gi...@apache.org>.
nastra commented on code in PR #12808:
URL: https://github.com/apache/iceberg/pull/12808#discussion_r2054342592


##########
bigquery/src/main/java/org/apache/iceberg/gcp/bigquery/BigQueryMetastoreCatalog.java:
##########
@@ -0,0 +1,402 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.iceberg.gcp.bigquery;
+
+import com.google.api.services.bigquery.model.Dataset;
+import com.google.api.services.bigquery.model.DatasetList.Datasets;
+import com.google.api.services.bigquery.model.DatasetReference;
+import com.google.api.services.bigquery.model.ExternalCatalogDatasetOptions;
+import com.google.api.services.bigquery.model.TableReference;
+import com.google.cloud.ServiceOptions;
+import com.google.cloud.bigquery.BigQueryOptions;
+import java.io.IOException;
+import java.security.GeneralSecurityException;
+import java.util.List;
+import java.util.Locale;
+import java.util.Map;
+import java.util.Objects;
+import java.util.Set;
+import org.apache.hadoop.conf.Configurable;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.iceberg.BaseMetastoreCatalog;
+import org.apache.iceberg.CatalogProperties;
+import org.apache.iceberg.CatalogUtil;
+import org.apache.iceberg.TableMetadata;
+import org.apache.iceberg.TableOperations;
+import org.apache.iceberg.catalog.Namespace;
+import org.apache.iceberg.catalog.SupportsNamespaces;
+import org.apache.iceberg.catalog.TableIdentifier;
+import org.apache.iceberg.exceptions.AlreadyExistsException;
+import org.apache.iceberg.exceptions.NoSuchNamespaceException;
+import org.apache.iceberg.exceptions.NoSuchTableException;
+import org.apache.iceberg.exceptions.ServiceFailureException;
+import org.apache.iceberg.exceptions.ValidationException;
+import org.apache.iceberg.io.FileIO;
+import org.apache.iceberg.relocated.com.google.common.annotations.VisibleForTesting;
+import org.apache.iceberg.relocated.com.google.common.base.Preconditions;
+import org.apache.iceberg.relocated.com.google.common.base.Strings;
+import org.apache.iceberg.relocated.com.google.common.collect.ImmutableList;
+import org.apache.iceberg.relocated.com.google.common.collect.ImmutableMap;
+import org.apache.iceberg.relocated.com.google.common.collect.Maps;
+import org.apache.iceberg.util.LocationUtil;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/** Iceberg Bigquery Metastore Catalog implementation. */
+public final class BigQueryMetastoreCatalog extends BaseMetastoreCatalog
+    implements SupportsNamespaces, Configurable {
+
+  // User provided properties.
+  public static final String PROPERTIES_KEY_GCP_PROJECT = "gcp-project";
+  public static final String PROPERTIES_KEY_GCP_LOCATION = "gcp-location";
+  public static final String PROPERTIES_KEY_FILTER_UNSUPPORTED_TABLES = "filter-unsupported-tables";
+  public static final String PROPERTIES_KEY_TESTING_ENABLED = "testing-enabled";
+
+  public static final String HIVE_METASTORE_WAREHOUSE_DIR = "hive.metastore.warehouse.dir";
+
+  private static final Logger LOG = LoggerFactory.getLogger(BigQueryMetastoreCatalog.class);
+
+  private static final String DEFAULT_GCP_LOCATION = "us";
+
+  private String catalogPluginName;
+  private Map<String, String> catalogProperties;
+  private FileIO fileIO;
+  private Configuration conf;
+  private String projectId;
+  private String location;
+  private BigQueryMetastoreClient client;
+  private boolean filterUnsupportedTables;
+
+  // Must have a no-arg constructor to be dynamically loaded
+  // initialize(String name, Map<String, String> properties) will be called to complete
+  // initialization
+  public BigQueryMetastoreCatalog() {}
+
+  @Override
+  public void initialize(String inputName, Map<String, String> properties) {
+    if (!properties.containsKey(PROPERTIES_KEY_GCP_PROJECT)) {
+      throw new ValidationException("GCP project must be specified");

Review Comment:
   we typically use `Preconditions.checkArgument` for argument validations like these



-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org

For queries about this service, please contact Infrastructure at:
users@infra.apache.org


---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For additional commands, e-mail: issues-help@iceberg.apache.org


Re: [PR] Catalog: Add BigQuery Metastore Catalog Support [iceberg]

Posted by "nastra (via GitHub)" <gi...@apache.org>.
nastra commented on code in PR #12808:
URL: https://github.com/apache/iceberg/pull/12808#discussion_r2054354585


##########
bigquery/src/main/java/org/apache/iceberg/gcp/bigquery/BigQueryMetastoreCatalog.java:
##########
@@ -0,0 +1,402 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.iceberg.gcp.bigquery;
+
+import com.google.api.services.bigquery.model.Dataset;
+import com.google.api.services.bigquery.model.DatasetList.Datasets;
+import com.google.api.services.bigquery.model.DatasetReference;
+import com.google.api.services.bigquery.model.ExternalCatalogDatasetOptions;
+import com.google.api.services.bigquery.model.TableReference;
+import com.google.cloud.ServiceOptions;
+import com.google.cloud.bigquery.BigQueryOptions;
+import java.io.IOException;
+import java.security.GeneralSecurityException;
+import java.util.List;
+import java.util.Locale;
+import java.util.Map;
+import java.util.Objects;
+import java.util.Set;
+import org.apache.hadoop.conf.Configurable;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.iceberg.BaseMetastoreCatalog;
+import org.apache.iceberg.CatalogProperties;
+import org.apache.iceberg.CatalogUtil;
+import org.apache.iceberg.TableMetadata;
+import org.apache.iceberg.TableOperations;
+import org.apache.iceberg.catalog.Namespace;
+import org.apache.iceberg.catalog.SupportsNamespaces;
+import org.apache.iceberg.catalog.TableIdentifier;
+import org.apache.iceberg.exceptions.AlreadyExistsException;
+import org.apache.iceberg.exceptions.NoSuchNamespaceException;
+import org.apache.iceberg.exceptions.NoSuchTableException;
+import org.apache.iceberg.exceptions.ServiceFailureException;
+import org.apache.iceberg.exceptions.ValidationException;
+import org.apache.iceberg.io.FileIO;
+import org.apache.iceberg.relocated.com.google.common.annotations.VisibleForTesting;
+import org.apache.iceberg.relocated.com.google.common.base.Preconditions;
+import org.apache.iceberg.relocated.com.google.common.base.Strings;
+import org.apache.iceberg.relocated.com.google.common.collect.ImmutableList;
+import org.apache.iceberg.relocated.com.google.common.collect.ImmutableMap;
+import org.apache.iceberg.relocated.com.google.common.collect.Maps;
+import org.apache.iceberg.util.LocationUtil;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/** Iceberg Bigquery Metastore Catalog implementation. */
+public final class BigQueryMetastoreCatalog extends BaseMetastoreCatalog
+    implements SupportsNamespaces, Configurable {
+
+  // User provided properties.
+  public static final String PROPERTIES_KEY_GCP_PROJECT = "gcp-project";
+  public static final String PROPERTIES_KEY_GCP_LOCATION = "gcp-location";
+  public static final String PROPERTIES_KEY_FILTER_UNSUPPORTED_TABLES = "filter-unsupported-tables";
+  public static final String PROPERTIES_KEY_TESTING_ENABLED = "testing-enabled";
+
+  public static final String HIVE_METASTORE_WAREHOUSE_DIR = "hive.metastore.warehouse.dir";
+
+  private static final Logger LOG = LoggerFactory.getLogger(BigQueryMetastoreCatalog.class);
+
+  private static final String DEFAULT_GCP_LOCATION = "us";
+
+  private String catalogPluginName;

Review Comment:
   ```suggestion
     private String catalogName;
   ```



-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org

For queries about this service, please contact Infrastructure at:
users@infra.apache.org


---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For additional commands, e-mail: issues-help@iceberg.apache.org


Re: [PR] Catalog: Add BigQuery Metastore Catalog Support [iceberg]

Posted by "gkalra18 (via GitHub)" <gi...@apache.org>.
gkalra18 commented on code in PR #12808:
URL: https://github.com/apache/iceberg/pull/12808#discussion_r2047681629


##########
bigquery/src/main/java/org/apache/iceberg/gcp/bigquery/BigQueryMetastoreCatalog.java:
##########
@@ -0,0 +1,403 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.iceberg.gcp.bigquery;
+
+import com.google.api.services.bigquery.model.Dataset;
+import com.google.api.services.bigquery.model.DatasetList.Datasets;
+import com.google.api.services.bigquery.model.DatasetReference;
+import com.google.api.services.bigquery.model.ExternalCatalogDatasetOptions;
+import com.google.api.services.bigquery.model.TableReference;
+import com.google.cloud.ServiceOptions;
+import com.google.cloud.bigquery.BigQueryOptions;
+import java.io.IOException;
+import java.security.GeneralSecurityException;
+import java.util.List;
+import java.util.Locale;
+import java.util.Map;
+import java.util.Objects;
+import java.util.Set;
+import org.apache.hadoop.conf.Configurable;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.iceberg.BaseMetastoreCatalog;
+import org.apache.iceberg.CatalogProperties;
+import org.apache.iceberg.CatalogUtil;
+import org.apache.iceberg.TableMetadata;
+import org.apache.iceberg.TableOperations;
+import org.apache.iceberg.catalog.Namespace;
+import org.apache.iceberg.catalog.SupportsNamespaces;
+import org.apache.iceberg.catalog.TableIdentifier;
+import org.apache.iceberg.exceptions.AlreadyExistsException;
+import org.apache.iceberg.exceptions.NoSuchNamespaceException;
+import org.apache.iceberg.exceptions.NoSuchTableException;
+import org.apache.iceberg.exceptions.ServiceFailureException;
+import org.apache.iceberg.exceptions.ValidationException;
+import org.apache.iceberg.io.FileIO;
+import org.apache.iceberg.relocated.com.google.common.annotations.VisibleForTesting;
+import org.apache.iceberg.relocated.com.google.common.base.Preconditions;
+import org.apache.iceberg.relocated.com.google.common.base.Strings;
+import org.apache.iceberg.relocated.com.google.common.collect.ImmutableList;
+import org.apache.iceberg.relocated.com.google.common.collect.ImmutableMap;
+import org.apache.iceberg.relocated.com.google.common.collect.Maps;
+import org.apache.iceberg.util.LocationUtil;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/** Iceberg Bigquery Metastore Catalog implementation. */
+public final class BigQueryMetastoreCatalog extends BaseMetastoreCatalog
+    implements SupportsNamespaces, Configurable {
+
+  // User provided properties.
+  public static final String PROPERTIES_KEY_GCP_PROJECT = "gcp-project";
+  public static final String PROPERTIES_KEY_GCP_LOCATION = "gcp-location";
+  public static final String PROPERTIES_KEY_FILTER_UNSUPPORTED_TABLES = "filter-unsupported-tables";
+  public static final String PROPERTIES_KEY_TESTING_ENABLED = "testing-enabled";
+
+  public static final String HIVE_METASTORE_WAREHOUSE_DIR = "hive.metastore.warehouse.dir";
+
+  private static final Logger LOG = LoggerFactory.getLogger(BigQueryMetastoreCatalog.class);
+
+  private static final String DEFAULT_GCP_LOCATION = "us";
+
+  private String catalogPluginName;
+  private Map<String, String> catalogProperties;
+  private FileIO fileIO;
+  private Configuration conf;
+  private String projectId;
+  private String location;
+  private BigQueryMetaStoreClient client;
+  private boolean filterUnsupportedTables;
+
+  // Must have a no-arg constructor to be dynamically loaded
+  // initialize(String name, Map<String, String> properties) will be called to complete
+  // initialization
+  public BigQueryMetastoreCatalog() {}
+
+  @Override
+  public void initialize(String inputName, Map<String, String> properties) {
+    if (!properties.containsKey(PROPERTIES_KEY_GCP_PROJECT)) {
+      throw new ValidationException("GCP project must be specified");
+    }
+    projectId = properties.get(PROPERTIES_KEY_GCP_PROJECT);
+    location = properties.getOrDefault(PROPERTIES_KEY_GCP_LOCATION, DEFAULT_GCP_LOCATION);
+    boolean testingEnabled =
+        Boolean.parseBoolean(properties.getOrDefault(PROPERTIES_KEY_TESTING_ENABLED, "false"));
+
+    BigQueryOptions options =
+        BigQueryOptions.newBuilder()
+            .setProjectId(projectId)
+            .setLocation(location)
+            .setRetrySettings(ServiceOptions.getDefaultRetrySettings())
+            .build();
+
+    try {
+      if (testingEnabled) {
+        client = new FakeBigQueryMetaStoreClient(options);
+
+      } else {
+        client = new BigQueryMetaStoreClientImpl(options);
+      }
+    } catch (IOException e) {
+      throw new ServiceFailureException(e, "Creating BigQuery client failed");
+    } catch (GeneralSecurityException e) {
+      throw new ValidationException(e, "Creating BigQuery client failed due to a security issue");
+    }
+    initialize(inputName, properties, projectId, location, client);
+  }
+
+  @VisibleForTesting
+  void initialize(
+      String inputName,
+      Map<String, String> properties,
+      String initialProjectId,
+      String initialLocation,
+      BigQueryMetaStoreClient bigQueryMetaStoreClient) {
+    this.catalogPluginName = inputName;
+    this.catalogProperties = ImmutableMap.copyOf(properties);
+    this.projectId = initialProjectId;
+    this.location = initialLocation;
+    this.client =
+        Preconditions.checkNotNull(bigQueryMetaStoreClient, "BigQuery client can not be null");
+
+    if (this.conf == null) {
+      LOG.warn("No configuration was set, using the default environment Configuration");
+      this.conf = new Configuration();
+    }
+
+    LOG.info("Using BigQuery Metastore Iceberg Catalog: {}", inputName);
+
+    if (properties.containsKey(CatalogProperties.WAREHOUSE_LOCATION)) {
+      // Iceberg always removes trailing slash to avoid paths like "<folder>//data" in file systems
+      // like s3.
+      this.conf.set(
+          HIVE_METASTORE_WAREHOUSE_DIR,
+          LocationUtil.stripTrailingSlash(properties.get(CatalogProperties.WAREHOUSE_LOCATION)));
+    }
+
+    String fileIoImpl =
+        properties.getOrDefault(
+            CatalogProperties.FILE_IO_IMPL, "org.apache.iceberg.hadoop.HadoopFileIO");
+    this.fileIO = CatalogUtil.loadFileIO(fileIoImpl, properties, conf);
+
+    this.filterUnsupportedTables =
+        Boolean.parseBoolean(
+            properties.getOrDefault(PROPERTIES_KEY_FILTER_UNSUPPORTED_TABLES, "false"));
+  }
+
+  @Override
+  protected TableOperations newTableOps(TableIdentifier identifier) {
+    return new BigQueryTableOperations(
+        client,
+        fileIO,
+        projectId,
+        // Sometimes extensions have the namespace contain the table name too, so we are forced to
+        // allow invalid namespace and just take the first part here like other catalog
+        // implementations do.
+        identifier.namespace().level(0),
+        identifier.name(),
+        conf);
+  }
+
+  @Override
+  protected String defaultWarehouseLocation(TableIdentifier identifier) {
+    String locationUri = null;
+    DatasetReference datasetReference = toDatasetReference(identifier.namespace());
+    Dataset dataset = client.getDataset(datasetReference);
+    if (dataset != null && dataset.getExternalCatalogDatasetOptions() != null) {
+      locationUri = dataset.getExternalCatalogDatasetOptions().getDefaultStorageLocationUri();
+    }
+    return String.format(
+        "%s/%s",
+        Strings.isNullOrEmpty(locationUri) ? datasetReference.getDatasetId() : locationUri,
+        identifier.name());
+  }
+
+  @Override
+  public List<TableIdentifier> listTables(Namespace namespace) {
+    validateNamespace(namespace);
+
+    return client.listTables(toDatasetReference(namespace), filterUnsupportedTables).stream()
+        .map(
+            table -> TableIdentifier.of(namespace.level(0), table.getTableReference().getTableId()))
+        .collect(ImmutableList.toImmutableList());
+  }
+
+  @Override
+  public boolean dropTable(TableIdentifier identifier, boolean purge) {
+    try {
+      TableOperations ops = newTableOps(identifier);
+      TableMetadata lastMetadata = ops.current();
+
+      client.deleteTable(toTableReference(identifier));
+
+      if (purge && lastMetadata != null) {
+        CatalogUtil.dropTableData(ops.io(), lastMetadata);
+      }
+    } catch (NoSuchTableException e) { // Not catching a NoSuchIcebergTableException on purpose
+      return false; // The documentation says just return false in this case
+    }
+    return true;
+  }
+
+  @Override
+  public void renameTable(TableIdentifier from, TableIdentifier to) {
+    validateNamespace(from.namespace());
+    validateNamespace(to.namespace());
+
+    try {
+      client.getDataset(toDatasetReference(to.namespace()));
+      client.getDataset(toDatasetReference(from.namespace()));
+    } catch (NoSuchNamespaceException e) {
+      throw e;
+    }
+
+    TableReference fromTableRef = toTableReference(from);
+    TableReference toTableRef = toTableReference(to);
+
+    // Check if the source table exists
+    try {
+      client.getTable(fromTableRef);
+    } catch (NoSuchTableException e) {
+      throw new NoSuchTableException("Table does not exist: %s", from.name());
+    }
+
+    // Check if the destination table already exists
+    try {
+      client.getTable(toTableRef);
+      // If getTable succeeds, the destination table exists
+      throw new AlreadyExistsException("Table already exists: %s", to.name());
+    } catch (NoSuchTableException e) {
+      // Destination table does not exist, proceed with throwing exception
+    }
+
+    if (!from.namespace().equals(to.namespace())) {
+      throw new ValidationException("New table name must be in the same namespace");
+    }
+
+    // TODO(b/354981675): Enable once supported by the API.
+    throw new UnsupportedOperationException(
+        String.format(
+            "Table rename operation is unsupported. Try the SQL operation directly on BigQuery: \"ALTER TABLE %s RENAME TO %s;\"",

Review Comment:
   Let's remove `Try the SQL operation directly on BigQuery: \"ALTER TABLE %s RENAME TO %s;`. We don't have this capability in BigQuery yet.



-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org

For queries about this service, please contact Infrastructure at:
users@infra.apache.org


---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For additional commands, e-mail: issues-help@iceberg.apache.org


Re: [PR] Catalog: Add BigQuery Metastore Catalog Support [iceberg]

Posted by "talatuyarer (via GitHub)" <gi...@apache.org>.
talatuyarer commented on code in PR #12808:
URL: https://github.com/apache/iceberg/pull/12808#discussion_r2064341361


##########
bigquery/src/main/java/org/apache/iceberg/gcp/bigquery/BigQueryMetastoreCatalog.java:
##########
@@ -0,0 +1,394 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.iceberg.gcp.bigquery;
+
+import com.google.api.services.bigquery.model.Dataset;
+import com.google.api.services.bigquery.model.DatasetList.Datasets;
+import com.google.api.services.bigquery.model.DatasetReference;
+import com.google.api.services.bigquery.model.ExternalCatalogDatasetOptions;
+import com.google.api.services.bigquery.model.TableReference;
+import com.google.cloud.ServiceOptions;
+import com.google.cloud.bigquery.BigQueryOptions;
+import java.io.IOException;
+import java.security.GeneralSecurityException;
+import java.util.List;
+import java.util.Locale;
+import java.util.Map;
+import java.util.Objects;
+import java.util.Set;
+import org.apache.hadoop.conf.Configurable;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.iceberg.BaseMetastoreCatalog;
+import org.apache.iceberg.CatalogProperties;
+import org.apache.iceberg.CatalogUtil;
+import org.apache.iceberg.TableMetadata;
+import org.apache.iceberg.TableOperations;
+import org.apache.iceberg.catalog.Namespace;
+import org.apache.iceberg.catalog.SupportsNamespaces;
+import org.apache.iceberg.catalog.TableIdentifier;
+import org.apache.iceberg.exceptions.AlreadyExistsException;
+import org.apache.iceberg.exceptions.NoSuchNamespaceException;
+import org.apache.iceberg.exceptions.NoSuchTableException;
+import org.apache.iceberg.exceptions.ServiceFailureException;
+import org.apache.iceberg.exceptions.ValidationException;
+import org.apache.iceberg.io.FileIO;
+import org.apache.iceberg.relocated.com.google.common.annotations.VisibleForTesting;
+import org.apache.iceberg.relocated.com.google.common.base.Preconditions;
+import org.apache.iceberg.relocated.com.google.common.base.Strings;
+import org.apache.iceberg.relocated.com.google.common.collect.ImmutableList;
+import org.apache.iceberg.relocated.com.google.common.collect.ImmutableMap;
+import org.apache.iceberg.relocated.com.google.common.collect.Maps;
+import org.apache.iceberg.util.LocationUtil;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/** Iceberg Bigquery Metastore Catalog implementation. */
+public class BigQueryMetastoreCatalog extends BaseMetastoreCatalog
+    implements SupportsNamespaces, Configurable {
+
+  // User provided properties.
+  public static final String PROJECT_ID = "gcp-project";
+  public static final String GCP_LOCATION = "gcp-location";
+  public static final String LIST_ALL_TABLES = "list-all-tables";
+
+  public static final String HIVE_METASTORE_WAREHOUSE_DIR = "hive.metastore.warehouse.dir";
+
+  private static final Logger LOG = LoggerFactory.getLogger(BigQueryMetastoreCatalog.class);
+
+  private static final String DEFAULT_GCP_LOCATION = "us";
+
+  private String catalogName;
+  private Map<String, String> catalogProperties;
+  private FileIO fileIO;
+  private Configuration conf;
+  private String projectId;
+  private String location;
+  private BigQueryMetastoreClient client;
+  private boolean listAllTables;
+
+  // Must have a no-arg constructor to be dynamically loaded
+  // initialize(String name, Map<String, String> properties) will be called to complete
+  // initialization
+  public BigQueryMetastoreCatalog() {}
+
+  @Override
+  public void initialize(String inputName, Map<String, String> properties) {
+    Preconditions.checkArgument(
+        properties.containsKey(PROJECT_ID), "GCP project must be specified");
+
+    projectId = properties.get(PROJECT_ID);
+    location = properties.getOrDefault(GCP_LOCATION, DEFAULT_GCP_LOCATION);
+
+    BigQueryOptions options =
+        BigQueryOptions.newBuilder()
+            .setProjectId(projectId)
+            .setLocation(location)
+            .setRetrySettings(ServiceOptions.getDefaultRetrySettings())
+            .build();
+
+    try {
+      client = new BigQueryMetastoreClientImpl(options);
+    } catch (IOException e) {
+      throw new ServiceFailureException(e, "Creating BigQuery client failed");
+    } catch (GeneralSecurityException e) {
+      throw new ValidationException(e, "Creating BigQuery client failed due to a security issue");
+    }
+
+    initialize(inputName, properties, projectId, location, client);
+  }
+
+  @VisibleForTesting
+  void initialize(
+      String inputName,
+      Map<String, String> properties,
+      String initialProjectId,
+      String initialLocation,
+      BigQueryMetastoreClient bigQueryMetaStoreClient) {
+    Preconditions.checkArgument(bigQueryMetaStoreClient != null, "BigQuery client can not be null");
+    this.catalogName = inputName;
+    this.catalogProperties = ImmutableMap.copyOf(properties);
+    this.projectId = initialProjectId;
+    this.location = initialLocation;
+    this.client = bigQueryMetaStoreClient;
+
+    if (this.conf == null) {
+      LOG.warn("No configuration was set, using the default environment Configuration");
+      this.conf = new Configuration();
+    }
+
+    LOG.info("Using BigQuery Metastore Iceberg Catalog: {}", inputName);
+
+    if (properties.containsKey(CatalogProperties.WAREHOUSE_LOCATION)) {
+      this.conf.set(
+          HIVE_METASTORE_WAREHOUSE_DIR,
+          LocationUtil.stripTrailingSlash(properties.get(CatalogProperties.WAREHOUSE_LOCATION)));
+    }
+
+    String fileIoImpl =
+        properties.getOrDefault(
+            CatalogProperties.FILE_IO_IMPL, "org.apache.iceberg.hadoop.HadoopFileIO");
+    this.fileIO = CatalogUtil.loadFileIO(fileIoImpl, properties, conf);
+
+    this.listAllTables = Boolean.parseBoolean(properties.getOrDefault(LIST_ALL_TABLES, "true"));
+  }
+
+  @Override
+  protected TableOperations newTableOps(TableIdentifier identifier) {
+    return new BigQueryTableOperations(client, fileIO, toTableReference(identifier), conf);
+  }
+
+  @Override
+  protected String defaultWarehouseLocation(TableIdentifier identifier) {
+    String locationUri = null;
+    DatasetReference datasetReference = toDatasetReference(identifier.namespace());
+    Dataset dataset = client.load(datasetReference);
+    if (dataset != null && dataset.getExternalCatalogDatasetOptions() != null) {
+      locationUri = dataset.getExternalCatalogDatasetOptions().getDefaultStorageLocationUri();
+    }
+
+    return String.format(
+        "%s/%s",
+        Strings.isNullOrEmpty(locationUri)
+            ? createDefaultStorageLocationUri(datasetReference.getDatasetId())
+            : LocationUtil.stripTrailingSlash(locationUri),
+        identifier.name());
+  }
+
+  @Override
+  public List<TableIdentifier> listTables(Namespace namespace) {
+    validateNamespace(namespace);
+
+    return client.list(toDatasetReference(namespace), listAllTables).stream()
+        .map(
+            table -> TableIdentifier.of(namespace.level(0), table.getTableReference().getTableId()))
+        .collect(ImmutableList.toImmutableList());
+  }
+
+  @Override
+  public boolean dropTable(TableIdentifier identifier, boolean purge) {
+    try {
+      TableOperations ops = newTableOps(identifier);
+      TableMetadata lastMetadata = ops.current();
+
+      client.delete(toTableReference(identifier));
+
+      if (purge && lastMetadata != null) {
+        CatalogUtil.dropTableData(ops.io(), lastMetadata);
+      }
+    } catch (NoSuchTableException e) {
+      return false;
+    }
+
+    return true;
+  }
+
+  @Override
+  public void renameTable(TableIdentifier from, TableIdentifier to) {
+    validateNamespace(from.namespace());
+    validateNamespace(to.namespace());
+
+    if (!namespaceExists(to.namespace())) {
+      throw new NoSuchNamespaceException("Namespace does not exist: %s", to.namespace());
+    }
+
+    if (!namespaceExists(from.namespace())) {
+      throw new NoSuchNamespaceException("Namespace does not exist: %s", from.namespace());
+    }
+
+    // Check if the source table exists
+    if (!tableExists(from)) {
+      throw new NoSuchTableException("Table does not exist: %s", from.name());
+    }
+
+    if (!from.namespace().equals(to.namespace())) {
+      throw new ValidationException("New table name must be in the same namespace");

Review Comment:
   I applied this to all errors in renameTable function



-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org

For queries about this service, please contact Infrastructure at:
users@infra.apache.org


---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For additional commands, e-mail: issues-help@iceberg.apache.org


Re: [PR] Catalog: Add BigQuery Metastore Catalog Support [iceberg]

Posted by "nastra (via GitHub)" <gi...@apache.org>.
nastra commented on code in PR #12808:
URL: https://github.com/apache/iceberg/pull/12808#discussion_r2066097914


##########
bigquery/src/test/java/org/apache/iceberg/gcp/bigquery/BigQueryTableOperationsTest.java:
##########
@@ -0,0 +1,274 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.iceberg.gcp.bigquery;
+
+import static org.apache.iceberg.gcp.bigquery.BigQueryMetastoreCatalog.PROJECT_ID;
+import static org.apache.iceberg.types.Types.NestedField.required;
+import static org.assertj.core.api.Assertions.assertThat;
+import static org.assertj.core.api.Assertions.assertThatThrownBy;
+import static org.mockito.Mockito.any;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.reset;
+import static org.mockito.Mockito.times;
+import static org.mockito.Mockito.verify;
+import static org.mockito.Mockito.when;
+
+import com.google.api.services.bigquery.model.Dataset;
+import com.google.api.services.bigquery.model.DatasetReference;
+import com.google.api.services.bigquery.model.ExternalCatalogDatasetOptions;
+import com.google.api.services.bigquery.model.ExternalCatalogTableOptions;
+import com.google.api.services.bigquery.model.StorageDescriptor;
+import com.google.api.services.bigquery.model.Table;
+import com.google.api.services.bigquery.model.TableReference;
+import java.io.File;
+import java.io.IOException;
+import java.util.Collections;
+import java.util.Optional;
+import org.apache.commons.io.FileUtils;
+import org.apache.commons.io.filefilter.TrueFileFilter;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.iceberg.CatalogProperties;
+import org.apache.iceberg.PartitionSpec;
+import org.apache.iceberg.Schema;
+import org.apache.iceberg.catalog.TableIdentifier;
+import org.apache.iceberg.exceptions.CommitFailedException;
+import org.apache.iceberg.exceptions.NoSuchTableException;
+import org.apache.iceberg.exceptions.ValidationException;
+import org.apache.iceberg.relocated.com.google.common.collect.ImmutableMap;
+import org.apache.iceberg.types.Types;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.io.TempDir;
+import org.mockito.ArgumentCaptor;
+
+public class BigQueryTableOperationsTest {
+
+  @TempDir private File tempFolder;
+  public static final String METADATA_LOCATION_PROP = "metadata_location";
+  private static final String GCP_PROJECT = "my-project";
+  private static final String GCP_REGION = "us";
+  private static final String DATASET_ID = "db";
+  private static final String TABLE_ID = "tbl";
+  private static final TableIdentifier SPARK_TABLE_ID = TableIdentifier.of(DATASET_ID, TABLE_ID);
+
+  private static final TableReference TABLE_REFERENCE =
+      new TableReference().setProjectId(GCP_PROJECT).setDatasetId(DATASET_ID).setTableId(TABLE_ID);
+
+  private final BigQueryMetastoreClient bigQueryMetaStoreClient =
+      mock(BigQueryMetastoreClient.class);
+
+  private BigQueryMetastoreCatalog bigQueryMetastoreCatalog;
+  private BigQueryTableOperations tableOps;
+
+  @BeforeEach
+  public void before() {
+    this.bigQueryMetastoreCatalog = new BigQueryMetastoreCatalog();
+    this.bigQueryMetastoreCatalog.setConf(new Configuration());
+    String warehouseLocation = tempFolder.toPath().resolve("hive-warehouse").toString();
+
+    bigQueryMetastoreCatalog.initialize(
+        "CATALOG_ID",
+        ImmutableMap.of(
+            PROJECT_ID,
+            GCP_PROJECT,
+            CatalogProperties.WAREHOUSE_LOCATION,
+            warehouseLocation,
+            CatalogProperties.FILE_IO_IMPL,
+            "org.apache.iceberg.hadoop.HadoopFileIO"),
+        GCP_PROJECT,
+        GCP_REGION,
+        bigQueryMetaStoreClient);
+    this.tableOps = (BigQueryTableOperations) bigQueryMetastoreCatalog.newTableOps(SPARK_TABLE_ID);
+  }
+
+  @Test
+  public void testFetchLatestMetadataFromBigQuery() throws Exception {
+    Table createdTable = createTestTable();
+    reset(bigQueryMetaStoreClient);
+    when(bigQueryMetaStoreClient.load(TABLE_REFERENCE)).thenReturn(createdTable);
+
+    tableOps.refresh();
+    assertThat(
+            createdTable
+                .getExternalCatalogTableOptions()
+                .getParameters()
+                .getOrDefault(METADATA_LOCATION_PROP, ""))
+        .isEqualTo(tableOps.currentMetadataLocation());
+
+    reset(bigQueryMetaStoreClient);
+    when(bigQueryMetaStoreClient.load(TABLE_REFERENCE))
+        .thenThrow(new NoSuchTableException("error message getTable"));
+    // Refresh fails when table is not found but metadata already presents.
+    assertThatThrownBy(() -> tableOps.refresh())
+        .isInstanceOf(NoSuchTableException.class)
+        .hasMessageContaining("error message getTable");
+  }
+
+  @Test
+  public void testFailForNonIcebergTable() {
+    when(bigQueryMetaStoreClient.load(TABLE_REFERENCE))
+        .thenReturn(new Table().setTableReference(TABLE_REFERENCE));
+
+    assertThatThrownBy(() -> tableOps.refresh())
+        .isInstanceOf(ValidationException.class)
+        .hasMessageContaining("metadata location not found");
+  }
+
+  @Test
+  public void testNoOpWhenMetadataAndTableNotFound() {
+    when(bigQueryMetaStoreClient.load(TABLE_REFERENCE))
+        .thenThrow(new NoSuchTableException("error message getTable"));
+    // Table not found won't cause errors when the metadata is null.
+    assertThat(tableOps.currentMetadataLocation()).isNull();
+    tableOps.refresh();

Review Comment:
   I would rather wrap this in a `assertThatNoException().isThrownBy(..)` to make it clearer that this doesn't throw an exception here, otherwise the test code looks like the refresh could be removed



-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org

For queries about this service, please contact Infrastructure at:
users@infra.apache.org


---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For additional commands, e-mail: issues-help@iceberg.apache.org


Re: [PR] Catalog: Add BigQuery Metastore Catalog Support [iceberg]

Posted by "nastra (via GitHub)" <gi...@apache.org>.
nastra commented on code in PR #12808:
URL: https://github.com/apache/iceberg/pull/12808#discussion_r2055539916


##########
bigquery/src/test/java/org/apache/iceberg/gcp/bigquery/TestBigQueryCatalog.java:
##########
@@ -0,0 +1,673 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.iceberg.gcp.bigquery;
+
+import static org.apache.iceberg.CatalogUtil.ICEBERG_CATALOG_TYPE;
+import static org.apache.iceberg.CatalogUtil.ICEBERG_CATALOG_TYPE_BIGQUERY;
+import static org.apache.iceberg.gcp.bigquery.BigQueryMetastoreCatalog.PROJECT_ID;
+import static org.apache.iceberg.gcp.bigquery.BigQueryMetastoreCatalog.TESTING_ENABLED;
+import static org.apache.iceberg.types.Types.NestedField.optional;
+import static org.assertj.core.api.Assertions.assertThat;
+import static org.assertj.core.api.Assertions.assertThatThrownBy;
+
+import java.io.File;
+import java.util.List;
+import java.util.Map;
+import org.apache.iceberg.CatalogProperties;
+import org.apache.iceberg.CatalogUtil;
+import org.apache.iceberg.PartitionSpec;
+import org.apache.iceberg.Schema;
+import org.apache.iceberg.SortOrder;
+import org.apache.iceberg.Table;
+import org.apache.iceberg.TableUtil;
+import org.apache.iceberg.Transaction;
+import org.apache.iceberg.catalog.CatalogTests;
+import org.apache.iceberg.catalog.Namespace;
+import org.apache.iceberg.catalog.TableIdentifier;
+import org.apache.iceberg.exceptions.NoSuchNamespaceException;
+import org.apache.iceberg.exceptions.NoSuchTableException;
+import org.apache.iceberg.expressions.Expressions;
+import org.apache.iceberg.relocated.com.google.common.collect.ImmutableMap;
+import org.apache.iceberg.types.Types;
+import org.junit.jupiter.api.AfterEach;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Disabled;
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.io.TempDir;
+import org.junit.jupiter.params.ParameterizedTest;
+import org.junit.jupiter.params.provider.ValueSource;
+
+public class TestBigQueryCatalog extends CatalogTests<BigQueryMetastoreCatalog> {
+
+  private static final String GCP_PROJECT = "project-id";
+  private static final String CATALOG_ID = "catalog-name";
+
+  // another schema that is not the same
+  protected static final Schema OTHER_SCHEMA_OPTIONAL =
+      new Schema(optional(1, "some_id", Types.IntegerType.get()));
+
+  // Schema passed to create tables
+  protected static final Schema SCHEMA_OPTIONAL =
+      new Schema(
+          optional(3, "id", Types.IntegerType.get(), "unique ID 🤪"),
+          optional(4, "data", Types.StringType.get()));
+
+  // This is the actual schema for the table, with column IDs reassigned
+  protected static final Schema REPLACE_SCHEMA_OPTIONAL =
+      new Schema(
+          optional(2, "id", Types.IntegerType.get(), "unique ID 🤪"),
+          optional(3, "data", Types.StringType.get()));
+
+  protected static final PartitionSpec REPLACE_SPEC_OPTIONAL =
+      PartitionSpec.builderFor(REPLACE_SCHEMA_OPTIONAL).bucket("id", 16).withSpecId(1).build();
+
+  protected static final SortOrder REPLACE_WRITE_ORDER_OPTIONAL =
+      SortOrder.builderFor(REPLACE_SCHEMA_OPTIONAL)
+          .asc(Expressions.bucket("id", 16))
+          .asc("id")
+          .build();
+
+  @TempDir private File tempFolder;
+  private BigQueryMetastoreCatalog catalog;
+  private BigQueryMetastoreClient fakeBigQueryMetastoreClient;
+  private String warehouseLocation;
+
+  @BeforeEach
+  public void before() throws Exception {
+    catalog = initCatalog(CATALOG_ID, ImmutableMap.of());
+  }
+
+  @AfterEach
+  public void after() throws Exception {
+    // Drop all tables in all datasets
+    List<Namespace> namespaces = catalog().listNamespaces(Namespace.empty());
+    for (Namespace namespace : namespaces) {
+      List<TableIdentifier> tables = catalog().listTables(namespace);
+      for (TableIdentifier table : tables) {
+        try {
+          catalog().dropTable(table, true); // drop with purge
+        } catch (NoSuchTableException e) {
+          // Table might be dropped by a previous iteration, ignore
+        }
+      }
+
+      // Drop all datasets (except any initial ones)
+      try {
+        catalog().dropNamespace(namespace);
+      } catch (NoSuchNamespaceException e) {
+        // Namespace might be dropped by a previous iteration, ignore
+      }
+    }
+  }
+
+  @Override
+  protected boolean requiresNamespaceCreate() {
+    return true;
+  }
+
+  @Override
+  protected boolean supportsNamesWithSlashes() {
+    return false;
+  }
+
+  @Override
+  protected boolean supportsNamesWithDot() {
+    return false;
+  }
+
+  @Override
+  public BigQueryMetastoreCatalog catalog() {
+    return catalog;
+  }
+
+  @Override
+  protected BigQueryMetastoreCatalog initCatalog(
+      String catalogName, Map<String, String> additionalProperties) {
+
+    warehouseLocation = tempFolder.toPath().resolve("hive-warehouse").toString();
+
+    Map<String, String> properties =
+        Map.of(
+            ICEBERG_CATALOG_TYPE,
+            ICEBERG_CATALOG_TYPE_BIGQUERY,
+            PROJECT_ID,
+            GCP_PROJECT,
+            CatalogProperties.WAREHOUSE_LOCATION,
+            warehouseLocation,
+            TESTING_ENABLED,
+            "true",
+            CatalogProperties.TABLE_DEFAULT_PREFIX + "default-key1",
+            "catalog-default-key1",
+            CatalogProperties.TABLE_DEFAULT_PREFIX + "default-key2",
+            "catalog-default-key2",
+            CatalogProperties.TABLE_DEFAULT_PREFIX + "override-key3",
+            "catalog-default-key3",
+            CatalogProperties.TABLE_OVERRIDE_PREFIX + "override-key3",
+            "catalog-override-key3",
+            CatalogProperties.TABLE_OVERRIDE_PREFIX + "override-key4",
+            "catalog-override-key4");
+
+    BigQueryMetastoreCatalog tmpCatalog =
+        (BigQueryMetastoreCatalog)
+            CatalogUtil.buildIcebergCatalog(
+                catalogName,
+                ImmutableMap.<String, String>builder()
+                    .putAll(properties)
+                    .putAll(additionalProperties)
+                    .build(),
+                null);
+
+    return tmpCatalog;
+  }
+
+  @Test
+  public void testRenameTable() {
+    if (requiresNamespaceCreate()) {
+      catalog.createNamespace(NS);
+    }
+
+    assertThat(catalog.tableExists(TABLE))
+        .as("Source table should not exist before create")
+        .isFalse();
+
+    catalog.buildTable(TABLE, SCHEMA).create();
+    assertThat(catalog.tableExists(TABLE)).as("Table should exist after create").isTrue();
+
+    assertThat(catalog.tableExists(RENAMED_TABLE))
+        .as("Destination table should not exist before rename")
+        .isFalse();
+
+    assertThatThrownBy(() -> catalog.renameTable(TABLE, RENAMED_TABLE))
+        .isInstanceOf(UnsupportedOperationException.class)
+        .hasMessageContaining("Table rename operation is unsupported");
+
+    catalog.dropTable(TABLE);
+  }
+
+  @Disabled(
+      "BigQuery does not allow adding a REQUIRED column to an existing table schema. "
+          + "When this operation is attempted, an exception gets thrown.")
+  @Test
+  public void testOverrideTablePropertiesReplaceTransaction() {}
+
+  @Test
+  public void testConcurrentReplaceTransactionSchema() {

Review Comment:
   TBH I don't really understand why all of these custom tests are needed and also why certain tests are Disabled as they all pass (maybe the disabling was from the old PR?). I was able to trim down the test class to the diff below and everything was passing.
   ```
   public class TestBigQueryCatalog extends CatalogTests<BigQueryMetastoreCatalog> {
   
     @TempDir private File tempFolder;
     private BigQueryMetastoreCatalog catalog;
   
     @BeforeEach
     public void before() {
       catalog = initCatalog("catalog-name", ImmutableMap.of());
     }
   
     @AfterEach
     public void after() throws Exception {
       // Drop all tables in all datasets
       List<Namespace> namespaces = catalog().listNamespaces(Namespace.empty());
       for (Namespace namespace : namespaces) {
         List<TableIdentifier> tables = catalog().listTables(namespace);
         for (TableIdentifier table : tables) {
           try {
             catalog().dropTable(table, true); // drop with purge
           } catch (NoSuchTableException e) {
             // Table might be dropped by a previous iteration, ignore
           }
         }
   
         // Drop all datasets (except any initial ones)
         try {
           catalog().dropNamespace(namespace);
         } catch (NoSuchNamespaceException e) {
           // Namespace might be dropped by a previous iteration, ignore
         }
       }
     }
   
     @Override
     protected boolean requiresNamespaceCreate() {
       return true;
     }
   
     @Override
     protected boolean supportsNamesWithSlashes() {
       return false;
     }
   
     @Override
     protected boolean supportsNamesWithDot() {
       return false;
     }
   
     @Override
     public BigQueryMetastoreCatalog catalog() {
       return catalog;
     }
   
     @Override
     protected BigQueryMetastoreCatalog initCatalog(
         String catalogName, Map<String, String> additionalProperties) {
   
       String warehouseLocation = tempFolder.toPath().resolve("hive-warehouse").toString();
   
       Map<String, String> properties =
           Map.of(
               ICEBERG_CATALOG_TYPE,
               ICEBERG_CATALOG_TYPE_BIGQUERY,
               BigQueryMetastoreCatalog.PROJECT_ID,
               "projectId",
               CatalogProperties.WAREHOUSE_LOCATION,
               warehouseLocation,
               TESTING_ENABLED,
               "true",
               CatalogProperties.TABLE_DEFAULT_PREFIX + "default-key1",
               "catalog-default-key1",
               CatalogProperties.TABLE_DEFAULT_PREFIX + "default-key2",
               "catalog-default-key2",
               CatalogProperties.TABLE_DEFAULT_PREFIX + "override-key3",
               "catalog-default-key3",
               CatalogProperties.TABLE_OVERRIDE_PREFIX + "override-key3",
               "catalog-override-key3",
               CatalogProperties.TABLE_OVERRIDE_PREFIX + "override-key4",
               "catalog-override-key4");
   
       BigQueryMetastoreCatalog tmpCatalog =
           (BigQueryMetastoreCatalog)
               CatalogUtil.buildIcebergCatalog(
                   catalogName,
                   ImmutableMap.<String, String>builder()
                       .putAll(properties)
                       .putAll(additionalProperties)
                       .build(),
                   null);
   
       return tmpCatalog;
     }
   
     @Test
     public void testRenameTable() {
       if (requiresNamespaceCreate()) {
         catalog.createNamespace(NS);
       }
   
       assertThat(catalog.tableExists(TABLE))
           .as("Source table should not exist before create")
           .isFalse();
   
       catalog.buildTable(TABLE, SCHEMA).create();
       assertThat(catalog.tableExists(TABLE)).as("Table should exist after create").isTrue();
   
       assertThat(catalog.tableExists(RENAMED_TABLE))
           .as("Destination table should not exist before rename")
           .isFalse();
   
       assertThatThrownBy(() -> catalog.renameTable(TABLE, RENAMED_TABLE))
           .isInstanceOf(UnsupportedOperationException.class)
           .hasMessageContaining("Table rename operation is unsupported");
   
       catalog.dropTable(TABLE);
     }
   }
   ```



-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org

For queries about this service, please contact Infrastructure at:
users@infra.apache.org


---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For additional commands, e-mail: issues-help@iceberg.apache.org


Re: [PR] Catalog: Add BigQuery Metastore Catalog Support [iceberg]

Posted by "nastra (via GitHub)" <gi...@apache.org>.
nastra commented on code in PR #12808:
URL: https://github.com/apache/iceberg/pull/12808#discussion_r2055643247


##########
bigquery/src/main/java/org/apache/iceberg/gcp/bigquery/BigQueryMetastoreCatalog.java:
##########
@@ -0,0 +1,400 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.iceberg.gcp.bigquery;
+
+import com.google.api.services.bigquery.model.Dataset;
+import com.google.api.services.bigquery.model.DatasetList.Datasets;
+import com.google.api.services.bigquery.model.DatasetReference;
+import com.google.api.services.bigquery.model.ExternalCatalogDatasetOptions;
+import com.google.api.services.bigquery.model.TableReference;
+import com.google.cloud.ServiceOptions;
+import com.google.cloud.bigquery.BigQueryOptions;
+import java.io.IOException;
+import java.security.GeneralSecurityException;
+import java.util.List;
+import java.util.Locale;
+import java.util.Map;
+import java.util.Objects;
+import java.util.Set;
+import org.apache.hadoop.conf.Configurable;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.iceberg.BaseMetastoreCatalog;
+import org.apache.iceberg.CatalogProperties;
+import org.apache.iceberg.CatalogUtil;
+import org.apache.iceberg.TableMetadata;
+import org.apache.iceberg.TableOperations;
+import org.apache.iceberg.catalog.Namespace;
+import org.apache.iceberg.catalog.SupportsNamespaces;
+import org.apache.iceberg.catalog.TableIdentifier;
+import org.apache.iceberg.exceptions.AlreadyExistsException;
+import org.apache.iceberg.exceptions.NoSuchNamespaceException;
+import org.apache.iceberg.exceptions.NoSuchTableException;
+import org.apache.iceberg.exceptions.ServiceFailureException;
+import org.apache.iceberg.exceptions.ValidationException;
+import org.apache.iceberg.io.FileIO;
+import org.apache.iceberg.relocated.com.google.common.annotations.VisibleForTesting;
+import org.apache.iceberg.relocated.com.google.common.base.Preconditions;
+import org.apache.iceberg.relocated.com.google.common.base.Strings;
+import org.apache.iceberg.relocated.com.google.common.collect.ImmutableList;
+import org.apache.iceberg.relocated.com.google.common.collect.ImmutableMap;
+import org.apache.iceberg.relocated.com.google.common.collect.Maps;
+import org.apache.iceberg.util.LocationUtil;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/** Iceberg Bigquery Metastore Catalog implementation. */
+public final class BigQueryMetastoreCatalog extends BaseMetastoreCatalog
+    implements SupportsNamespaces, Configurable {
+
+  // User provided properties.
+  public static final String PROJECT_ID = "gcp-project";
+  public static final String GCP_LOCATION = "gcp-location";
+  public static final String FILTER_UNSUPPORTED_TABLES = "filter-unsupported-tables";
+  public static final String TESTING_ENABLED = "testing-enabled";
+
+  public static final String HIVE_METASTORE_WAREHOUSE_DIR = "hive.metastore.warehouse.dir";
+
+  private static final Logger LOG = LoggerFactory.getLogger(BigQueryMetastoreCatalog.class);
+
+  private static final String DEFAULT_GCP_LOCATION = "us";
+
+  private String catalogName;
+  private Map<String, String> catalogProperties;
+  private FileIO fileIO;
+  private Configuration conf;
+  private String projectId;
+  private String location;
+  private BigQueryMetastoreClient client;
+  private boolean filterUnsupportedTables;
+
+  // Must have a no-arg constructor to be dynamically loaded
+  // initialize(String name, Map<String, String> properties) will be called to complete
+  // initialization
+  public BigQueryMetastoreCatalog() {}
+
+  @Override
+  public void initialize(String inputName, Map<String, String> properties) {
+    Preconditions.checkArgument(
+        properties.containsKey(PROJECT_ID), "GCP project must be specified");
+
+    projectId = properties.get(PROJECT_ID);
+    location = properties.getOrDefault(GCP_LOCATION, DEFAULT_GCP_LOCATION);
+    boolean testingEnabled =
+        Boolean.parseBoolean(properties.getOrDefault(TESTING_ENABLED, "false"));
+
+    BigQueryOptions options =
+        BigQueryOptions.newBuilder()
+            .setProjectId(projectId)
+            .setLocation(location)
+            .setRetrySettings(ServiceOptions.getDefaultRetrySettings())
+            .build();
+
+    try {
+      if (testingEnabled) {
+        client = new FakeBigQueryMetastoreClient(options);
+
+      } else {
+        client = new BigQueryMetastoreClientImpl(options);
+      }
+    } catch (IOException e) {
+      throw new ServiceFailureException(e, "Creating BigQuery client failed");
+    } catch (GeneralSecurityException e) {
+      throw new ValidationException(e, "Creating BigQuery client failed due to a security issue");
+    }
+    initialize(inputName, properties, projectId, location, client);
+  }
+
+  @VisibleForTesting
+  void initialize(
+      String inputName,
+      Map<String, String> properties,
+      String initialProjectId,
+      String initialLocation,
+      BigQueryMetastoreClient bigQueryMetaStoreClient) {
+    Preconditions.checkArgument(bigQueryMetaStoreClient != null, "BigQuery client can not be null");
+    this.catalogName = inputName;
+    this.catalogProperties = ImmutableMap.copyOf(properties);
+    this.projectId = initialProjectId;
+    this.location = initialLocation;
+    this.client = bigQueryMetaStoreClient;
+
+    if (this.conf == null) {
+      LOG.warn("No configuration was set, using the default environment Configuration");
+      this.conf = new Configuration();
+    }
+
+    LOG.info("Using BigQuery Metastore Iceberg Catalog: {}", inputName);
+
+    if (properties.containsKey(CatalogProperties.WAREHOUSE_LOCATION)) {
+      // Iceberg always removes trailing slash to avoid paths like "<folder>//data" in file systems
+      // like s3.
+      this.conf.set(
+          HIVE_METASTORE_WAREHOUSE_DIR,
+          LocationUtil.stripTrailingSlash(properties.get(CatalogProperties.WAREHOUSE_LOCATION)));
+    }
+
+    String fileIoImpl =
+        properties.getOrDefault(
+            CatalogProperties.FILE_IO_IMPL, "org.apache.iceberg.hadoop.HadoopFileIO");
+    this.fileIO = CatalogUtil.loadFileIO(fileIoImpl, properties, conf);
+
+    this.filterUnsupportedTables =
+        Boolean.parseBoolean(properties.getOrDefault(FILTER_UNSUPPORTED_TABLES, "false"));
+  }
+
+  @Override
+  protected TableOperations newTableOps(TableIdentifier identifier) {
+    return new BigQueryTableOperations(
+        client,
+        fileIO,
+        projectId,
+        // Sometimes extensions have the namespace contain the table name too, so we are forced to
+        // allow invalid namespace and just take the first part here like other catalog
+        // implementations do.
+        identifier.namespace().level(0),
+        identifier.name(),
+        conf);
+  }
+
+  @Override
+  protected String defaultWarehouseLocation(TableIdentifier identifier) {
+    String locationUri = null;
+    DatasetReference datasetReference = toDatasetReference(identifier.namespace());
+    Dataset dataset = client.getDataset(datasetReference);
+    if (dataset != null && dataset.getExternalCatalogDatasetOptions() != null) {
+      locationUri = dataset.getExternalCatalogDatasetOptions().getDefaultStorageLocationUri();
+    }
+    return String.format(
+        "%s/%s",
+        Strings.isNullOrEmpty(locationUri)
+            ? datasetReference.getDatasetId()
+            : LocationUtil.stripTrailingSlash(locationUri),
+        identifier.name());
+  }
+
+  @Override
+  public List<TableIdentifier> listTables(Namespace namespace) {
+    validateNamespace(namespace);
+
+    return client.listTables(toDatasetReference(namespace), filterUnsupportedTables).stream()
+        .map(
+            table -> TableIdentifier.of(namespace.level(0), table.getTableReference().getTableId()))
+        .collect(ImmutableList.toImmutableList());
+  }
+
+  @Override
+  public boolean dropTable(TableIdentifier identifier, boolean purge) {
+    try {
+      TableOperations ops = newTableOps(identifier);
+      TableMetadata lastMetadata = ops.current();
+
+      client.deleteTable(toTableReference(identifier));
+
+      if (purge && lastMetadata != null) {
+        CatalogUtil.dropTableData(ops.io(), lastMetadata);
+      }
+    } catch (NoSuchTableException e) { // Not catching a NoSuchIcebergTableException on purpose
+      return false; // The documentation says just return false in this case
+    }
+    return true;
+  }
+
+  @Override
+  public void renameTable(TableIdentifier from, TableIdentifier to) {
+    validateNamespace(from.namespace());
+    validateNamespace(to.namespace());
+
+    try {
+      client.getDataset(toDatasetReference(to.namespace()));
+      client.getDataset(toDatasetReference(from.namespace()));
+    } catch (NoSuchNamespaceException e) {
+      throw e;
+    }
+
+    TableReference fromTableRef = toTableReference(from);
+    TableReference toTableRef = toTableReference(to);
+
+    // Check if the source table exists
+    try {
+      client.getTable(fromTableRef);
+    } catch (NoSuchTableException e) {
+      throw new NoSuchTableException("Table does not exist: %s", from.name());
+    }
+
+    // Check if the destination table already exists
+    try {
+      client.getTable(toTableRef);
+      // If getTable succeeds, the destination table exists
+      throw new AlreadyExistsException("Table already exists: %s", to.name());
+    } catch (NoSuchTableException e) {
+      // Destination table does not exist, proceed with throwing exception
+    }
+
+    if (!from.namespace().equals(to.namespace())) {
+      throw new ValidationException("New table name must be in the same namespace");
+    }
+
+    // TODO(b/354981675): Enable once supported by the API.
+    throw new UnsupportedOperationException("Table rename operation is unsupported.");
+  }
+
+  @Override
+  public void createNamespace(Namespace namespace, Map<String, String> metadata) {
+    Dataset builder = new Dataset();
+    DatasetReference datasetReference = toDatasetReference(namespace);
+    builder.setLocation(this.location);
+    builder.setDatasetReference(datasetReference);
+    builder.setExternalCatalogDatasetOptions(
+        BigQueryMetastoreUtils.createExternalCatalogDatasetOptions(
+            datasetReference.getDatasetId(), metadata));
+
+    client.createDataset(builder);
+  }
+
+  /**
+   * Since this catalog only supports one-level namespaces, it always returns an empty list unless
+   * passed an empty namespace to list all namespaces within the catalog.
+   */
+  @Override
+  public List<Namespace> listNamespaces(Namespace namespace) {
+    if (namespace.levels().length != 0) {
+      // BQMS does not support namespaces under database or tables, returns empty.
+      // It is called when dropping a namespace to make sure it's empty (listTables is called as
+      // well), returns empty to unblock deletion.
+      return ImmutableList.of();
+    }
+    return client.listDatasets(projectId).stream()
+        .map(BigQueryMetastoreCatalog::getNamespace)
+        .collect(ImmutableList.toImmutableList());
+  }
+
+  @Override
+  public boolean dropNamespace(Namespace namespace) {
+    try {
+      client.deleteDataset(toDatasetReference(namespace));
+      // We don't delete the data folder for safety, which aligns with Hive Metastore's default
+      // behavior.
+      // We can support database or catalog level config controlling file deletion in the future.
+      return true;
+    } catch (NoSuchNamespaceException e) {
+      return false;
+    }
+  }
+
+  @Override
+  public boolean setProperties(Namespace namespace, Map<String, String> properties) {
+    Dataset dataset = client.getDataset(toDatasetReference(namespace));
+
+    ExternalCatalogDatasetOptions existingOptions = dataset.getExternalCatalogDatasetOptions();
+    Map<String, String> existingParameters =
+        existingOptions != null ? existingOptions.getParameters() : null;
+
+    Map<String, String> newParameters = Maps.newHashMap();
+    if (existingParameters != null) {
+      newParameters.putAll(existingParameters);
+    }
+    newParameters.putAll(properties);
+
+    if (Objects.equals(existingParameters, newParameters)) {
+      // No change in parameters detected
+      return false;
+    }
+
+    client.setDatasetParameters(toDatasetReference(namespace), properties);
+    return true;
+  }
+
+  @Override
+  public boolean removeProperties(Namespace namespace, Set<String> properties) {
+    client.removeDatasetParameters(toDatasetReference(namespace), properties);
+    return true;
+  }
+
+  @SuppressWarnings("FormatStringAnnotation")
+  @Override
+  public Map<String, String> loadNamespaceMetadata(Namespace namespace) {
+    try {
+      return getMetadata(client.getDataset(toDatasetReference(namespace)));
+    } catch (IllegalArgumentException e) {
+      throw new NoSuchNamespaceException(e.getMessage());
+    }
+  }
+
+  @Override
+  public String name() {
+    return catalogName;
+  }
+
+  @Override
+  protected Map<String, String> properties() {
+    return catalogProperties == null ? ImmutableMap.of() : catalogProperties;
+  }
+
+  @Override
+  public void setConf(Configuration conf) {
+    this.conf = new Configuration(conf);
+  }
+
+  @Override
+  public Configuration getConf() {
+    return this.conf;
+  }
+
+  private static Namespace getNamespace(Datasets dataset) {
+    return Namespace.of(dataset.getDatasetReference().getDatasetId());
+  }
+
+  private DatasetReference toDatasetReference(Namespace namespace) {
+    validateNamespace(namespace);
+    return new DatasetReference().setProjectId(projectId).setDatasetId(namespace.level(0));
+  }
+
+  private TableReference toTableReference(TableIdentifier tableIdentifier) {
+    DatasetReference datasetReference = toDatasetReference(tableIdentifier.namespace());
+    return new TableReference()
+        .setProjectId(datasetReference.getProjectId())
+        .setDatasetId(datasetReference.getDatasetId())
+        .setTableId(tableIdentifier.name());
+  }
+
+  private static Map<String, String> getMetadata(Dataset dataset) {
+    ExternalCatalogDatasetOptions options = dataset.getExternalCatalogDatasetOptions();
+    Map<String, String> metadata = Maps.newHashMap();
+    if (options != null) {
+      if (options.getParameters() != null) {
+        metadata.putAll(options.getParameters());
+      }
+      if (!Strings.isNullOrEmpty(options.getDefaultStorageLocationUri())) {
+        metadata.put("location", options.getDefaultStorageLocationUri());

Review Comment:
   is this the same as `gcp-location`?



-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org

For queries about this service, please contact Infrastructure at:
users@infra.apache.org


---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For additional commands, e-mail: issues-help@iceberg.apache.org


Re: [PR] Catalog: Add BigQuery Metastore Catalog Support [iceberg]

Posted by "nastra (via GitHub)" <gi...@apache.org>.
nastra commented on code in PR #12808:
URL: https://github.com/apache/iceberg/pull/12808#discussion_r2055644245


##########
bigquery/src/main/java/org/apache/iceberg/gcp/bigquery/BigQueryMetastoreCatalog.java:
##########
@@ -0,0 +1,400 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.iceberg.gcp.bigquery;
+
+import com.google.api.services.bigquery.model.Dataset;
+import com.google.api.services.bigquery.model.DatasetList.Datasets;
+import com.google.api.services.bigquery.model.DatasetReference;
+import com.google.api.services.bigquery.model.ExternalCatalogDatasetOptions;
+import com.google.api.services.bigquery.model.TableReference;
+import com.google.cloud.ServiceOptions;
+import com.google.cloud.bigquery.BigQueryOptions;
+import java.io.IOException;
+import java.security.GeneralSecurityException;
+import java.util.List;
+import java.util.Locale;
+import java.util.Map;
+import java.util.Objects;
+import java.util.Set;
+import org.apache.hadoop.conf.Configurable;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.iceberg.BaseMetastoreCatalog;
+import org.apache.iceberg.CatalogProperties;
+import org.apache.iceberg.CatalogUtil;
+import org.apache.iceberg.TableMetadata;
+import org.apache.iceberg.TableOperations;
+import org.apache.iceberg.catalog.Namespace;
+import org.apache.iceberg.catalog.SupportsNamespaces;
+import org.apache.iceberg.catalog.TableIdentifier;
+import org.apache.iceberg.exceptions.AlreadyExistsException;
+import org.apache.iceberg.exceptions.NoSuchNamespaceException;
+import org.apache.iceberg.exceptions.NoSuchTableException;
+import org.apache.iceberg.exceptions.ServiceFailureException;
+import org.apache.iceberg.exceptions.ValidationException;
+import org.apache.iceberg.io.FileIO;
+import org.apache.iceberg.relocated.com.google.common.annotations.VisibleForTesting;
+import org.apache.iceberg.relocated.com.google.common.base.Preconditions;
+import org.apache.iceberg.relocated.com.google.common.base.Strings;
+import org.apache.iceberg.relocated.com.google.common.collect.ImmutableList;
+import org.apache.iceberg.relocated.com.google.common.collect.ImmutableMap;
+import org.apache.iceberg.relocated.com.google.common.collect.Maps;
+import org.apache.iceberg.util.LocationUtil;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/** Iceberg Bigquery Metastore Catalog implementation. */
+public final class BigQueryMetastoreCatalog extends BaseMetastoreCatalog
+    implements SupportsNamespaces, Configurable {
+
+  // User provided properties.
+  public static final String PROJECT_ID = "gcp-project";
+  public static final String GCP_LOCATION = "gcp-location";
+  public static final String FILTER_UNSUPPORTED_TABLES = "filter-unsupported-tables";
+  public static final String TESTING_ENABLED = "testing-enabled";
+
+  public static final String HIVE_METASTORE_WAREHOUSE_DIR = "hive.metastore.warehouse.dir";
+
+  private static final Logger LOG = LoggerFactory.getLogger(BigQueryMetastoreCatalog.class);
+
+  private static final String DEFAULT_GCP_LOCATION = "us";
+
+  private String catalogName;
+  private Map<String, String> catalogProperties;
+  private FileIO fileIO;
+  private Configuration conf;
+  private String projectId;
+  private String location;
+  private BigQueryMetastoreClient client;
+  private boolean filterUnsupportedTables;
+
+  // Must have a no-arg constructor to be dynamically loaded
+  // initialize(String name, Map<String, String> properties) will be called to complete
+  // initialization
+  public BigQueryMetastoreCatalog() {}
+
+  @Override
+  public void initialize(String inputName, Map<String, String> properties) {
+    Preconditions.checkArgument(
+        properties.containsKey(PROJECT_ID), "GCP project must be specified");
+
+    projectId = properties.get(PROJECT_ID);
+    location = properties.getOrDefault(GCP_LOCATION, DEFAULT_GCP_LOCATION);
+    boolean testingEnabled =
+        Boolean.parseBoolean(properties.getOrDefault(TESTING_ENABLED, "false"));
+
+    BigQueryOptions options =
+        BigQueryOptions.newBuilder()
+            .setProjectId(projectId)
+            .setLocation(location)
+            .setRetrySettings(ServiceOptions.getDefaultRetrySettings())
+            .build();
+
+    try {
+      if (testingEnabled) {
+        client = new FakeBigQueryMetastoreClient(options);
+
+      } else {
+        client = new BigQueryMetastoreClientImpl(options);
+      }
+    } catch (IOException e) {
+      throw new ServiceFailureException(e, "Creating BigQuery client failed");
+    } catch (GeneralSecurityException e) {
+      throw new ValidationException(e, "Creating BigQuery client failed due to a security issue");
+    }
+    initialize(inputName, properties, projectId, location, client);
+  }
+
+  @VisibleForTesting
+  void initialize(
+      String inputName,
+      Map<String, String> properties,
+      String initialProjectId,
+      String initialLocation,
+      BigQueryMetastoreClient bigQueryMetaStoreClient) {
+    Preconditions.checkArgument(bigQueryMetaStoreClient != null, "BigQuery client can not be null");
+    this.catalogName = inputName;
+    this.catalogProperties = ImmutableMap.copyOf(properties);
+    this.projectId = initialProjectId;
+    this.location = initialLocation;
+    this.client = bigQueryMetaStoreClient;
+
+    if (this.conf == null) {
+      LOG.warn("No configuration was set, using the default environment Configuration");
+      this.conf = new Configuration();
+    }
+
+    LOG.info("Using BigQuery Metastore Iceberg Catalog: {}", inputName);
+
+    if (properties.containsKey(CatalogProperties.WAREHOUSE_LOCATION)) {
+      // Iceberg always removes trailing slash to avoid paths like "<folder>//data" in file systems
+      // like s3.
+      this.conf.set(
+          HIVE_METASTORE_WAREHOUSE_DIR,
+          LocationUtil.stripTrailingSlash(properties.get(CatalogProperties.WAREHOUSE_LOCATION)));
+    }
+
+    String fileIoImpl =
+        properties.getOrDefault(
+            CatalogProperties.FILE_IO_IMPL, "org.apache.iceberg.hadoop.HadoopFileIO");
+    this.fileIO = CatalogUtil.loadFileIO(fileIoImpl, properties, conf);
+
+    this.filterUnsupportedTables =
+        Boolean.parseBoolean(properties.getOrDefault(FILTER_UNSUPPORTED_TABLES, "false"));
+  }
+
+  @Override
+  protected TableOperations newTableOps(TableIdentifier identifier) {
+    return new BigQueryTableOperations(
+        client,
+        fileIO,
+        projectId,
+        // Sometimes extensions have the namespace contain the table name too, so we are forced to
+        // allow invalid namespace and just take the first part here like other catalog
+        // implementations do.
+        identifier.namespace().level(0),
+        identifier.name(),
+        conf);
+  }
+
+  @Override
+  protected String defaultWarehouseLocation(TableIdentifier identifier) {
+    String locationUri = null;
+    DatasetReference datasetReference = toDatasetReference(identifier.namespace());
+    Dataset dataset = client.getDataset(datasetReference);
+    if (dataset != null && dataset.getExternalCatalogDatasetOptions() != null) {
+      locationUri = dataset.getExternalCatalogDatasetOptions().getDefaultStorageLocationUri();
+    }
+    return String.format(
+        "%s/%s",
+        Strings.isNullOrEmpty(locationUri)
+            ? datasetReference.getDatasetId()
+            : LocationUtil.stripTrailingSlash(locationUri),
+        identifier.name());
+  }
+
+  @Override
+  public List<TableIdentifier> listTables(Namespace namespace) {
+    validateNamespace(namespace);
+
+    return client.listTables(toDatasetReference(namespace), filterUnsupportedTables).stream()
+        .map(
+            table -> TableIdentifier.of(namespace.level(0), table.getTableReference().getTableId()))
+        .collect(ImmutableList.toImmutableList());
+  }
+
+  @Override
+  public boolean dropTable(TableIdentifier identifier, boolean purge) {
+    try {
+      TableOperations ops = newTableOps(identifier);
+      TableMetadata lastMetadata = ops.current();
+
+      client.deleteTable(toTableReference(identifier));
+
+      if (purge && lastMetadata != null) {
+        CatalogUtil.dropTableData(ops.io(), lastMetadata);
+      }
+    } catch (NoSuchTableException e) { // Not catching a NoSuchIcebergTableException on purpose
+      return false; // The documentation says just return false in this case
+    }
+    return true;
+  }
+
+  @Override
+  public void renameTable(TableIdentifier from, TableIdentifier to) {
+    validateNamespace(from.namespace());
+    validateNamespace(to.namespace());
+
+    try {
+      client.getDataset(toDatasetReference(to.namespace()));
+      client.getDataset(toDatasetReference(from.namespace()));
+    } catch (NoSuchNamespaceException e) {
+      throw e;
+    }
+
+    TableReference fromTableRef = toTableReference(from);
+    TableReference toTableRef = toTableReference(to);
+
+    // Check if the source table exists
+    try {
+      client.getTable(fromTableRef);
+    } catch (NoSuchTableException e) {
+      throw new NoSuchTableException("Table does not exist: %s", from.name());
+    }
+
+    // Check if the destination table already exists
+    try {
+      client.getTable(toTableRef);
+      // If getTable succeeds, the destination table exists
+      throw new AlreadyExistsException("Table already exists: %s", to.name());
+    } catch (NoSuchTableException e) {
+      // Destination table does not exist, proceed with throwing exception
+    }
+
+    if (!from.namespace().equals(to.namespace())) {
+      throw new ValidationException("New table name must be in the same namespace");
+    }
+
+    // TODO(b/354981675): Enable once supported by the API.

Review Comment:
   this TODO still refers to an internal ticketing system



-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org

For queries about this service, please contact Infrastructure at:
users@infra.apache.org


---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For additional commands, e-mail: issues-help@iceberg.apache.org


Re: [PR] Catalog: Add BigQuery Metastore Catalog Support [iceberg]

Posted by "nastra (via GitHub)" <gi...@apache.org>.
nastra commented on code in PR #12808:
URL: https://github.com/apache/iceberg/pull/12808#discussion_r2059739213


##########
bigquery/src/test/java/org/apache/iceberg/gcp/bigquery/TestBigQueryCatalog.java:
##########
@@ -0,0 +1,673 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.iceberg.gcp.bigquery;
+
+import static org.apache.iceberg.CatalogUtil.ICEBERG_CATALOG_TYPE;
+import static org.apache.iceberg.CatalogUtil.ICEBERG_CATALOG_TYPE_BIGQUERY;
+import static org.apache.iceberg.gcp.bigquery.BigQueryMetastoreCatalog.PROJECT_ID;
+import static org.apache.iceberg.gcp.bigquery.BigQueryMetastoreCatalog.TESTING_ENABLED;
+import static org.apache.iceberg.types.Types.NestedField.optional;
+import static org.assertj.core.api.Assertions.assertThat;
+import static org.assertj.core.api.Assertions.assertThatThrownBy;
+
+import java.io.File;
+import java.util.List;
+import java.util.Map;
+import org.apache.iceberg.CatalogProperties;
+import org.apache.iceberg.CatalogUtil;
+import org.apache.iceberg.PartitionSpec;
+import org.apache.iceberg.Schema;
+import org.apache.iceberg.SortOrder;
+import org.apache.iceberg.Table;
+import org.apache.iceberg.TableUtil;
+import org.apache.iceberg.Transaction;
+import org.apache.iceberg.catalog.CatalogTests;
+import org.apache.iceberg.catalog.Namespace;
+import org.apache.iceberg.catalog.TableIdentifier;
+import org.apache.iceberg.exceptions.NoSuchNamespaceException;
+import org.apache.iceberg.exceptions.NoSuchTableException;
+import org.apache.iceberg.expressions.Expressions;
+import org.apache.iceberg.relocated.com.google.common.collect.ImmutableMap;
+import org.apache.iceberg.types.Types;
+import org.junit.jupiter.api.AfterEach;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Disabled;
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.io.TempDir;
+import org.junit.jupiter.params.ParameterizedTest;
+import org.junit.jupiter.params.provider.ValueSource;
+
+public class TestBigQueryCatalog extends CatalogTests<BigQueryMetastoreCatalog> {
+
+  private static final String GCP_PROJECT = "project-id";

Review Comment:
   this can probably also be inlined, since it's only used at a single place



-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org

For queries about this service, please contact Infrastructure at:
users@infra.apache.org


---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For additional commands, e-mail: issues-help@iceberg.apache.org


Re: [PR] Catalog: Add BigQuery Metastore Catalog Support [iceberg]

Posted by "talatuyarer (via GitHub)" <gi...@apache.org>.
talatuyarer commented on code in PR #12808:
URL: https://github.com/apache/iceberg/pull/12808#discussion_r2059827099


##########
bigquery/src/main/java/org/apache/iceberg/gcp/bigquery/BigQueryMetastoreCatalog.java:
##########
@@ -0,0 +1,400 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.iceberg.gcp.bigquery;
+
+import com.google.api.services.bigquery.model.Dataset;
+import com.google.api.services.bigquery.model.DatasetList.Datasets;
+import com.google.api.services.bigquery.model.DatasetReference;
+import com.google.api.services.bigquery.model.ExternalCatalogDatasetOptions;
+import com.google.api.services.bigquery.model.TableReference;
+import com.google.cloud.ServiceOptions;
+import com.google.cloud.bigquery.BigQueryOptions;
+import java.io.IOException;
+import java.security.GeneralSecurityException;
+import java.util.List;
+import java.util.Locale;
+import java.util.Map;
+import java.util.Objects;
+import java.util.Set;
+import org.apache.hadoop.conf.Configurable;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.iceberg.BaseMetastoreCatalog;
+import org.apache.iceberg.CatalogProperties;
+import org.apache.iceberg.CatalogUtil;
+import org.apache.iceberg.TableMetadata;
+import org.apache.iceberg.TableOperations;
+import org.apache.iceberg.catalog.Namespace;
+import org.apache.iceberg.catalog.SupportsNamespaces;
+import org.apache.iceberg.catalog.TableIdentifier;
+import org.apache.iceberg.exceptions.AlreadyExistsException;
+import org.apache.iceberg.exceptions.NoSuchNamespaceException;
+import org.apache.iceberg.exceptions.NoSuchTableException;
+import org.apache.iceberg.exceptions.ServiceFailureException;
+import org.apache.iceberg.exceptions.ValidationException;
+import org.apache.iceberg.io.FileIO;
+import org.apache.iceberg.relocated.com.google.common.annotations.VisibleForTesting;
+import org.apache.iceberg.relocated.com.google.common.base.Preconditions;
+import org.apache.iceberg.relocated.com.google.common.base.Strings;
+import org.apache.iceberg.relocated.com.google.common.collect.ImmutableList;
+import org.apache.iceberg.relocated.com.google.common.collect.ImmutableMap;
+import org.apache.iceberg.relocated.com.google.common.collect.Maps;
+import org.apache.iceberg.util.LocationUtil;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/** Iceberg Bigquery Metastore Catalog implementation. */
+public final class BigQueryMetastoreCatalog extends BaseMetastoreCatalog
+    implements SupportsNamespaces, Configurable {
+
+  // User provided properties.
+  public static final String PROJECT_ID = "gcp-project";
+  public static final String GCP_LOCATION = "gcp-location";
+  public static final String FILTER_UNSUPPORTED_TABLES = "filter-unsupported-tables";
+  public static final String TESTING_ENABLED = "testing-enabled";
+
+  public static final String HIVE_METASTORE_WAREHOUSE_DIR = "hive.metastore.warehouse.dir";
+
+  private static final Logger LOG = LoggerFactory.getLogger(BigQueryMetastoreCatalog.class);
+
+  private static final String DEFAULT_GCP_LOCATION = "us";
+
+  private String catalogName;
+  private Map<String, String> catalogProperties;
+  private FileIO fileIO;
+  private Configuration conf;
+  private String projectId;
+  private String location;
+  private BigQueryMetastoreClient client;
+  private boolean filterUnsupportedTables;
+
+  // Must have a no-arg constructor to be dynamically loaded
+  // initialize(String name, Map<String, String> properties) will be called to complete
+  // initialization
+  public BigQueryMetastoreCatalog() {}
+
+  @Override
+  public void initialize(String inputName, Map<String, String> properties) {
+    Preconditions.checkArgument(
+        properties.containsKey(PROJECT_ID), "GCP project must be specified");
+
+    projectId = properties.get(PROJECT_ID);
+    location = properties.getOrDefault(GCP_LOCATION, DEFAULT_GCP_LOCATION);
+    boolean testingEnabled =
+        Boolean.parseBoolean(properties.getOrDefault(TESTING_ENABLED, "false"));
+
+    BigQueryOptions options =
+        BigQueryOptions.newBuilder()
+            .setProjectId(projectId)
+            .setLocation(location)
+            .setRetrySettings(ServiceOptions.getDefaultRetrySettings())
+            .build();
+
+    try {
+      if (testingEnabled) {
+        client = new FakeBigQueryMetastoreClient(options);
+
+      } else {
+        client = new BigQueryMetastoreClientImpl(options);
+      }
+    } catch (IOException e) {
+      throw new ServiceFailureException(e, "Creating BigQuery client failed");
+    } catch (GeneralSecurityException e) {
+      throw new ValidationException(e, "Creating BigQuery client failed due to a security issue");
+    }
+    initialize(inputName, properties, projectId, location, client);
+  }
+
+  @VisibleForTesting
+  void initialize(
+      String inputName,
+      Map<String, String> properties,
+      String initialProjectId,
+      String initialLocation,
+      BigQueryMetastoreClient bigQueryMetaStoreClient) {
+    Preconditions.checkArgument(bigQueryMetaStoreClient != null, "BigQuery client can not be null");
+    this.catalogName = inputName;
+    this.catalogProperties = ImmutableMap.copyOf(properties);
+    this.projectId = initialProjectId;
+    this.location = initialLocation;
+    this.client = bigQueryMetaStoreClient;
+
+    if (this.conf == null) {
+      LOG.warn("No configuration was set, using the default environment Configuration");
+      this.conf = new Configuration();
+    }
+
+    LOG.info("Using BigQuery Metastore Iceberg Catalog: {}", inputName);
+
+    if (properties.containsKey(CatalogProperties.WAREHOUSE_LOCATION)) {
+      // Iceberg always removes trailing slash to avoid paths like "<folder>//data" in file systems
+      // like s3.
+      this.conf.set(
+          HIVE_METASTORE_WAREHOUSE_DIR,
+          LocationUtil.stripTrailingSlash(properties.get(CatalogProperties.WAREHOUSE_LOCATION)));
+    }
+
+    String fileIoImpl =
+        properties.getOrDefault(
+            CatalogProperties.FILE_IO_IMPL, "org.apache.iceberg.hadoop.HadoopFileIO");
+    this.fileIO = CatalogUtil.loadFileIO(fileIoImpl, properties, conf);
+
+    this.filterUnsupportedTables =
+        Boolean.parseBoolean(properties.getOrDefault(FILTER_UNSUPPORTED_TABLES, "false"));
+  }
+
+  @Override
+  protected TableOperations newTableOps(TableIdentifier identifier) {
+    return new BigQueryTableOperations(
+        client,
+        fileIO,
+        projectId,
+        // Sometimes extensions have the namespace contain the table name too, so we are forced to
+        // allow invalid namespace and just take the first part here like other catalog
+        // implementations do.
+        identifier.namespace().level(0),
+        identifier.name(),
+        conf);
+  }
+
+  @Override
+  protected String defaultWarehouseLocation(TableIdentifier identifier) {
+    String locationUri = null;
+    DatasetReference datasetReference = toDatasetReference(identifier.namespace());
+    Dataset dataset = client.getDataset(datasetReference);
+    if (dataset != null && dataset.getExternalCatalogDatasetOptions() != null) {
+      locationUri = dataset.getExternalCatalogDatasetOptions().getDefaultStorageLocationUri();
+    }
+    return String.format(
+        "%s/%s",
+        Strings.isNullOrEmpty(locationUri)
+            ? datasetReference.getDatasetId()
+            : LocationUtil.stripTrailingSlash(locationUri),
+        identifier.name());
+  }
+
+  @Override
+  public List<TableIdentifier> listTables(Namespace namespace) {
+    validateNamespace(namespace);
+
+    return client.listTables(toDatasetReference(namespace), filterUnsupportedTables).stream()
+        .map(
+            table -> TableIdentifier.of(namespace.level(0), table.getTableReference().getTableId()))
+        .collect(ImmutableList.toImmutableList());
+  }
+
+  @Override
+  public boolean dropTable(TableIdentifier identifier, boolean purge) {
+    try {
+      TableOperations ops = newTableOps(identifier);
+      TableMetadata lastMetadata = ops.current();
+
+      client.deleteTable(toTableReference(identifier));
+
+      if (purge && lastMetadata != null) {
+        CatalogUtil.dropTableData(ops.io(), lastMetadata);
+      }
+    } catch (NoSuchTableException e) { // Not catching a NoSuchIcebergTableException on purpose
+      return false; // The documentation says just return false in this case
+    }
+    return true;
+  }
+
+  @Override
+  public void renameTable(TableIdentifier from, TableIdentifier to) {
+    validateNamespace(from.namespace());
+    validateNamespace(to.namespace());
+
+    try {
+      client.getDataset(toDatasetReference(to.namespace()));
+      client.getDataset(toDatasetReference(from.namespace()));
+    } catch (NoSuchNamespaceException e) {
+      throw e;
+    }
+
+    TableReference fromTableRef = toTableReference(from);
+    TableReference toTableRef = toTableReference(to);
+
+    // Check if the source table exists
+    try {
+      client.getTable(fromTableRef);
+    } catch (NoSuchTableException e) {
+      throw new NoSuchTableException("Table does not exist: %s", from.name());
+    }
+
+    // Check if the destination table already exists
+    try {
+      client.getTable(toTableRef);
+      // If getTable succeeds, the destination table exists
+      throw new AlreadyExistsException("Table already exists: %s", to.name());
+    } catch (NoSuchTableException e) {
+      // Destination table does not exist, proceed with throwing exception
+    }
+
+    if (!from.namespace().equals(to.namespace())) {
+      throw new ValidationException("New table name must be in the same namespace");
+    }
+
+    // TODO(b/354981675): Enable once supported by the API.
+    throw new UnsupportedOperationException("Table rename operation is unsupported.");
+  }
+
+  @Override
+  public void createNamespace(Namespace namespace, Map<String, String> metadata) {
+    Dataset builder = new Dataset();
+    DatasetReference datasetReference = toDatasetReference(namespace);
+    builder.setLocation(this.location);
+    builder.setDatasetReference(datasetReference);
+    builder.setExternalCatalogDatasetOptions(
+        BigQueryMetastoreUtils.createExternalCatalogDatasetOptions(
+            datasetReference.getDatasetId(), metadata));
+
+    client.createDataset(builder);
+  }
+
+  /**
+   * Since this catalog only supports one-level namespaces, it always returns an empty list unless
+   * passed an empty namespace to list all namespaces within the catalog.
+   */
+  @Override
+  public List<Namespace> listNamespaces(Namespace namespace) {
+    if (namespace.levels().length != 0) {
+      // BQMS does not support namespaces under database or tables, returns empty.
+      // It is called when dropping a namespace to make sure it's empty (listTables is called as
+      // well), returns empty to unblock deletion.
+      return ImmutableList.of();
+    }
+    return client.listDatasets(projectId).stream()
+        .map(BigQueryMetastoreCatalog::getNamespace)
+        .collect(ImmutableList.toImmutableList());
+  }
+
+  @Override
+  public boolean dropNamespace(Namespace namespace) {
+    try {
+      client.deleteDataset(toDatasetReference(namespace));
+      // We don't delete the data folder for safety, which aligns with Hive Metastore's default
+      // behavior.
+      // We can support database or catalog level config controlling file deletion in the future.
+      return true;
+    } catch (NoSuchNamespaceException e) {
+      return false;
+    }
+  }
+
+  @Override
+  public boolean setProperties(Namespace namespace, Map<String, String> properties) {
+    Dataset dataset = client.getDataset(toDatasetReference(namespace));
+
+    ExternalCatalogDatasetOptions existingOptions = dataset.getExternalCatalogDatasetOptions();
+    Map<String, String> existingParameters =
+        existingOptions != null ? existingOptions.getParameters() : null;
+
+    Map<String, String> newParameters = Maps.newHashMap();
+    if (existingParameters != null) {
+      newParameters.putAll(existingParameters);
+    }
+    newParameters.putAll(properties);
+
+    if (Objects.equals(existingParameters, newParameters)) {
+      // No change in parameters detected
+      return false;
+    }
+
+    client.setDatasetParameters(toDatasetReference(namespace), properties);
+    return true;
+  }
+
+  @Override
+  public boolean removeProperties(Namespace namespace, Set<String> properties) {
+    client.removeDatasetParameters(toDatasetReference(namespace), properties);
+    return true;
+  }
+
+  @SuppressWarnings("FormatStringAnnotation")
+  @Override
+  public Map<String, String> loadNamespaceMetadata(Namespace namespace) {
+    try {
+      return getMetadata(client.getDataset(toDatasetReference(namespace)));
+    } catch (IllegalArgumentException e) {

Review Comment:
   If Namespace string does not support from BigQuery it will throw IAE. It is NSNE for user because they can not create in first place. 



##########
bigquery/src/main/java/org/apache/iceberg/gcp/bigquery/BigQueryMetastoreCatalog.java:
##########
@@ -0,0 +1,400 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.iceberg.gcp.bigquery;
+
+import com.google.api.services.bigquery.model.Dataset;
+import com.google.api.services.bigquery.model.DatasetList.Datasets;
+import com.google.api.services.bigquery.model.DatasetReference;
+import com.google.api.services.bigquery.model.ExternalCatalogDatasetOptions;
+import com.google.api.services.bigquery.model.TableReference;
+import com.google.cloud.ServiceOptions;
+import com.google.cloud.bigquery.BigQueryOptions;
+import java.io.IOException;
+import java.security.GeneralSecurityException;
+import java.util.List;
+import java.util.Locale;
+import java.util.Map;
+import java.util.Objects;
+import java.util.Set;
+import org.apache.hadoop.conf.Configurable;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.iceberg.BaseMetastoreCatalog;
+import org.apache.iceberg.CatalogProperties;
+import org.apache.iceberg.CatalogUtil;
+import org.apache.iceberg.TableMetadata;
+import org.apache.iceberg.TableOperations;
+import org.apache.iceberg.catalog.Namespace;
+import org.apache.iceberg.catalog.SupportsNamespaces;
+import org.apache.iceberg.catalog.TableIdentifier;
+import org.apache.iceberg.exceptions.AlreadyExistsException;
+import org.apache.iceberg.exceptions.NoSuchNamespaceException;
+import org.apache.iceberg.exceptions.NoSuchTableException;
+import org.apache.iceberg.exceptions.ServiceFailureException;
+import org.apache.iceberg.exceptions.ValidationException;
+import org.apache.iceberg.io.FileIO;
+import org.apache.iceberg.relocated.com.google.common.annotations.VisibleForTesting;
+import org.apache.iceberg.relocated.com.google.common.base.Preconditions;
+import org.apache.iceberg.relocated.com.google.common.base.Strings;
+import org.apache.iceberg.relocated.com.google.common.collect.ImmutableList;
+import org.apache.iceberg.relocated.com.google.common.collect.ImmutableMap;
+import org.apache.iceberg.relocated.com.google.common.collect.Maps;
+import org.apache.iceberg.util.LocationUtil;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/** Iceberg Bigquery Metastore Catalog implementation. */
+public final class BigQueryMetastoreCatalog extends BaseMetastoreCatalog
+    implements SupportsNamespaces, Configurable {
+
+  // User provided properties.
+  public static final String PROJECT_ID = "gcp-project";
+  public static final String GCP_LOCATION = "gcp-location";
+  public static final String FILTER_UNSUPPORTED_TABLES = "filter-unsupported-tables";
+  public static final String TESTING_ENABLED = "testing-enabled";
+
+  public static final String HIVE_METASTORE_WAREHOUSE_DIR = "hive.metastore.warehouse.dir";
+
+  private static final Logger LOG = LoggerFactory.getLogger(BigQueryMetastoreCatalog.class);
+
+  private static final String DEFAULT_GCP_LOCATION = "us";
+
+  private String catalogName;
+  private Map<String, String> catalogProperties;
+  private FileIO fileIO;
+  private Configuration conf;
+  private String projectId;
+  private String location;
+  private BigQueryMetastoreClient client;
+  private boolean filterUnsupportedTables;
+
+  // Must have a no-arg constructor to be dynamically loaded
+  // initialize(String name, Map<String, String> properties) will be called to complete
+  // initialization
+  public BigQueryMetastoreCatalog() {}
+
+  @Override
+  public void initialize(String inputName, Map<String, String> properties) {
+    Preconditions.checkArgument(
+        properties.containsKey(PROJECT_ID), "GCP project must be specified");
+
+    projectId = properties.get(PROJECT_ID);
+    location = properties.getOrDefault(GCP_LOCATION, DEFAULT_GCP_LOCATION);
+    boolean testingEnabled =
+        Boolean.parseBoolean(properties.getOrDefault(TESTING_ENABLED, "false"));
+
+    BigQueryOptions options =
+        BigQueryOptions.newBuilder()
+            .setProjectId(projectId)
+            .setLocation(location)
+            .setRetrySettings(ServiceOptions.getDefaultRetrySettings())
+            .build();
+
+    try {
+      if (testingEnabled) {
+        client = new FakeBigQueryMetastoreClient(options);
+
+      } else {
+        client = new BigQueryMetastoreClientImpl(options);
+      }
+    } catch (IOException e) {
+      throw new ServiceFailureException(e, "Creating BigQuery client failed");
+    } catch (GeneralSecurityException e) {
+      throw new ValidationException(e, "Creating BigQuery client failed due to a security issue");
+    }
+    initialize(inputName, properties, projectId, location, client);
+  }
+
+  @VisibleForTesting
+  void initialize(
+      String inputName,
+      Map<String, String> properties,
+      String initialProjectId,
+      String initialLocation,
+      BigQueryMetastoreClient bigQueryMetaStoreClient) {
+    Preconditions.checkArgument(bigQueryMetaStoreClient != null, "BigQuery client can not be null");
+    this.catalogName = inputName;
+    this.catalogProperties = ImmutableMap.copyOf(properties);
+    this.projectId = initialProjectId;
+    this.location = initialLocation;
+    this.client = bigQueryMetaStoreClient;
+
+    if (this.conf == null) {
+      LOG.warn("No configuration was set, using the default environment Configuration");
+      this.conf = new Configuration();
+    }
+
+    LOG.info("Using BigQuery Metastore Iceberg Catalog: {}", inputName);
+
+    if (properties.containsKey(CatalogProperties.WAREHOUSE_LOCATION)) {
+      // Iceberg always removes trailing slash to avoid paths like "<folder>//data" in file systems
+      // like s3.
+      this.conf.set(
+          HIVE_METASTORE_WAREHOUSE_DIR,
+          LocationUtil.stripTrailingSlash(properties.get(CatalogProperties.WAREHOUSE_LOCATION)));
+    }
+
+    String fileIoImpl =
+        properties.getOrDefault(
+            CatalogProperties.FILE_IO_IMPL, "org.apache.iceberg.hadoop.HadoopFileIO");
+    this.fileIO = CatalogUtil.loadFileIO(fileIoImpl, properties, conf);
+
+    this.filterUnsupportedTables =
+        Boolean.parseBoolean(properties.getOrDefault(FILTER_UNSUPPORTED_TABLES, "false"));
+  }
+
+  @Override
+  protected TableOperations newTableOps(TableIdentifier identifier) {
+    return new BigQueryTableOperations(
+        client,
+        fileIO,
+        projectId,
+        // Sometimes extensions have the namespace contain the table name too, so we are forced to
+        // allow invalid namespace and just take the first part here like other catalog
+        // implementations do.
+        identifier.namespace().level(0),
+        identifier.name(),
+        conf);
+  }
+
+  @Override
+  protected String defaultWarehouseLocation(TableIdentifier identifier) {
+    String locationUri = null;
+    DatasetReference datasetReference = toDatasetReference(identifier.namespace());
+    Dataset dataset = client.getDataset(datasetReference);
+    if (dataset != null && dataset.getExternalCatalogDatasetOptions() != null) {
+      locationUri = dataset.getExternalCatalogDatasetOptions().getDefaultStorageLocationUri();
+    }
+    return String.format(
+        "%s/%s",
+        Strings.isNullOrEmpty(locationUri)
+            ? datasetReference.getDatasetId()
+            : LocationUtil.stripTrailingSlash(locationUri),
+        identifier.name());
+  }
+
+  @Override
+  public List<TableIdentifier> listTables(Namespace namespace) {
+    validateNamespace(namespace);
+
+    return client.listTables(toDatasetReference(namespace), filterUnsupportedTables).stream()
+        .map(
+            table -> TableIdentifier.of(namespace.level(0), table.getTableReference().getTableId()))
+        .collect(ImmutableList.toImmutableList());
+  }
+
+  @Override
+  public boolean dropTable(TableIdentifier identifier, boolean purge) {
+    try {
+      TableOperations ops = newTableOps(identifier);
+      TableMetadata lastMetadata = ops.current();
+
+      client.deleteTable(toTableReference(identifier));
+
+      if (purge && lastMetadata != null) {
+        CatalogUtil.dropTableData(ops.io(), lastMetadata);
+      }
+    } catch (NoSuchTableException e) { // Not catching a NoSuchIcebergTableException on purpose

Review Comment:
   Done



-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org

For queries about this service, please contact Infrastructure at:
users@infra.apache.org


---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For additional commands, e-mail: issues-help@iceberg.apache.org


Re: [PR] Catalog: Add BigQuery Metastore Catalog Support [iceberg]

Posted by "ebyhr (via GitHub)" <gi...@apache.org>.
ebyhr commented on code in PR #12808:
URL: https://github.com/apache/iceberg/pull/12808#discussion_r2060014802


##########
bigquery/src/test/java/org/apache/iceberg/gcp/bigquery/BigQueryMetastoreTestUtils.java:
##########
@@ -0,0 +1,87 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.iceberg.gcp.bigquery;
+
+import static org.apache.iceberg.types.Types.NestedField.required;
+import static org.assertj.core.api.Assertions.assertThat;
+
+import com.google.api.services.bigquery.model.ExternalCatalogTableOptions;
+import com.google.api.services.bigquery.model.StorageDescriptor;
+import com.google.api.services.bigquery.model.Table;
+import com.google.api.services.bigquery.model.TableReference;
+import java.io.File;
+import java.io.IOException;
+import java.util.Collections;
+import java.util.Optional;
+import org.apache.commons.io.FileUtils;
+import org.apache.commons.io.filefilter.TrueFileFilter;
+import org.apache.iceberg.Schema;
+import org.apache.iceberg.catalog.TableIdentifier;
+import org.apache.iceberg.types.Types;
+
+/** Test utility methods for BigQuery Metastore Iceberg catalog. */
+public final class BigQueryMetastoreTestUtils {
+  private BigQueryMetastoreTestUtils() {}
+
+  public static final String METADATA_LOCATION_PROP = "metadata_location";
+
+  public static Schema getTestSchema() {
+    return new Schema(
+        required(1, "id", Types.IntegerType.get(), "unique ID"),
+        required(2, "data", Types.StringType.get()));
+  }
+
+  public static Table createTestTable(
+      File tempFolder,
+      BigQueryMetastoreCatalog bigQueryMetastoreCatalog,
+      TableReference tableReference)
+      throws IOException {
+    Schema schema = getTestSchema();
+    TableIdentifier tableIdentifier =
+        TableIdentifier.of(tableReference.getDatasetId(), tableReference.getTableId());
+    String tableDir = tempFolder.toPath().resolve(tableReference.getTableId()).toString();
+
+    bigQueryMetastoreCatalog
+        .buildTable(tableIdentifier, schema)
+        .withLocation(tableDir)
+        .createTransaction()
+        .commitTransaction();
+
+    Optional<String> metadataLocation = getIcebergMetadataFilePath(tableDir);
+    assertThat(metadataLocation.isPresent());

Review Comment:
   ↑Reminder. 



-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org

For queries about this service, please contact Infrastructure at:
users@infra.apache.org


---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For additional commands, e-mail: issues-help@iceberg.apache.org


Re: [PR] Catalog: Add BigQuery Metastore Catalog Support [iceberg]

Posted by "nastra (via GitHub)" <gi...@apache.org>.
nastra commented on code in PR #12808:
URL: https://github.com/apache/iceberg/pull/12808#discussion_r2055604154


##########
bigquery/src/main/java/org/apache/iceberg/gcp/bigquery/BigQueryMetastoreCatalog.java:
##########
@@ -0,0 +1,400 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.iceberg.gcp.bigquery;
+
+import com.google.api.services.bigquery.model.Dataset;
+import com.google.api.services.bigquery.model.DatasetList.Datasets;
+import com.google.api.services.bigquery.model.DatasetReference;
+import com.google.api.services.bigquery.model.ExternalCatalogDatasetOptions;
+import com.google.api.services.bigquery.model.TableReference;
+import com.google.cloud.ServiceOptions;
+import com.google.cloud.bigquery.BigQueryOptions;
+import java.io.IOException;
+import java.security.GeneralSecurityException;
+import java.util.List;
+import java.util.Locale;
+import java.util.Map;
+import java.util.Objects;
+import java.util.Set;
+import org.apache.hadoop.conf.Configurable;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.iceberg.BaseMetastoreCatalog;
+import org.apache.iceberg.CatalogProperties;
+import org.apache.iceberg.CatalogUtil;
+import org.apache.iceberg.TableMetadata;
+import org.apache.iceberg.TableOperations;
+import org.apache.iceberg.catalog.Namespace;
+import org.apache.iceberg.catalog.SupportsNamespaces;
+import org.apache.iceberg.catalog.TableIdentifier;
+import org.apache.iceberg.exceptions.AlreadyExistsException;
+import org.apache.iceberg.exceptions.NoSuchNamespaceException;
+import org.apache.iceberg.exceptions.NoSuchTableException;
+import org.apache.iceberg.exceptions.ServiceFailureException;
+import org.apache.iceberg.exceptions.ValidationException;
+import org.apache.iceberg.io.FileIO;
+import org.apache.iceberg.relocated.com.google.common.annotations.VisibleForTesting;
+import org.apache.iceberg.relocated.com.google.common.base.Preconditions;
+import org.apache.iceberg.relocated.com.google.common.base.Strings;
+import org.apache.iceberg.relocated.com.google.common.collect.ImmutableList;
+import org.apache.iceberg.relocated.com.google.common.collect.ImmutableMap;
+import org.apache.iceberg.relocated.com.google.common.collect.Maps;
+import org.apache.iceberg.util.LocationUtil;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/** Iceberg Bigquery Metastore Catalog implementation. */
+public final class BigQueryMetastoreCatalog extends BaseMetastoreCatalog
+    implements SupportsNamespaces, Configurable {
+
+  // User provided properties.
+  public static final String PROJECT_ID = "gcp-project";
+  public static final String GCP_LOCATION = "gcp-location";
+  public static final String FILTER_UNSUPPORTED_TABLES = "filter-unsupported-tables";
+  public static final String TESTING_ENABLED = "testing-enabled";
+
+  public static final String HIVE_METASTORE_WAREHOUSE_DIR = "hive.metastore.warehouse.dir";
+
+  private static final Logger LOG = LoggerFactory.getLogger(BigQueryMetastoreCatalog.class);
+
+  private static final String DEFAULT_GCP_LOCATION = "us";
+
+  private String catalogName;
+  private Map<String, String> catalogProperties;
+  private FileIO fileIO;
+  private Configuration conf;
+  private String projectId;
+  private String location;
+  private BigQueryMetastoreClient client;
+  private boolean filterUnsupportedTables;
+
+  // Must have a no-arg constructor to be dynamically loaded
+  // initialize(String name, Map<String, String> properties) will be called to complete
+  // initialization
+  public BigQueryMetastoreCatalog() {}
+
+  @Override
+  public void initialize(String inputName, Map<String, String> properties) {
+    Preconditions.checkArgument(
+        properties.containsKey(PROJECT_ID), "GCP project must be specified");
+
+    projectId = properties.get(PROJECT_ID);
+    location = properties.getOrDefault(GCP_LOCATION, DEFAULT_GCP_LOCATION);
+    boolean testingEnabled =
+        Boolean.parseBoolean(properties.getOrDefault(TESTING_ENABLED, "false"));
+
+    BigQueryOptions options =
+        BigQueryOptions.newBuilder()
+            .setProjectId(projectId)
+            .setLocation(location)
+            .setRetrySettings(ServiceOptions.getDefaultRetrySettings())
+            .build();
+
+    try {
+      if (testingEnabled) {
+        client = new FakeBigQueryMetastoreClient(options);
+
+      } else {
+        client = new BigQueryMetastoreClientImpl(options);
+      }
+    } catch (IOException e) {
+      throw new ServiceFailureException(e, "Creating BigQuery client failed");
+    } catch (GeneralSecurityException e) {
+      throw new ValidationException(e, "Creating BigQuery client failed due to a security issue");
+    }
+    initialize(inputName, properties, projectId, location, client);
+  }
+
+  @VisibleForTesting
+  void initialize(
+      String inputName,
+      Map<String, String> properties,
+      String initialProjectId,
+      String initialLocation,
+      BigQueryMetastoreClient bigQueryMetaStoreClient) {
+    Preconditions.checkArgument(bigQueryMetaStoreClient != null, "BigQuery client can not be null");
+    this.catalogName = inputName;
+    this.catalogProperties = ImmutableMap.copyOf(properties);
+    this.projectId = initialProjectId;
+    this.location = initialLocation;
+    this.client = bigQueryMetaStoreClient;
+
+    if (this.conf == null) {
+      LOG.warn("No configuration was set, using the default environment Configuration");
+      this.conf = new Configuration();
+    }
+
+    LOG.info("Using BigQuery Metastore Iceberg Catalog: {}", inputName);
+
+    if (properties.containsKey(CatalogProperties.WAREHOUSE_LOCATION)) {
+      // Iceberg always removes trailing slash to avoid paths like "<folder>//data" in file systems
+      // like s3.
+      this.conf.set(
+          HIVE_METASTORE_WAREHOUSE_DIR,
+          LocationUtil.stripTrailingSlash(properties.get(CatalogProperties.WAREHOUSE_LOCATION)));
+    }
+
+    String fileIoImpl =
+        properties.getOrDefault(
+            CatalogProperties.FILE_IO_IMPL, "org.apache.iceberg.hadoop.HadoopFileIO");
+    this.fileIO = CatalogUtil.loadFileIO(fileIoImpl, properties, conf);
+
+    this.filterUnsupportedTables =
+        Boolean.parseBoolean(properties.getOrDefault(FILTER_UNSUPPORTED_TABLES, "false"));
+  }
+
+  @Override
+  protected TableOperations newTableOps(TableIdentifier identifier) {
+    return new BigQueryTableOperations(
+        client,
+        fileIO,
+        projectId,
+        // Sometimes extensions have the namespace contain the table name too, so we are forced to
+        // allow invalid namespace and just take the first part here like other catalog
+        // implementations do.
+        identifier.namespace().level(0),
+        identifier.name(),
+        conf);
+  }
+
+  @Override
+  protected String defaultWarehouseLocation(TableIdentifier identifier) {
+    String locationUri = null;
+    DatasetReference datasetReference = toDatasetReference(identifier.namespace());
+    Dataset dataset = client.getDataset(datasetReference);
+    if (dataset != null && dataset.getExternalCatalogDatasetOptions() != null) {
+      locationUri = dataset.getExternalCatalogDatasetOptions().getDefaultStorageLocationUri();
+    }
+    return String.format(
+        "%s/%s",
+        Strings.isNullOrEmpty(locationUri)
+            ? datasetReference.getDatasetId()
+            : LocationUtil.stripTrailingSlash(locationUri),
+        identifier.name());
+  }
+
+  @Override
+  public List<TableIdentifier> listTables(Namespace namespace) {
+    validateNamespace(namespace);
+
+    return client.listTables(toDatasetReference(namespace), filterUnsupportedTables).stream()
+        .map(
+            table -> TableIdentifier.of(namespace.level(0), table.getTableReference().getTableId()))
+        .collect(ImmutableList.toImmutableList());
+  }
+
+  @Override
+  public boolean dropTable(TableIdentifier identifier, boolean purge) {
+    try {
+      TableOperations ops = newTableOps(identifier);
+      TableMetadata lastMetadata = ops.current();
+
+      client.deleteTable(toTableReference(identifier));
+
+      if (purge && lastMetadata != null) {
+        CatalogUtil.dropTableData(ops.io(), lastMetadata);
+      }
+    } catch (NoSuchTableException e) { // Not catching a NoSuchIcebergTableException on purpose
+      return false; // The documentation says just return false in this case
+    }
+    return true;
+  }
+
+  @Override
+  public void renameTable(TableIdentifier from, TableIdentifier to) {
+    validateNamespace(from.namespace());
+    validateNamespace(to.namespace());
+
+    try {
+      client.getDataset(toDatasetReference(to.namespace()));
+      client.getDataset(toDatasetReference(from.namespace()));
+    } catch (NoSuchNamespaceException e) {
+      throw e;
+    }
+
+    TableReference fromTableRef = toTableReference(from);
+    TableReference toTableRef = toTableReference(to);
+
+    // Check if the source table exists
+    try {
+      client.getTable(fromTableRef);
+    } catch (NoSuchTableException e) {
+      throw new NoSuchTableException("Table does not exist: %s", from.name());
+    }
+
+    // Check if the destination table already exists
+    try {
+      client.getTable(toTableRef);
+      // If getTable succeeds, the destination table exists
+      throw new AlreadyExistsException("Table already exists: %s", to.name());
+    } catch (NoSuchTableException e) {
+      // Destination table does not exist, proceed with throwing exception
+    }
+
+    if (!from.namespace().equals(to.namespace())) {
+      throw new ValidationException("New table name must be in the same namespace");
+    }
+
+    // TODO(b/354981675): Enable once supported by the API.
+    throw new UnsupportedOperationException("Table rename operation is unsupported.");
+  }
+
+  @Override
+  public void createNamespace(Namespace namespace, Map<String, String> metadata) {
+    Dataset builder = new Dataset();
+    DatasetReference datasetReference = toDatasetReference(namespace);
+    builder.setLocation(this.location);
+    builder.setDatasetReference(datasetReference);
+    builder.setExternalCatalogDatasetOptions(
+        BigQueryMetastoreUtils.createExternalCatalogDatasetOptions(
+            datasetReference.getDatasetId(), metadata));
+
+    client.createDataset(builder);
+  }
+
+  /**
+   * Since this catalog only supports one-level namespaces, it always returns an empty list unless
+   * passed an empty namespace to list all namespaces within the catalog.
+   */
+  @Override
+  public List<Namespace> listNamespaces(Namespace namespace) {
+    if (namespace.levels().length != 0) {
+      // BQMS does not support namespaces under database or tables, returns empty.
+      // It is called when dropping a namespace to make sure it's empty (listTables is called as
+      // well), returns empty to unblock deletion.
+      return ImmutableList.of();
+    }
+    return client.listDatasets(projectId).stream()
+        .map(BigQueryMetastoreCatalog::getNamespace)
+        .collect(ImmutableList.toImmutableList());
+  }
+
+  @Override
+  public boolean dropNamespace(Namespace namespace) {
+    try {
+      client.deleteDataset(toDatasetReference(namespace));
+      // We don't delete the data folder for safety, which aligns with Hive Metastore's default
+      // behavior.
+      // We can support database or catalog level config controlling file deletion in the future.
+      return true;
+    } catch (NoSuchNamespaceException e) {
+      return false;
+    }
+  }
+
+  @Override
+  public boolean setProperties(Namespace namespace, Map<String, String> properties) {
+    Dataset dataset = client.getDataset(toDatasetReference(namespace));
+
+    ExternalCatalogDatasetOptions existingOptions = dataset.getExternalCatalogDatasetOptions();
+    Map<String, String> existingParameters =
+        existingOptions != null ? existingOptions.getParameters() : null;
+
+    Map<String, String> newParameters = Maps.newHashMap();
+    if (existingParameters != null) {
+      newParameters.putAll(existingParameters);
+    }
+    newParameters.putAll(properties);
+
+    if (Objects.equals(existingParameters, newParameters)) {
+      // No change in parameters detected
+      return false;
+    }
+
+    client.setDatasetParameters(toDatasetReference(namespace), properties);
+    return true;
+  }
+
+  @Override
+  public boolean removeProperties(Namespace namespace, Set<String> properties) {
+    client.removeDatasetParameters(toDatasetReference(namespace), properties);
+    return true;
+  }
+
+  @SuppressWarnings("FormatStringAnnotation")
+  @Override
+  public Map<String, String> loadNamespaceMetadata(Namespace namespace) {
+    try {
+      return getMetadata(client.getDataset(toDatasetReference(namespace)));
+    } catch (IllegalArgumentException e) {
+      throw new NoSuchNamespaceException(e.getMessage());
+    }
+  }
+
+  @Override
+  public String name() {
+    return catalogName;
+  }
+
+  @Override
+  protected Map<String, String> properties() {
+    return catalogProperties == null ? ImmutableMap.of() : catalogProperties;
+  }
+
+  @Override
+  public void setConf(Configuration conf) {
+    this.conf = new Configuration(conf);
+  }
+
+  @Override
+  public Configuration getConf() {
+    return this.conf;
+  }
+
+  private static Namespace getNamespace(Datasets dataset) {
+    return Namespace.of(dataset.getDatasetReference().getDatasetId());
+  }
+
+  private DatasetReference toDatasetReference(Namespace namespace) {
+    validateNamespace(namespace);
+    return new DatasetReference().setProjectId(projectId).setDatasetId(namespace.level(0));
+  }
+
+  private TableReference toTableReference(TableIdentifier tableIdentifier) {
+    DatasetReference datasetReference = toDatasetReference(tableIdentifier.namespace());
+    return new TableReference()
+        .setProjectId(datasetReference.getProjectId())
+        .setDatasetId(datasetReference.getDatasetId())
+        .setTableId(tableIdentifier.name());
+  }
+
+  private static Map<String, String> getMetadata(Dataset dataset) {
+    ExternalCatalogDatasetOptions options = dataset.getExternalCatalogDatasetOptions();
+    Map<String, String> metadata = Maps.newHashMap();
+    if (options != null) {
+      if (options.getParameters() != null) {
+        metadata.putAll(options.getParameters());
+      }
+      if (!Strings.isNullOrEmpty(options.getDefaultStorageLocationUri())) {
+        metadata.put("location", options.getDefaultStorageLocationUri());
+      }
+    }
+    return metadata;
+  }
+
+  private static void validateNamespace(Namespace namespace) {

Review Comment:
   this should probably also be checked in `dropNamespace`/ `createNamespace`.



-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org

For queries about this service, please contact Infrastructure at:
users@infra.apache.org


---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For additional commands, e-mail: issues-help@iceberg.apache.org


Re: [PR] Catalog: Add BigQuery Metastore Catalog Support [iceberg]

Posted by "talatuyarer (via GitHub)" <gi...@apache.org>.
talatuyarer commented on code in PR #12808:
URL: https://github.com/apache/iceberg/pull/12808#discussion_r2059699417


##########
bigquery/src/main/java/org/apache/iceberg/gcp/bigquery/BigQueryMetastoreCatalog.java:
##########
@@ -0,0 +1,400 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.iceberg.gcp.bigquery;
+
+import com.google.api.services.bigquery.model.Dataset;
+import com.google.api.services.bigquery.model.DatasetList.Datasets;
+import com.google.api.services.bigquery.model.DatasetReference;
+import com.google.api.services.bigquery.model.ExternalCatalogDatasetOptions;
+import com.google.api.services.bigquery.model.TableReference;
+import com.google.cloud.ServiceOptions;
+import com.google.cloud.bigquery.BigQueryOptions;
+import java.io.IOException;
+import java.security.GeneralSecurityException;
+import java.util.List;
+import java.util.Locale;
+import java.util.Map;
+import java.util.Objects;
+import java.util.Set;
+import org.apache.hadoop.conf.Configurable;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.iceberg.BaseMetastoreCatalog;
+import org.apache.iceberg.CatalogProperties;
+import org.apache.iceberg.CatalogUtil;
+import org.apache.iceberg.TableMetadata;
+import org.apache.iceberg.TableOperations;
+import org.apache.iceberg.catalog.Namespace;
+import org.apache.iceberg.catalog.SupportsNamespaces;
+import org.apache.iceberg.catalog.TableIdentifier;
+import org.apache.iceberg.exceptions.AlreadyExistsException;
+import org.apache.iceberg.exceptions.NoSuchNamespaceException;
+import org.apache.iceberg.exceptions.NoSuchTableException;
+import org.apache.iceberg.exceptions.ServiceFailureException;
+import org.apache.iceberg.exceptions.ValidationException;
+import org.apache.iceberg.io.FileIO;
+import org.apache.iceberg.relocated.com.google.common.annotations.VisibleForTesting;
+import org.apache.iceberg.relocated.com.google.common.base.Preconditions;
+import org.apache.iceberg.relocated.com.google.common.base.Strings;
+import org.apache.iceberg.relocated.com.google.common.collect.ImmutableList;
+import org.apache.iceberg.relocated.com.google.common.collect.ImmutableMap;
+import org.apache.iceberg.relocated.com.google.common.collect.Maps;
+import org.apache.iceberg.util.LocationUtil;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/** Iceberg Bigquery Metastore Catalog implementation. */
+public final class BigQueryMetastoreCatalog extends BaseMetastoreCatalog
+    implements SupportsNamespaces, Configurable {
+
+  // User provided properties.
+  public static final String PROJECT_ID = "gcp-project";
+  public static final String GCP_LOCATION = "gcp-location";
+  public static final String FILTER_UNSUPPORTED_TABLES = "filter-unsupported-tables";
+  public static final String TESTING_ENABLED = "testing-enabled";
+
+  public static final String HIVE_METASTORE_WAREHOUSE_DIR = "hive.metastore.warehouse.dir";
+
+  private static final Logger LOG = LoggerFactory.getLogger(BigQueryMetastoreCatalog.class);
+
+  private static final String DEFAULT_GCP_LOCATION = "us";
+
+  private String catalogName;
+  private Map<String, String> catalogProperties;
+  private FileIO fileIO;
+  private Configuration conf;
+  private String projectId;
+  private String location;
+  private BigQueryMetastoreClient client;
+  private boolean filterUnsupportedTables;
+
+  // Must have a no-arg constructor to be dynamically loaded
+  // initialize(String name, Map<String, String> properties) will be called to complete
+  // initialization
+  public BigQueryMetastoreCatalog() {}
+
+  @Override
+  public void initialize(String inputName, Map<String, String> properties) {
+    Preconditions.checkArgument(
+        properties.containsKey(PROJECT_ID), "GCP project must be specified");
+
+    projectId = properties.get(PROJECT_ID);
+    location = properties.getOrDefault(GCP_LOCATION, DEFAULT_GCP_LOCATION);
+    boolean testingEnabled =
+        Boolean.parseBoolean(properties.getOrDefault(TESTING_ENABLED, "false"));
+
+    BigQueryOptions options =
+        BigQueryOptions.newBuilder()
+            .setProjectId(projectId)
+            .setLocation(location)
+            .setRetrySettings(ServiceOptions.getDefaultRetrySettings())
+            .build();
+
+    try {
+      if (testingEnabled) {
+        client = new FakeBigQueryMetastoreClient(options);
+
+      } else {
+        client = new BigQueryMetastoreClientImpl(options);
+      }
+    } catch (IOException e) {
+      throw new ServiceFailureException(e, "Creating BigQuery client failed");
+    } catch (GeneralSecurityException e) {
+      throw new ValidationException(e, "Creating BigQuery client failed due to a security issue");
+    }
+    initialize(inputName, properties, projectId, location, client);
+  }
+
+  @VisibleForTesting
+  void initialize(
+      String inputName,
+      Map<String, String> properties,
+      String initialProjectId,
+      String initialLocation,
+      BigQueryMetastoreClient bigQueryMetaStoreClient) {
+    Preconditions.checkArgument(bigQueryMetaStoreClient != null, "BigQuery client can not be null");
+    this.catalogName = inputName;
+    this.catalogProperties = ImmutableMap.copyOf(properties);
+    this.projectId = initialProjectId;
+    this.location = initialLocation;
+    this.client = bigQueryMetaStoreClient;
+
+    if (this.conf == null) {
+      LOG.warn("No configuration was set, using the default environment Configuration");
+      this.conf = new Configuration();
+    }
+
+    LOG.info("Using BigQuery Metastore Iceberg Catalog: {}", inputName);
+
+    if (properties.containsKey(CatalogProperties.WAREHOUSE_LOCATION)) {
+      // Iceberg always removes trailing slash to avoid paths like "<folder>//data" in file systems

Review Comment:
   Done



-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org

For queries about this service, please contact Infrastructure at:
users@infra.apache.org


---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For additional commands, e-mail: issues-help@iceberg.apache.org


Re: [PR] Catalog: Add BigQuery Metastore Catalog Support [iceberg]

Posted by "nastra (via GitHub)" <gi...@apache.org>.
nastra commented on code in PR #12808:
URL: https://github.com/apache/iceberg/pull/12808#discussion_r2059745760


##########
bigquery/src/test/java/org/apache/iceberg/gcp/bigquery/TestBigQueryCatalog.java:
##########
@@ -0,0 +1,673 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.iceberg.gcp.bigquery;
+
+import static org.apache.iceberg.CatalogUtil.ICEBERG_CATALOG_TYPE;
+import static org.apache.iceberg.CatalogUtil.ICEBERG_CATALOG_TYPE_BIGQUERY;
+import static org.apache.iceberg.gcp.bigquery.BigQueryMetastoreCatalog.PROJECT_ID;
+import static org.apache.iceberg.gcp.bigquery.BigQueryMetastoreCatalog.TESTING_ENABLED;
+import static org.apache.iceberg.types.Types.NestedField.optional;
+import static org.assertj.core.api.Assertions.assertThat;
+import static org.assertj.core.api.Assertions.assertThatThrownBy;
+
+import java.io.File;
+import java.util.List;
+import java.util.Map;
+import org.apache.iceberg.CatalogProperties;
+import org.apache.iceberg.CatalogUtil;
+import org.apache.iceberg.PartitionSpec;
+import org.apache.iceberg.Schema;
+import org.apache.iceberg.SortOrder;
+import org.apache.iceberg.Table;
+import org.apache.iceberg.TableUtil;
+import org.apache.iceberg.Transaction;
+import org.apache.iceberg.catalog.CatalogTests;
+import org.apache.iceberg.catalog.Namespace;
+import org.apache.iceberg.catalog.TableIdentifier;
+import org.apache.iceberg.exceptions.NoSuchNamespaceException;
+import org.apache.iceberg.exceptions.NoSuchTableException;
+import org.apache.iceberg.expressions.Expressions;
+import org.apache.iceberg.relocated.com.google.common.collect.ImmutableMap;
+import org.apache.iceberg.types.Types;
+import org.junit.jupiter.api.AfterEach;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Disabled;
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.io.TempDir;
+import org.junit.jupiter.params.ParameterizedTest;
+import org.junit.jupiter.params.provider.ValueSource;
+
+public class TestBigQueryCatalog extends CatalogTests<BigQueryMetastoreCatalog> {
+
+  private static final String GCP_PROJECT = "project-id";
+  private static final String CATALOG_ID = "catalog-name";
+
+  // another schema that is not the same
+  protected static final Schema OTHER_SCHEMA_OPTIONAL =
+      new Schema(optional(1, "some_id", Types.IntegerType.get()));
+
+  // Schema passed to create tables
+  protected static final Schema SCHEMA_OPTIONAL =
+      new Schema(
+          optional(3, "id", Types.IntegerType.get(), "unique ID 🤪"),
+          optional(4, "data", Types.StringType.get()));
+
+  // This is the actual schema for the table, with column IDs reassigned
+  protected static final Schema REPLACE_SCHEMA_OPTIONAL =
+      new Schema(
+          optional(2, "id", Types.IntegerType.get(), "unique ID 🤪"),
+          optional(3, "data", Types.StringType.get()));
+
+  protected static final PartitionSpec REPLACE_SPEC_OPTIONAL =
+      PartitionSpec.builderFor(REPLACE_SCHEMA_OPTIONAL).bucket("id", 16).withSpecId(1).build();
+
+  protected static final SortOrder REPLACE_WRITE_ORDER_OPTIONAL =
+      SortOrder.builderFor(REPLACE_SCHEMA_OPTIONAL)
+          .asc(Expressions.bucket("id", 16))
+          .asc("id")
+          .build();
+
+  @TempDir private File tempFolder;
+  private BigQueryMetastoreCatalog catalog;
+  private String warehouseLocation;
+
+  @BeforeEach
+  public void before() throws Exception {
+    catalog = initCatalog(CATALOG_ID, ImmutableMap.of());
+  }
+
+  @AfterEach
+  public void after() throws Exception {
+    // Drop all tables in all datasets
+    List<Namespace> namespaces = catalog().listNamespaces(Namespace.empty());
+    for (Namespace namespace : namespaces) {
+      List<TableIdentifier> tables = catalog().listTables(namespace);
+      for (TableIdentifier table : tables) {
+        try {
+          catalog().dropTable(table, true); // drop with purge
+        } catch (NoSuchTableException e) {
+          // Table might be dropped by a previous iteration, ignore
+        }
+      }
+
+      // Drop all datasets (except any initial ones)
+      try {
+        catalog().dropNamespace(namespace);
+      } catch (NoSuchNamespaceException e) {
+        // Namespace might be dropped by a previous iteration, ignore
+      }
+    }
+  }
+
+  @Override
+  protected boolean requiresNamespaceCreate() {
+    return true;
+  }
+
+  @Override
+  protected boolean supportsNamesWithSlashes() {
+    return false;
+  }
+
+  @Override
+  protected boolean supportsNamesWithDot() {
+    return false;
+  }
+
+  @Override
+  public BigQueryMetastoreCatalog catalog() {
+    return catalog;
+  }
+
+  @Override
+  protected BigQueryMetastoreCatalog initCatalog(
+      String catalogName, Map<String, String> additionalProperties) {
+
+    warehouseLocation = tempFolder.toPath().resolve("hive-warehouse").toString();
+
+    Map<String, String> properties =
+        Map.of(
+            ICEBERG_CATALOG_TYPE,
+            ICEBERG_CATALOG_TYPE_BIGQUERY,
+            PROJECT_ID,
+            GCP_PROJECT,
+            CatalogProperties.WAREHOUSE_LOCATION,
+            warehouseLocation,
+            TESTING_ENABLED,
+            "true",
+            CatalogProperties.TABLE_DEFAULT_PREFIX + "default-key1",
+            "catalog-default-key1",
+            CatalogProperties.TABLE_DEFAULT_PREFIX + "default-key2",
+            "catalog-default-key2",
+            CatalogProperties.TABLE_DEFAULT_PREFIX + "override-key3",
+            "catalog-default-key3",
+            CatalogProperties.TABLE_OVERRIDE_PREFIX + "override-key3",
+            "catalog-override-key3",
+            CatalogProperties.TABLE_OVERRIDE_PREFIX + "override-key4",
+            "catalog-override-key4");
+
+    BigQueryMetastoreCatalog tmpCatalog =
+        (BigQueryMetastoreCatalog)
+            CatalogUtil.buildIcebergCatalog(
+                catalogName,
+                ImmutableMap.<String, String>builder()
+                    .putAll(properties)
+                    .putAll(additionalProperties)
+                    .build(),
+                null);
+
+    return tmpCatalog;
+  }
+
+  @Test
+  public void testRenameTable() {
+    if (requiresNamespaceCreate()) {
+      catalog.createNamespace(NS);
+    }
+
+    assertThat(catalog.tableExists(TABLE))
+        .as("Source table should not exist before create")
+        .isFalse();
+
+    catalog.buildTable(TABLE, SCHEMA).create();
+    assertThat(catalog.tableExists(TABLE)).as("Table should exist after create").isTrue();
+
+    assertThat(catalog.tableExists(RENAMED_TABLE))
+        .as("Destination table should not exist before rename")
+        .isFalse();
+
+    assertThatThrownBy(() -> catalog.renameTable(TABLE, RENAMED_TABLE))
+        .isInstanceOf(UnsupportedOperationException.class)
+        .hasMessageContaining("Table rename operation is unsupported");
+
+    catalog.dropTable(TABLE);
+  }
+
+  @Disabled(
+      "BigQuery does not allow adding a REQUIRED column to an existing table schema. "
+          + "When this operation is attempted, an exception gets thrown.")
+  @Test
+  public void testOverrideTablePropertiesReplaceTransaction() {}

Review Comment:
   given that the tests here that live in the Iceberg codebase are always running against the Fake client, I would suggest to remove all of those `Disabled` tests (and basically trim the test down to what I suggested in https://github.com/apache/iceberg/pull/12808/files#r2055539916), since they all pass against the Fake client. If you want to run the catalog tests against an actual client, you can then basically just create a new `TestRealBigQueryCatalog` (I'm sure there's a better name) that initializes the correct client where you disable the tests that don't work and also test with optional vs required fields in the schema



-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org

For queries about this service, please contact Infrastructure at:
users@infra.apache.org


---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For additional commands, e-mail: issues-help@iceberg.apache.org


Re: [PR] Catalog: Add BigQuery Metastore Catalog Support [iceberg]

Posted by "talatuyarer (via GitHub)" <gi...@apache.org>.
talatuyarer commented on code in PR #12808:
URL: https://github.com/apache/iceberg/pull/12808#discussion_r2059825065


##########
bigquery/src/main/java/org/apache/iceberg/gcp/bigquery/BigQueryMetastoreCatalog.java:
##########
@@ -0,0 +1,400 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.iceberg.gcp.bigquery;
+
+import com.google.api.services.bigquery.model.Dataset;
+import com.google.api.services.bigquery.model.DatasetList.Datasets;
+import com.google.api.services.bigquery.model.DatasetReference;
+import com.google.api.services.bigquery.model.ExternalCatalogDatasetOptions;
+import com.google.api.services.bigquery.model.TableReference;
+import com.google.cloud.ServiceOptions;
+import com.google.cloud.bigquery.BigQueryOptions;
+import java.io.IOException;
+import java.security.GeneralSecurityException;
+import java.util.List;
+import java.util.Locale;
+import java.util.Map;
+import java.util.Objects;
+import java.util.Set;
+import org.apache.hadoop.conf.Configurable;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.iceberg.BaseMetastoreCatalog;
+import org.apache.iceberg.CatalogProperties;
+import org.apache.iceberg.CatalogUtil;
+import org.apache.iceberg.TableMetadata;
+import org.apache.iceberg.TableOperations;
+import org.apache.iceberg.catalog.Namespace;
+import org.apache.iceberg.catalog.SupportsNamespaces;
+import org.apache.iceberg.catalog.TableIdentifier;
+import org.apache.iceberg.exceptions.AlreadyExistsException;
+import org.apache.iceberg.exceptions.NoSuchNamespaceException;
+import org.apache.iceberg.exceptions.NoSuchTableException;
+import org.apache.iceberg.exceptions.ServiceFailureException;
+import org.apache.iceberg.exceptions.ValidationException;
+import org.apache.iceberg.io.FileIO;
+import org.apache.iceberg.relocated.com.google.common.annotations.VisibleForTesting;
+import org.apache.iceberg.relocated.com.google.common.base.Preconditions;
+import org.apache.iceberg.relocated.com.google.common.base.Strings;
+import org.apache.iceberg.relocated.com.google.common.collect.ImmutableList;
+import org.apache.iceberg.relocated.com.google.common.collect.ImmutableMap;
+import org.apache.iceberg.relocated.com.google.common.collect.Maps;
+import org.apache.iceberg.util.LocationUtil;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/** Iceberg Bigquery Metastore Catalog implementation. */
+public final class BigQueryMetastoreCatalog extends BaseMetastoreCatalog
+    implements SupportsNamespaces, Configurable {
+
+  // User provided properties.
+  public static final String PROJECT_ID = "gcp-project";
+  public static final String GCP_LOCATION = "gcp-location";
+  public static final String FILTER_UNSUPPORTED_TABLES = "filter-unsupported-tables";
+  public static final String TESTING_ENABLED = "testing-enabled";
+
+  public static final String HIVE_METASTORE_WAREHOUSE_DIR = "hive.metastore.warehouse.dir";
+
+  private static final Logger LOG = LoggerFactory.getLogger(BigQueryMetastoreCatalog.class);
+
+  private static final String DEFAULT_GCP_LOCATION = "us";
+
+  private String catalogName;
+  private Map<String, String> catalogProperties;
+  private FileIO fileIO;
+  private Configuration conf;
+  private String projectId;
+  private String location;
+  private BigQueryMetastoreClient client;
+  private boolean filterUnsupportedTables;
+
+  // Must have a no-arg constructor to be dynamically loaded
+  // initialize(String name, Map<String, String> properties) will be called to complete
+  // initialization
+  public BigQueryMetastoreCatalog() {}
+
+  @Override
+  public void initialize(String inputName, Map<String, String> properties) {
+    Preconditions.checkArgument(
+        properties.containsKey(PROJECT_ID), "GCP project must be specified");
+
+    projectId = properties.get(PROJECT_ID);
+    location = properties.getOrDefault(GCP_LOCATION, DEFAULT_GCP_LOCATION);
+    boolean testingEnabled =

Review Comment:
   Ok Changed implementation. And also moved FakeClient under Test



-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org

For queries about this service, please contact Infrastructure at:
users@infra.apache.org


---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For additional commands, e-mail: issues-help@iceberg.apache.org


Re: [PR] Catalog: Add BigQuery Metastore Catalog Support [iceberg]

Posted by "talatuyarer (via GitHub)" <gi...@apache.org>.
talatuyarer commented on code in PR #12808:
URL: https://github.com/apache/iceberg/pull/12808#discussion_r2059769207


##########
bigquery/src/test/java/org/apache/iceberg/gcp/bigquery/TestBigQueryCatalog.java:
##########
@@ -0,0 +1,673 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.iceberg.gcp.bigquery;
+
+import static org.apache.iceberg.CatalogUtil.ICEBERG_CATALOG_TYPE;
+import static org.apache.iceberg.CatalogUtil.ICEBERG_CATALOG_TYPE_BIGQUERY;
+import static org.apache.iceberg.gcp.bigquery.BigQueryMetastoreCatalog.PROJECT_ID;
+import static org.apache.iceberg.gcp.bigquery.BigQueryMetastoreCatalog.TESTING_ENABLED;
+import static org.apache.iceberg.types.Types.NestedField.optional;
+import static org.assertj.core.api.Assertions.assertThat;
+import static org.assertj.core.api.Assertions.assertThatThrownBy;
+
+import java.io.File;
+import java.util.List;
+import java.util.Map;
+import org.apache.iceberg.CatalogProperties;
+import org.apache.iceberg.CatalogUtil;
+import org.apache.iceberg.PartitionSpec;
+import org.apache.iceberg.Schema;
+import org.apache.iceberg.SortOrder;
+import org.apache.iceberg.Table;
+import org.apache.iceberg.TableUtil;
+import org.apache.iceberg.Transaction;
+import org.apache.iceberg.catalog.CatalogTests;
+import org.apache.iceberg.catalog.Namespace;
+import org.apache.iceberg.catalog.TableIdentifier;
+import org.apache.iceberg.exceptions.NoSuchNamespaceException;
+import org.apache.iceberg.exceptions.NoSuchTableException;
+import org.apache.iceberg.expressions.Expressions;
+import org.apache.iceberg.relocated.com.google.common.collect.ImmutableMap;
+import org.apache.iceberg.types.Types;
+import org.junit.jupiter.api.AfterEach;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Disabled;
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.io.TempDir;
+import org.junit.jupiter.params.ParameterizedTest;
+import org.junit.jupiter.params.provider.ValueSource;
+
+public class TestBigQueryCatalog extends CatalogTests<BigQueryMetastoreCatalog> {
+
+  private static final String GCP_PROJECT = "project-id";
+  private static final String CATALOG_ID = "catalog-name";

Review Comment:
   Replaced with inlined version



-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org

For queries about this service, please contact Infrastructure at:
users@infra.apache.org


---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For additional commands, e-mail: issues-help@iceberg.apache.org


Re: [PR] Catalog: Add BigQuery Metastore Catalog Support [iceberg]

Posted by "nastra (via GitHub)" <gi...@apache.org>.
nastra commented on code in PR #12808:
URL: https://github.com/apache/iceberg/pull/12808#discussion_r2055669509


##########
bigquery/src/main/java/org/apache/iceberg/gcp/bigquery/BigQueryTableOperations.java:
##########
@@ -0,0 +1,297 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.iceberg.gcp.bigquery;
+
+import com.google.api.services.bigquery.model.ExternalCatalogTableOptions;
+import com.google.api.services.bigquery.model.Table;
+import com.google.api.services.bigquery.model.TableReference;
+import java.util.Locale;
+import java.util.Map;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hive.common.StatsSetupConst;
+import org.apache.iceberg.BaseMetastoreOperations;
+import org.apache.iceberg.BaseMetastoreTableOperations;
+import org.apache.iceberg.SnapshotSummary;
+import org.apache.iceberg.TableMetadata;
+import org.apache.iceberg.TableProperties;
+import org.apache.iceberg.exceptions.AlreadyExistsException;
+import org.apache.iceberg.exceptions.CommitFailedException;
+import org.apache.iceberg.exceptions.CommitStateUnknownException;
+import org.apache.iceberg.exceptions.NoSuchTableException;
+import org.apache.iceberg.exceptions.ValidationException;
+import org.apache.iceberg.hadoop.ConfigProperties;
+import org.apache.iceberg.io.FileIO;
+import org.apache.iceberg.relocated.com.google.common.collect.Maps;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/** Handles BigQuery metastore table operations. */
+public final class BigQueryTableOperations extends BaseMetastoreTableOperations {
+
+  private static final Logger LOG = LoggerFactory.getLogger(BigQueryTableOperations.class);
+
+  public static final String TABLE_PROPERTIES_BQ_CONNECTION = "bq_connection";
+
+  private final BigQueryMetastoreClient client;
+  private final FileIO fileIO;
+  private final TableReference tableReference;
+  private final Configuration conf;
+
+  BigQueryTableOperations(
+      BigQueryMetastoreClient client,
+      FileIO fileIO,
+      String project,
+      String dataset,
+      String table,
+      Configuration conf) {
+    this.client = client;
+    this.fileIO = fileIO;
+    this.tableReference =
+        new TableReference().setProjectId(project).setDatasetId(dataset).setTableId(table);
+    this.conf = conf;
+  }
+
+  // The doRefresh method should provide implementation on how to get the metadata location.
+  @Override
+  public void doRefresh() {
+    // Must default to null.
+    String metadataLocation = null;
+    try {
+      metadataLocation =
+          getMetadataLocationOrThrow(
+              client.getTable(this.tableReference).getExternalCatalogTableOptions());
+    } catch (NoSuchTableException e) {
+      if (currentMetadataLocation() != null) {
+        // Re-throws the exception because the table must exist in this case.
+        throw e;
+      }
+    }
+    refreshFromMetadataLocation(metadataLocation);
+  }
+
+  // The doCommit method should provide implementation on how to update with metadata location
+  // atomically
+  @Override
+  public void doCommit(TableMetadata base, TableMetadata metadata) {
+    String newMetadataLocation =
+        base == null && metadata.metadataFileLocation() != null
+            ? metadata.metadataFileLocation()
+            : writeNewMetadata(metadata, currentVersion() + 1);
+    BaseMetastoreOperations.CommitStatus commitStatus =
+        BaseMetastoreOperations.CommitStatus.FAILURE;
+    try {
+      if (base == null) {
+        createTable(newMetadataLocation, metadata);
+      } else {
+        updateTable(base.metadataFileLocation(), newMetadataLocation, metadata);
+      }
+      commitStatus = BaseMetastoreOperations.CommitStatus.SUCCESS;
+    } catch (CommitFailedException | CommitStateUnknownException e) {
+      throw e;
+    } catch (Throwable e) {
+      LOG.error("Exception thrown on commit: ", e);
+      if (e instanceof AlreadyExistsException) {
+        throw e;
+      }
+      commitStatus =
+          BaseMetastoreOperations.CommitStatus.valueOf(
+              checkCommitStatus(newMetadataLocation, metadata).name());
+      if (commitStatus == BaseMetastoreOperations.CommitStatus.FAILURE) {
+        throw new CommitFailedException(e, "Failed to commit");
+      }
+      if (commitStatus == BaseMetastoreOperations.CommitStatus.UNKNOWN) {
+        throw new CommitStateUnknownException(e);
+      }
+    } finally {
+      try {
+        if (commitStatus == BaseMetastoreOperations.CommitStatus.FAILURE) {
+          LOG.warn("Failed to commit updates to table {}", tableName());
+          io().deleteFile(newMetadataLocation);
+        }
+      } catch (RuntimeException e) {
+        LOG.error(
+            "Failed to cleanup metadata file at {} for table {}",
+            newMetadataLocation,
+            tableName(),
+            e);
+      }
+    }
+  }
+
+  @Override
+  public String tableName() {
+    return String.format("%s.%s", tableReference.getDatasetId(), tableReference.getTableId());
+  }
+
+  @Override
+  public FileIO io() {
+    return fileIO;
+  }
+
+  private void createTable(String newMetadataLocation, TableMetadata metadata) {
+    LOG.debug("Creating a new Iceberg table: {}", tableName());
+    Table tableBuilder = makeNewTable(metadata, newMetadataLocation);
+    tableBuilder.setTableReference(this.tableReference);
+    addConnectionIfProvided(tableBuilder, metadata.properties());
+
+    client.createTable(tableBuilder);
+  }
+
+  private void addConnectionIfProvided(Table tableBuilder, Map<String, String> metadataProperties) {
+    if (metadataProperties.containsKey(TABLE_PROPERTIES_BQ_CONNECTION)) {
+      tableBuilder
+          .getExternalCatalogTableOptions()
+          .setConnectionId(metadataProperties.get(TABLE_PROPERTIES_BQ_CONNECTION));
+    }
+  }
+
+  /** Update table properties with concurrent update detection using etag. */
+  private void updateTable(
+      String oldMetadataLocation, String newMetadataLocation, TableMetadata metadata) {
+    Table table = client.getTable(this.tableReference);
+    if (table.getEtag().isEmpty()) {
+      throw new ValidationException(
+          "Etag of legacy table %s is empty, manually update the table via the BigQuery API or"
+              + " recreate and retry",
+          tableName());
+    }
+    ExternalCatalogTableOptions options = table.getExternalCatalogTableOptions();
+    addConnectionIfProvided(table, metadata.properties());
+
+    // If `metadataLocationFromMetastore` is different from metadata location of base, it means
+    // someone has updated metadata location in metastore, which is a conflict update.
+    String metadataLocationFromMetastore =
+        options.getParameters().getOrDefault(METADATA_LOCATION_PROP, "");
+    if (!metadataLocationFromMetastore.isEmpty()
+        && !metadataLocationFromMetastore.equals(oldMetadataLocation)) {
+      throw new CommitFailedException(
+          "Cannot commit base metadata location '%s' is not same as the current table metadata location '%s' for"
+              + " %s.%s",
+          oldMetadataLocation,
+          metadataLocationFromMetastore,
+          tableReference.getDatasetId(),
+          tableReference.getTableId());
+    }
+
+    options.setParameters(buildTableParameters(newMetadataLocation, metadata));
+    try {
+      client.patchTable(tableReference, table);
+    } catch (ValidationException e) {
+      if (e.getMessage().toLowerCase(Locale.ENGLISH).contains("etag mismatch")) {
+        throw new CommitFailedException(
+            "Updating table failed due to conflict updates (etag mismatch). Retry the update");
+      }
+
+      throw e;
+    }
+  }
+
+  // To make the table queryable from Hive, the user would likely be setting the HIVE_ENGINE_ENABLED
+  // parameter.
+  //
+  // TODO: We need to make a decision on how to make the table queryable from Hive.
+  // (could be a server side change or a client side change - that's TBD).
+  private Table makeNewTable(TableMetadata metadata, String metadataFileLocation) {
+    boolean hiveEngineEnabled = getHiveEngineEnabled(metadata);
+    return new Table()
+        .setExternalCatalogTableOptions(
+            BigQueryMetastoreUtils.createExternalCatalogTableOptions(
+                metadata.location(),
+                buildTableParameters(metadataFileLocation, metadata),
+                hiveEngineEnabled));
+  }
+
+  // Follow Iceberg's HiveTableOperations to populate more table parameters for HMS compatibility.
+  private Map<String, String> buildTableParameters(
+      String metadataFileLocation, TableMetadata metadata) {
+    Map<String, String> parameters = Maps.newHashMap(metadata.properties());
+    if (metadata.uuid() != null) {
+      parameters.put(TableProperties.UUID, metadata.uuid());
+    }
+    if (currentMetadataLocation() != null && !currentMetadataLocation().isEmpty()) {
+      parameters.put(PREVIOUS_METADATA_LOCATION_PROP, currentMetadataLocation());
+    }
+    parameters.put(METADATA_LOCATION_PROP, metadataFileLocation);
+    parameters.put(TABLE_TYPE_PROP, ICEBERG_TABLE_TYPE_VALUE);
+    // Follow HMS to use the EXTERNAL type.
+    parameters.put("EXTERNAL", "TRUE");
+
+    // Hive style basic statistics.
+    updateParametersWithSnapshotMetadata(metadata, parameters);
+    // More Iceberg metadata can be exposed, e.g., statistic, schema, partition spec, as HMS do. But
+    // we should be careful that these metadata could be huge and make the metadata API response
+    // less readable (e.g., list tables). Users can always inspect these metadata in Spark, so they
+    // are not set for now.
+    return parameters;
+  }
+
+  /** Adds Hive-style basic statistics from snapshot metadata if it exists. */
+  private static void updateParametersWithSnapshotMetadata(
+      TableMetadata metadata, Map<String, String> parameters) {
+    if (metadata.currentSnapshot() == null) {
+      return;
+    }
+
+    Map<String, String> summary = metadata.currentSnapshot().summary();
+    if (summary.get(SnapshotSummary.TOTAL_DATA_FILES_PROP) != null) {
+      parameters.put(StatsSetupConst.NUM_FILES, summary.get(SnapshotSummary.TOTAL_DATA_FILES_PROP));
+    }
+    if (summary.get(SnapshotSummary.TOTAL_RECORDS_PROP) != null) {
+      parameters.put(StatsSetupConst.ROW_COUNT, summary.get(SnapshotSummary.TOTAL_RECORDS_PROP));
+    }
+    if (summary.get(SnapshotSummary.TOTAL_FILE_SIZE_PROP) != null) {
+      parameters.put(StatsSetupConst.TOTAL_SIZE, summary.get(SnapshotSummary.TOTAL_FILE_SIZE_PROP));
+    }
+  }
+
+  private String getMetadataLocationOrThrow(ExternalCatalogTableOptions tableOptions) {
+    if (tableOptions == null || !tableOptions.getParameters().containsKey(METADATA_LOCATION_PROP)) {
+      throw new ValidationException(
+          "Table %s is not a valid BigQuery Metastore Iceberg table, metadata location not found",
+          tableName());
+    }
+    return tableOptions.getParameters().get(METADATA_LOCATION_PROP);
+  }
+
+  /**
+   * Returns if the hive engine related values should be enabled on the table, or not.
+   *
+   * <p>The decision is made like this:
+   *
+   * <ol>
+   *   <li>Table property value {@link TableProperties#ENGINE_HIVE_ENABLED}
+   *   <li>If the table property is not set then check the hive-site.xml property value {@link
+   *       ConfigProperties#ENGINE_HIVE_ENABLED}
+   *   <li>If none of the above is enabled then use the default value {@link
+   *       TableProperties#ENGINE_HIVE_ENABLED_DEFAULT}
+   * </ol>
+   *
+   * @param metadata Table metadata to use
+   * @return if the hive engine related values should be enabled or not
+   */
+  private boolean getHiveEngineEnabled(TableMetadata metadata) {

Review Comment:
   ```suggestion
     private boolean hiveEngineEnabled(TableMetadata metadata) {
   ```



-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org

For queries about this service, please contact Infrastructure at:
users@infra.apache.org


---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For additional commands, e-mail: issues-help@iceberg.apache.org


Re: [PR] Catalog: Add BigQuery Metastore Catalog Support [iceberg]

Posted by "nastra (via GitHub)" <gi...@apache.org>.
nastra commented on code in PR #12808:
URL: https://github.com/apache/iceberg/pull/12808#discussion_r2055673312


##########
bigquery/src/test/java/org/apache/iceberg/gcp/bigquery/TestBigQueryCatalog.java:
##########
@@ -0,0 +1,673 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.iceberg.gcp.bigquery;
+
+import static org.apache.iceberg.CatalogUtil.ICEBERG_CATALOG_TYPE;
+import static org.apache.iceberg.CatalogUtil.ICEBERG_CATALOG_TYPE_BIGQUERY;
+import static org.apache.iceberg.gcp.bigquery.BigQueryMetastoreCatalog.PROJECT_ID;
+import static org.apache.iceberg.gcp.bigquery.BigQueryMetastoreCatalog.TESTING_ENABLED;
+import static org.apache.iceberg.types.Types.NestedField.optional;
+import static org.assertj.core.api.Assertions.assertThat;
+import static org.assertj.core.api.Assertions.assertThatThrownBy;
+
+import java.io.File;
+import java.util.List;
+import java.util.Map;
+import org.apache.iceberg.CatalogProperties;
+import org.apache.iceberg.CatalogUtil;
+import org.apache.iceberg.PartitionSpec;
+import org.apache.iceberg.Schema;
+import org.apache.iceberg.SortOrder;
+import org.apache.iceberg.Table;
+import org.apache.iceberg.TableUtil;
+import org.apache.iceberg.Transaction;
+import org.apache.iceberg.catalog.CatalogTests;
+import org.apache.iceberg.catalog.Namespace;
+import org.apache.iceberg.catalog.TableIdentifier;
+import org.apache.iceberg.exceptions.NoSuchNamespaceException;
+import org.apache.iceberg.exceptions.NoSuchTableException;
+import org.apache.iceberg.expressions.Expressions;
+import org.apache.iceberg.relocated.com.google.common.collect.ImmutableMap;
+import org.apache.iceberg.types.Types;
+import org.junit.jupiter.api.AfterEach;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Disabled;
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.io.TempDir;
+import org.junit.jupiter.params.ParameterizedTest;
+import org.junit.jupiter.params.provider.ValueSource;
+
+public class TestBigQueryCatalog extends CatalogTests<BigQueryMetastoreCatalog> {
+
+  private static final String GCP_PROJECT = "project-id";
+  private static final String CATALOG_ID = "catalog-name";
+
+  // another schema that is not the same
+  protected static final Schema OTHER_SCHEMA_OPTIONAL =
+      new Schema(optional(1, "some_id", Types.IntegerType.get()));
+
+  // Schema passed to create tables
+  protected static final Schema SCHEMA_OPTIONAL =
+      new Schema(
+          optional(3, "id", Types.IntegerType.get(), "unique ID 🤪"),
+          optional(4, "data", Types.StringType.get()));
+
+  // This is the actual schema for the table, with column IDs reassigned
+  protected static final Schema REPLACE_SCHEMA_OPTIONAL =
+      new Schema(
+          optional(2, "id", Types.IntegerType.get(), "unique ID 🤪"),
+          optional(3, "data", Types.StringType.get()));
+
+  protected static final PartitionSpec REPLACE_SPEC_OPTIONAL =
+      PartitionSpec.builderFor(REPLACE_SCHEMA_OPTIONAL).bucket("id", 16).withSpecId(1).build();
+
+  protected static final SortOrder REPLACE_WRITE_ORDER_OPTIONAL =
+      SortOrder.builderFor(REPLACE_SCHEMA_OPTIONAL)
+          .asc(Expressions.bucket("id", 16))
+          .asc("id")
+          .build();
+
+  @TempDir private File tempFolder;
+  private BigQueryMetastoreCatalog catalog;
+  private BigQueryMetastoreClient fakeBigQueryMetastoreClient;
+  private String warehouseLocation;
+
+  @BeforeEach
+  public void before() throws Exception {
+    catalog = initCatalog(CATALOG_ID, ImmutableMap.of());
+  }
+
+  @AfterEach
+  public void after() throws Exception {

Review Comment:
   you might want to double-check whether cleanup is done correctly here. My local `bigquery` directory had leftover folders after running these tests



-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org

For queries about this service, please contact Infrastructure at:
users@infra.apache.org


---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For additional commands, e-mail: issues-help@iceberg.apache.org


Re: [PR] Catalog: Add BigQuery Metastore Catalog Support [iceberg]

Posted by "nastra (via GitHub)" <gi...@apache.org>.
nastra commented on code in PR #12808:
URL: https://github.com/apache/iceberg/pull/12808#discussion_r2055663935


##########
bigquery/src/main/java/org/apache/iceberg/gcp/bigquery/BigQueryTableOperations.java:
##########
@@ -0,0 +1,297 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.iceberg.gcp.bigquery;
+
+import com.google.api.services.bigquery.model.ExternalCatalogTableOptions;
+import com.google.api.services.bigquery.model.Table;
+import com.google.api.services.bigquery.model.TableReference;
+import java.util.Locale;
+import java.util.Map;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hive.common.StatsSetupConst;
+import org.apache.iceberg.BaseMetastoreOperations;
+import org.apache.iceberg.BaseMetastoreTableOperations;
+import org.apache.iceberg.SnapshotSummary;
+import org.apache.iceberg.TableMetadata;
+import org.apache.iceberg.TableProperties;
+import org.apache.iceberg.exceptions.AlreadyExistsException;
+import org.apache.iceberg.exceptions.CommitFailedException;
+import org.apache.iceberg.exceptions.CommitStateUnknownException;
+import org.apache.iceberg.exceptions.NoSuchTableException;
+import org.apache.iceberg.exceptions.ValidationException;
+import org.apache.iceberg.hadoop.ConfigProperties;
+import org.apache.iceberg.io.FileIO;
+import org.apache.iceberg.relocated.com.google.common.collect.Maps;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/** Handles BigQuery metastore table operations. */
+public final class BigQueryTableOperations extends BaseMetastoreTableOperations {
+
+  private static final Logger LOG = LoggerFactory.getLogger(BigQueryTableOperations.class);
+
+  public static final String TABLE_PROPERTIES_BQ_CONNECTION = "bq_connection";
+
+  private final BigQueryMetastoreClient client;
+  private final FileIO fileIO;
+  private final TableReference tableReference;
+  private final Configuration conf;
+
+  BigQueryTableOperations(
+      BigQueryMetastoreClient client,
+      FileIO fileIO,
+      String project,
+      String dataset,
+      String table,
+      Configuration conf) {
+    this.client = client;
+    this.fileIO = fileIO;
+    this.tableReference =

Review Comment:
   maybe just pass the `TableReference` directly to the constructor instead of having to pass three different parameters



-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org

For queries about this service, please contact Infrastructure at:
users@infra.apache.org


---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For additional commands, e-mail: issues-help@iceberg.apache.org


Re: [PR] Catalog: Add BigQuery Metastore Catalog Support [iceberg]

Posted by "nastra (via GitHub)" <gi...@apache.org>.
nastra commented on code in PR #12808:
URL: https://github.com/apache/iceberg/pull/12808#discussion_r2059736716


##########
bigquery/src/test/java/org/apache/iceberg/gcp/bigquery/BigQueryTableOperationsTest.java:
##########
@@ -0,0 +1,226 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.iceberg.gcp.bigquery;
+
+import static org.apache.iceberg.gcp.bigquery.BigQueryMetastoreCatalog.PROJECT_ID;
+import static org.assertj.core.api.Assertions.assertThat;
+import static org.assertj.core.api.Assertions.assertThatThrownBy;
+import static org.mockito.Mockito.any;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.reset;
+import static org.mockito.Mockito.times;
+import static org.mockito.Mockito.verify;
+import static org.mockito.Mockito.when;
+
+import com.google.api.services.bigquery.model.Dataset;
+import com.google.api.services.bigquery.model.DatasetReference;
+import com.google.api.services.bigquery.model.ExternalCatalogDatasetOptions;
+import com.google.api.services.bigquery.model.ExternalCatalogTableOptions;
+import com.google.api.services.bigquery.model.Table;
+import com.google.api.services.bigquery.model.TableReference;
+import java.io.File;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.iceberg.CatalogProperties;
+import org.apache.iceberg.PartitionSpec;
+import org.apache.iceberg.Schema;
+import org.apache.iceberg.catalog.TableIdentifier;
+import org.apache.iceberg.exceptions.CommitFailedException;
+import org.apache.iceberg.exceptions.NoSuchTableException;
+import org.apache.iceberg.exceptions.ValidationException;
+import org.apache.iceberg.relocated.com.google.common.collect.ImmutableMap;
+import org.apache.iceberg.types.Types;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.io.TempDir;
+import org.mockito.ArgumentCaptor;
+
+public class BigQueryTableOperationsTest {
+
+  @TempDir private File tempFolder;
+
+  private static final String GCP_PROJECT = "my-project";
+  private static final String GCP_REGION = "us";
+  private static final String DATASET_ID = "db";
+  private static final String TABLE_ID = "tbl";
+  private static final TableIdentifier SPARK_TABLE_ID = TableIdentifier.of(DATASET_ID, TABLE_ID);
+
+  private static final TableReference TABLE_REFERENCE =
+      new TableReference().setProjectId(GCP_PROJECT).setDatasetId(DATASET_ID).setTableId(TABLE_ID);
+
+  private final BigQueryMetastoreClient bigQueryMetaStoreClient =
+      mock(BigQueryMetastoreClient.class);
+
+  private BigQueryMetastoreCatalog bigQueryMetastoreCatalog;
+  private BigQueryTableOperations tableOps;
+
+  @BeforeEach
+  public void before() {
+    this.bigQueryMetastoreCatalog = new BigQueryMetastoreCatalog();
+    this.bigQueryMetastoreCatalog.setConf(new Configuration());
+    String warehouseLocation = tempFolder.toPath().resolve("hive-warehouse").toString();
+
+    bigQueryMetastoreCatalog.initialize(
+        "CATALOG_ID",
+        /* properties= */ ImmutableMap.of(

Review Comment:
   ```suggestion
           ImmutableMap.of(
   ```



-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org

For queries about this service, please contact Infrastructure at:
users@infra.apache.org


---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For additional commands, e-mail: issues-help@iceberg.apache.org


Re: [PR] Catalog: Add BigQuery Metastore Catalog Support [iceberg]

Posted by "locchipinti (via GitHub)" <gi...@apache.org>.
locchipinti commented on PR #12808:
URL: https://github.com/apache/iceberg/pull/12808#issuecomment-3073828175

   Hello @stevenzwu I mention you because I saw you handling the 1.10.0 milestone and I wanted to ask if this PR will be included in the 1.10.0 version
   Thanks!


-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org

For queries about this service, please contact Infrastructure at:
users@infra.apache.org


---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For additional commands, e-mail: issues-help@iceberg.apache.org


Re: [PR] Catalog: Add BigQuery Metastore Catalog Support [iceberg]

Posted by "ebyhr (via GitHub)" <gi...@apache.org>.
ebyhr commented on code in PR #12808:
URL: https://github.com/apache/iceberg/pull/12808#discussion_r2074891138


##########
bigquery/src/main/java/org/apache/iceberg/gcp/bigquery/BigQueryMetastoreCatalog.java:
##########
@@ -0,0 +1,385 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.iceberg.gcp.bigquery;
+
+import com.google.api.services.bigquery.model.Dataset;
+import com.google.api.services.bigquery.model.DatasetList.Datasets;
+import com.google.api.services.bigquery.model.DatasetReference;
+import com.google.api.services.bigquery.model.ExternalCatalogDatasetOptions;
+import com.google.api.services.bigquery.model.TableReference;
+import com.google.cloud.ServiceOptions;
+import com.google.cloud.bigquery.BigQueryOptions;
+import java.io.IOException;
+import java.io.UncheckedIOException;
+import java.security.GeneralSecurityException;
+import java.util.List;
+import java.util.Locale;
+import java.util.Map;
+import java.util.Objects;
+import java.util.Set;
+import org.apache.hadoop.conf.Configurable;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.iceberg.BaseMetastoreCatalog;
+import org.apache.iceberg.CatalogProperties;
+import org.apache.iceberg.CatalogUtil;
+import org.apache.iceberg.TableMetadata;
+import org.apache.iceberg.TableOperations;
+import org.apache.iceberg.catalog.Namespace;
+import org.apache.iceberg.catalog.SupportsNamespaces;
+import org.apache.iceberg.catalog.TableIdentifier;
+import org.apache.iceberg.exceptions.NoSuchNamespaceException;
+import org.apache.iceberg.exceptions.NoSuchTableException;
+import org.apache.iceberg.io.FileIO;
+import org.apache.iceberg.relocated.com.google.common.annotations.VisibleForTesting;
+import org.apache.iceberg.relocated.com.google.common.base.Preconditions;
+import org.apache.iceberg.relocated.com.google.common.base.Strings;
+import org.apache.iceberg.relocated.com.google.common.collect.ImmutableList;
+import org.apache.iceberg.relocated.com.google.common.collect.ImmutableMap;
+import org.apache.iceberg.relocated.com.google.common.collect.Maps;
+import org.apache.iceberg.util.LocationUtil;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/** Iceberg Bigquery Metastore Catalog implementation. */
+public class BigQueryMetastoreCatalog extends BaseMetastoreCatalog
+    implements SupportsNamespaces, Configurable {
+
+  // User provided properties.
+  public static final String PROJECT_ID = "gcp.bigquery.project-id";
+  public static final String GCP_LOCATION = "gcp.bigquery.location";
+  public static final String LIST_ALL_TABLES = "gcp.bigquery.list-all-tables";
+
+  private static final Logger LOG = LoggerFactory.getLogger(BigQueryMetastoreCatalog.class);
+
+  private static final String DEFAULT_GCP_LOCATION = "us";
+
+  private String catalogName;
+  private Map<String, String> catalogProperties;
+  private FileIO fileIO;
+  private Configuration conf;
+  private String projectId;
+  private String projectLocation;
+  private BigQueryMetastoreClient client;
+  private boolean listAllTables;
+  private String warehouseLocation;
+
+  public BigQueryMetastoreCatalog() {}
+
+  @Override
+  public void initialize(String name, Map<String, String> properties) {
+    Preconditions.checkArgument(
+        properties.containsKey(PROJECT_ID),
+        "Invalid GCP project: %s must be specified",
+        PROJECT_ID);

Review Comment:
   Google credentials can contain project-id in the JSON string/file as you may know. We could make this property optional if pickup credentials from JSON string/file. https://github.com/apache/iceberg/pull/12808/files#r2074868327



-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org

For queries about this service, please contact Infrastructure at:
users@infra.apache.org


---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For additional commands, e-mail: issues-help@iceberg.apache.org


Re: [PR] Catalog: Add BigQuery Metastore Catalog Support [iceberg]

Posted by "nastra (via GitHub)" <gi...@apache.org>.
nastra commented on code in PR #12808:
URL: https://github.com/apache/iceberg/pull/12808#discussion_r2063987477


##########
bigquery/src/main/java/org/apache/iceberg/gcp/bigquery/BigQueryMetastoreCatalog.java:
##########
@@ -0,0 +1,394 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.iceberg.gcp.bigquery;
+
+import com.google.api.services.bigquery.model.Dataset;
+import com.google.api.services.bigquery.model.DatasetList.Datasets;
+import com.google.api.services.bigquery.model.DatasetReference;
+import com.google.api.services.bigquery.model.ExternalCatalogDatasetOptions;
+import com.google.api.services.bigquery.model.TableReference;
+import com.google.cloud.ServiceOptions;
+import com.google.cloud.bigquery.BigQueryOptions;
+import java.io.IOException;
+import java.security.GeneralSecurityException;
+import java.util.List;
+import java.util.Locale;
+import java.util.Map;
+import java.util.Objects;
+import java.util.Set;
+import org.apache.hadoop.conf.Configurable;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.iceberg.BaseMetastoreCatalog;
+import org.apache.iceberg.CatalogProperties;
+import org.apache.iceberg.CatalogUtil;
+import org.apache.iceberg.TableMetadata;
+import org.apache.iceberg.TableOperations;
+import org.apache.iceberg.catalog.Namespace;
+import org.apache.iceberg.catalog.SupportsNamespaces;
+import org.apache.iceberg.catalog.TableIdentifier;
+import org.apache.iceberg.exceptions.AlreadyExistsException;
+import org.apache.iceberg.exceptions.NoSuchNamespaceException;
+import org.apache.iceberg.exceptions.NoSuchTableException;
+import org.apache.iceberg.exceptions.ServiceFailureException;
+import org.apache.iceberg.exceptions.ValidationException;
+import org.apache.iceberg.io.FileIO;
+import org.apache.iceberg.relocated.com.google.common.annotations.VisibleForTesting;
+import org.apache.iceberg.relocated.com.google.common.base.Preconditions;
+import org.apache.iceberg.relocated.com.google.common.base.Strings;
+import org.apache.iceberg.relocated.com.google.common.collect.ImmutableList;
+import org.apache.iceberg.relocated.com.google.common.collect.ImmutableMap;
+import org.apache.iceberg.relocated.com.google.common.collect.Maps;
+import org.apache.iceberg.util.LocationUtil;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/** Iceberg Bigquery Metastore Catalog implementation. */
+public class BigQueryMetastoreCatalog extends BaseMetastoreCatalog
+    implements SupportsNamespaces, Configurable {
+
+  // User provided properties.
+  public static final String PROJECT_ID = "gcp-project";
+  public static final String GCP_LOCATION = "gcp-location";
+  public static final String LIST_ALL_TABLES = "list-all-tables";
+
+  public static final String HIVE_METASTORE_WAREHOUSE_DIR = "hive.metastore.warehouse.dir";
+
+  private static final Logger LOG = LoggerFactory.getLogger(BigQueryMetastoreCatalog.class);
+
+  private static final String DEFAULT_GCP_LOCATION = "us";
+
+  private String catalogName;
+  private Map<String, String> catalogProperties;
+  private FileIO fileIO;
+  private Configuration conf;
+  private String projectId;
+  private String location;
+  private BigQueryMetastoreClient client;
+  private boolean listAllTables;
+
+  // Must have a no-arg constructor to be dynamically loaded
+  // initialize(String name, Map<String, String> properties) will be called to complete
+  // initialization
+  public BigQueryMetastoreCatalog() {}
+
+  @Override
+  public void initialize(String inputName, Map<String, String> properties) {
+    Preconditions.checkArgument(
+        properties.containsKey(PROJECT_ID), "GCP project must be specified");
+
+    projectId = properties.get(PROJECT_ID);
+    location = properties.getOrDefault(GCP_LOCATION, DEFAULT_GCP_LOCATION);
+
+    BigQueryOptions options =
+        BigQueryOptions.newBuilder()
+            .setProjectId(projectId)
+            .setLocation(location)
+            .setRetrySettings(ServiceOptions.getDefaultRetrySettings())
+            .build();
+
+    try {
+      client = new BigQueryMetastoreClientImpl(options);
+    } catch (IOException e) {
+      throw new ServiceFailureException(e, "Creating BigQuery client failed");
+    } catch (GeneralSecurityException e) {
+      throw new ValidationException(e, "Creating BigQuery client failed due to a security issue");
+    }
+
+    initialize(inputName, properties, projectId, location, client);
+  }
+
+  @VisibleForTesting
+  void initialize(
+      String inputName,
+      Map<String, String> properties,
+      String initialProjectId,
+      String initialLocation,
+      BigQueryMetastoreClient bigQueryMetaStoreClient) {
+    Preconditions.checkArgument(bigQueryMetaStoreClient != null, "BigQuery client can not be null");
+    this.catalogName = inputName;
+    this.catalogProperties = ImmutableMap.copyOf(properties);
+    this.projectId = initialProjectId;
+    this.location = initialLocation;
+    this.client = bigQueryMetaStoreClient;
+
+    if (this.conf == null) {
+      LOG.warn("No configuration was set, using the default environment Configuration");
+      this.conf = new Configuration();
+    }
+
+    LOG.info("Using BigQuery Metastore Iceberg Catalog: {}", inputName);
+
+    if (properties.containsKey(CatalogProperties.WAREHOUSE_LOCATION)) {
+      this.conf.set(
+          HIVE_METASTORE_WAREHOUSE_DIR,
+          LocationUtil.stripTrailingSlash(properties.get(CatalogProperties.WAREHOUSE_LOCATION)));
+    }
+
+    String fileIoImpl =
+        properties.getOrDefault(
+            CatalogProperties.FILE_IO_IMPL, "org.apache.iceberg.hadoop.HadoopFileIO");
+    this.fileIO = CatalogUtil.loadFileIO(fileIoImpl, properties, conf);
+
+    this.listAllTables = Boolean.parseBoolean(properties.getOrDefault(LIST_ALL_TABLES, "true"));
+  }
+
+  @Override
+  protected TableOperations newTableOps(TableIdentifier identifier) {
+    return new BigQueryTableOperations(client, fileIO, toTableReference(identifier), conf);
+  }
+
+  @Override
+  protected String defaultWarehouseLocation(TableIdentifier identifier) {
+    String locationUri = null;
+    DatasetReference datasetReference = toDatasetReference(identifier.namespace());
+    Dataset dataset = client.load(datasetReference);
+    if (dataset != null && dataset.getExternalCatalogDatasetOptions() != null) {
+      locationUri = dataset.getExternalCatalogDatasetOptions().getDefaultStorageLocationUri();
+    }
+
+    return String.format(
+        "%s/%s",
+        Strings.isNullOrEmpty(locationUri)
+            ? createDefaultStorageLocationUri(datasetReference.getDatasetId())
+            : LocationUtil.stripTrailingSlash(locationUri),
+        identifier.name());
+  }
+
+  @Override
+  public List<TableIdentifier> listTables(Namespace namespace) {
+    validateNamespace(namespace);
+
+    return client.list(toDatasetReference(namespace), listAllTables).stream()
+        .map(
+            table -> TableIdentifier.of(namespace.level(0), table.getTableReference().getTableId()))
+        .collect(ImmutableList.toImmutableList());
+  }
+
+  @Override
+  public boolean dropTable(TableIdentifier identifier, boolean purge) {
+    try {
+      TableOperations ops = newTableOps(identifier);
+      TableMetadata lastMetadata = ops.current();
+
+      client.delete(toTableReference(identifier));
+
+      if (purge && lastMetadata != null) {
+        CatalogUtil.dropTableData(ops.io(), lastMetadata);
+      }
+    } catch (NoSuchTableException e) {
+      return false;
+    }
+
+    return true;
+  }
+
+  @Override
+  public void renameTable(TableIdentifier from, TableIdentifier to) {
+    validateNamespace(from.namespace());
+    validateNamespace(to.namespace());
+
+    if (!namespaceExists(to.namespace())) {
+      throw new NoSuchNamespaceException("Namespace does not exist: %s", to.namespace());
+    }
+
+    if (!namespaceExists(from.namespace())) {
+      throw new NoSuchNamespaceException("Namespace does not exist: %s", from.namespace());
+    }
+
+    // Check if the source table exists
+    if (!tableExists(from)) {
+      throw new NoSuchTableException("Table does not exist: %s", from.name());
+    }
+
+    if (!from.namespace().equals(to.namespace())) {
+      throw new ValidationException("New table name must be in the same namespace");
+    }
+
+    // Check if the destination table already exists
+    if (tableExists(to)) {
+      throw new AlreadyExistsException("Table already exists: %s", to.name());
+    }
+
+    // TODO: Enable once supported by BigQuery API.
+    throw new UnsupportedOperationException("Table rename operation is unsupported.");
+  }
+
+  @Override
+  public void createNamespace(Namespace namespace, Map<String, String> metadata) {
+    Dataset builder = new Dataset();
+    DatasetReference datasetReference = toDatasetReference(namespace);
+    builder.setLocation(this.location);
+    builder.setDatasetReference(datasetReference);
+    builder.setExternalCatalogDatasetOptions(
+        BigQueryMetastoreUtils.createExternalCatalogDatasetOptions(
+            createDefaultStorageLocationUri(datasetReference.getDatasetId()), metadata));
+
+    client.create(builder);
+  }
+
+  @Override
+  public List<Namespace> listNamespaces() {
+    try {
+      return listNamespaces(Namespace.empty());
+    } catch (NoSuchNamespaceException e) {
+      return ImmutableList.of();
+    }
+  }
+
+  /**
+   * Since this catalog only supports one-level namespaces, it always returns an empty list unless
+   * passed an empty namespace to list all namespaces within the catalog.
+   */
+  @Override
+  public List<Namespace> listNamespaces(Namespace namespace) {
+    List<Datasets> allDatasets = client.list(projectId);
+
+    ImmutableList<Namespace> namespaces =
+        allDatasets.stream()
+            .map(BigQueryMetastoreCatalog::toNamespace)
+            .collect(ImmutableList.toImmutableList());
+    if (namespaces.isEmpty()) {
+      throw new NoSuchNamespaceException("Namespace does not exist: %s", namespace);
+    }
+
+    return namespaces;
+  }
+
+  @Override
+  public boolean dropNamespace(Namespace namespace) {
+    try {
+      client.delete(toDatasetReference(namespace));
+      // We don't delete the data folder for safety, which aligns with Hive Metastore's default
+      // behavior.
+      // We can support database or catalog level config controlling file deletion in the future.
+      return true;
+    } catch (NoSuchNamespaceException e) {
+      return false;
+    }
+  }
+
+  @Override
+  public boolean setProperties(Namespace namespace, Map<String, String> properties) {
+    Dataset dataset = client.load(toDatasetReference(namespace));
+
+    ExternalCatalogDatasetOptions existingOptions = dataset.getExternalCatalogDatasetOptions();
+    Map<String, String> existingParameters =
+        existingOptions != null ? existingOptions.getParameters() : null;
+
+    Map<String, String> newParameters = Maps.newHashMap();
+    if (existingParameters != null) {
+      newParameters.putAll(existingParameters);
+    }
+
+    newParameters.putAll(properties);
+
+    if (Objects.equals(existingParameters, newParameters)) {
+      // No change in parameters detected
+      return false;
+    }
+
+    client.setParameters(toDatasetReference(namespace), properties);
+    return true;
+  }
+
+  @Override
+  public boolean removeProperties(Namespace namespace, Set<String> properties) {
+    client.removeParameters(toDatasetReference(namespace), properties);
+    return true;
+  }
+
+  @SuppressWarnings("FormatStringAnnotation")
+  @Override
+  public Map<String, String> loadNamespaceMetadata(Namespace namespace) {
+    try {
+      return toMetadata(client.load(toDatasetReference(namespace)));
+    } catch (IllegalArgumentException e) {
+      throw new NoSuchNamespaceException(e.getMessage());
+    }
+  }
+
+  @Override
+  public String name() {
+    return catalogName;
+  }
+
+  @Override
+  protected Map<String, String> properties() {
+    return catalogProperties == null ? ImmutableMap.of() : catalogProperties;
+  }
+
+  @Override
+  public void setConf(Configuration conf) {
+    this.conf = new Configuration(conf);
+  }
+
+  @Override
+  public Configuration getConf() {
+    return this.conf;
+  }
+
+  private String createDefaultStorageLocationUri(String dbId) {
+    String warehouseLocation = conf.get(HIVE_METASTORE_WAREHOUSE_DIR);
+    Preconditions.checkNotNull(warehouseLocation, "Data warehouse location is not set");
+    return String.format("%s/%s.db", LocationUtil.stripTrailingSlash(warehouseLocation), dbId);
+  }
+
+  private static Namespace toNamespace(Datasets dataset) {
+    return Namespace.of(dataset.getDatasetReference().getDatasetId());
+  }
+
+  private DatasetReference toDatasetReference(Namespace namespace) {
+    validateNamespace(namespace);
+    return new DatasetReference().setProjectId(projectId).setDatasetId(namespace.level(0));
+  }
+
+  private TableReference toTableReference(TableIdentifier tableIdentifier) {
+    DatasetReference datasetReference = toDatasetReference(tableIdentifier.namespace());
+    return new TableReference()
+        .setProjectId(datasetReference.getProjectId())
+        .setDatasetId(datasetReference.getDatasetId())
+        .setTableId(tableIdentifier.name());
+  }
+
+  private Map<String, String> toMetadata(Dataset dataset) {
+    ExternalCatalogDatasetOptions options = dataset.getExternalCatalogDatasetOptions();
+    Map<String, String> metadata = Maps.newHashMap();
+    if (options != null) {
+      if (options.getParameters() != null) {
+        metadata.putAll(options.getParameters());
+      }

Review Comment:
   nit: newline after this



-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org

For queries about this service, please contact Infrastructure at:
users@infra.apache.org


---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For additional commands, e-mail: issues-help@iceberg.apache.org


Re: [PR] Catalog: Add BigQuery Metastore Catalog Support [iceberg]

Posted by "talatuyarer (via GitHub)" <gi...@apache.org>.
talatuyarer commented on code in PR #12808:
URL: https://github.com/apache/iceberg/pull/12808#discussion_r2067701528


##########
bigquery/src/main/java/org/apache/iceberg/gcp/bigquery/BigQueryMetastoreCatalog.java:
##########
@@ -0,0 +1,405 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.iceberg.gcp.bigquery;
+
+import com.google.api.services.bigquery.model.Dataset;
+import com.google.api.services.bigquery.model.DatasetList.Datasets;
+import com.google.api.services.bigquery.model.DatasetReference;
+import com.google.api.services.bigquery.model.ExternalCatalogDatasetOptions;
+import com.google.api.services.bigquery.model.TableReference;
+import com.google.cloud.ServiceOptions;
+import com.google.cloud.bigquery.BigQueryOptions;
+import java.io.IOException;
+import java.security.GeneralSecurityException;
+import java.util.List;
+import java.util.Locale;
+import java.util.Map;
+import java.util.Objects;
+import java.util.Set;
+import org.apache.hadoop.conf.Configurable;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.iceberg.BaseMetastoreCatalog;
+import org.apache.iceberg.CatalogProperties;
+import org.apache.iceberg.CatalogUtil;
+import org.apache.iceberg.TableMetadata;
+import org.apache.iceberg.TableOperations;
+import org.apache.iceberg.catalog.Namespace;
+import org.apache.iceberg.catalog.SupportsNamespaces;
+import org.apache.iceberg.catalog.TableIdentifier;
+import org.apache.iceberg.exceptions.AlreadyExistsException;
+import org.apache.iceberg.exceptions.NoSuchNamespaceException;
+import org.apache.iceberg.exceptions.NoSuchTableException;
+import org.apache.iceberg.exceptions.ServiceFailureException;
+import org.apache.iceberg.exceptions.ValidationException;
+import org.apache.iceberg.io.FileIO;
+import org.apache.iceberg.relocated.com.google.common.annotations.VisibleForTesting;
+import org.apache.iceberg.relocated.com.google.common.base.Preconditions;
+import org.apache.iceberg.relocated.com.google.common.base.Strings;
+import org.apache.iceberg.relocated.com.google.common.collect.ImmutableList;
+import org.apache.iceberg.relocated.com.google.common.collect.ImmutableMap;
+import org.apache.iceberg.relocated.com.google.common.collect.Maps;
+import org.apache.iceberg.util.LocationUtil;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/** Iceberg Bigquery Metastore Catalog implementation. */
+public class BigQueryMetastoreCatalog extends BaseMetastoreCatalog
+    implements SupportsNamespaces, Configurable {
+
+  // User provided properties.
+  public static final String PROJECT_ID = "gcp-project";

Review Comment:
   Switched suggested format.



-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org

For queries about this service, please contact Infrastructure at:
users@infra.apache.org


---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For additional commands, e-mail: issues-help@iceberg.apache.org


Re: [PR] Catalog: Add BigQuery Metastore Catalog Support [iceberg]

Posted by "nastra (via GitHub)" <gi...@apache.org>.
nastra commented on code in PR #12808:
URL: https://github.com/apache/iceberg/pull/12808#discussion_r2067907024


##########
bigquery/src/test/java/org/apache/iceberg/gcp/bigquery/TestBigQueryTableOperations.java:
##########
@@ -0,0 +1,291 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.iceberg.gcp.bigquery;
+
+import static org.apache.iceberg.gcp.bigquery.BigQueryMetastoreCatalog.PROJECT_ID;
+import static org.apache.iceberg.types.Types.NestedField.required;
+import static org.assertj.core.api.Assertions.assertThat;
+import static org.assertj.core.api.Assertions.assertThatThrownBy;
+import static org.mockito.Mockito.any;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.reset;
+import static org.mockito.Mockito.times;
+import static org.mockito.Mockito.verify;
+import static org.mockito.Mockito.when;
+
+import com.google.api.services.bigquery.model.Dataset;
+import com.google.api.services.bigquery.model.DatasetReference;
+import com.google.api.services.bigquery.model.ExternalCatalogDatasetOptions;
+import com.google.api.services.bigquery.model.ExternalCatalogTableOptions;
+import com.google.api.services.bigquery.model.StorageDescriptor;
+import com.google.api.services.bigquery.model.Table;
+import com.google.api.services.bigquery.model.TableReference;
+import java.io.File;
+import java.io.IOException;
+import java.util.Collections;
+import java.util.Optional;
+import org.apache.commons.io.FileUtils;
+import org.apache.commons.io.filefilter.TrueFileFilter;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.iceberg.CatalogProperties;
+import org.apache.iceberg.PartitionSpec;
+import org.apache.iceberg.Schema;
+import org.apache.iceberg.catalog.TableIdentifier;
+import org.apache.iceberg.exceptions.CommitFailedException;
+import org.apache.iceberg.exceptions.NoSuchTableException;
+import org.apache.iceberg.exceptions.ValidationException;
+import org.apache.iceberg.relocated.com.google.common.collect.ImmutableMap;
+import org.apache.iceberg.types.Types;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.io.TempDir;
+import org.mockito.ArgumentCaptor;
+
+public class TestBigQueryTableOperations {
+
+  @TempDir private File tempFolder;
+  public static final String METADATA_LOCATION_PROP = "metadata_location";
+  private static final String GCP_PROJECT = "my-project";
+  private static final String GCP_REGION = "us";
+  private static final String NS = "db";
+  private static final String TABLE = "tbl";
+  private static final TableIdentifier IDENTIFIER = TableIdentifier.of(NS, TABLE);
+
+  private static final TableReference TABLE_REFERENCE =
+      new TableReference().setProjectId(GCP_PROJECT).setDatasetId(NS).setTableId(TABLE);
+
+  private static final Schema SCHEMA =
+      new Schema(
+          required(1, "id", Types.IntegerType.get(), "unique ID"),
+          required(2, "data", Types.StringType.get()));
+
+  private final BigQueryMetastoreClient client = mock(BigQueryMetastoreClient.class);
+
+  private BigQueryMetastoreCatalog catalog;
+  private BigQueryTableOperations tableOps;
+
+  @BeforeEach
+  public void before() {
+    this.catalog = new BigQueryMetastoreCatalog();
+    this.catalog.setConf(new Configuration());
+    String warehouseLocation = tempFolder.toPath().resolve("hive-warehouse").toString();
+
+    catalog.initialize(
+        "CATALOG_ID",
+        ImmutableMap.of(
+            PROJECT_ID,
+            GCP_PROJECT,
+            CatalogProperties.WAREHOUSE_LOCATION,
+            warehouseLocation,
+            CatalogProperties.FILE_IO_IMPL,
+            "org.apache.iceberg.hadoop.HadoopFileIO"),
+        GCP_PROJECT,
+        GCP_REGION,
+        client);
+    this.tableOps = (BigQueryTableOperations) catalog.newTableOps(IDENTIFIER);
+  }
+
+  @Test
+  public void fetchLatestMetadataFromBigQuery() throws Exception {
+    Table createdTable = createTestTable();
+    reset(client);
+    when(client.load(TABLE_REFERENCE)).thenReturn(createdTable);
+
+    tableOps.refresh();
+    assertThat(
+            createdTable
+                .getExternalCatalogTableOptions()
+                .getParameters()
+                .getOrDefault(METADATA_LOCATION_PROP, ""))
+        .isEqualTo(tableOps.currentMetadataLocation());
+
+    reset(client);
+    when(client.load(TABLE_REFERENCE))
+        .thenThrow(new NoSuchTableException("error message getTable"));
+    // Refresh fails when table is not found but metadata already presents.
+    assertThatThrownBy(() -> tableOps.refresh())
+        .isInstanceOf(NoSuchTableException.class)
+        .hasMessageContaining("error message getTable");
+  }
+
+  @Test
+  public void loadNonIcebergTableFails() {
+    when(client.load(TABLE_REFERENCE)).thenReturn(new Table().setTableReference(TABLE_REFERENCE));
+
+    assertThatThrownBy(() -> tableOps.refresh())
+        .isInstanceOf(ValidationException.class)
+        .hasMessageContaining("metadata location not found");
+  }
+
+  @Test
+  public void loadNoOpWhenMetadataAndTableNotFound() {
+    when(client.load(TABLE_REFERENCE))
+        .thenThrow(new NoSuchTableException("error message getTable"));
+    // Table not found won't cause errors when the metadata is null.
+    assertThat(tableOps.currentMetadataLocation()).isNull();

Review Comment:
   previously you had `tableOps.refresh()` after this line and I think you wanted to keep it as otherwise the NSTE won't be thrown. That's also why I mentioned that it would be better to wrap this in a `assertThatNoException().isThrownBy(..)` to make it explicit about what's expected in this test



-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org

For queries about this service, please contact Infrastructure at:
users@infra.apache.org


---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For additional commands, e-mail: issues-help@iceberg.apache.org


Re: [PR] Catalog: Add BigQuery Metastore Catalog Support [iceberg]

Posted by "danielcweeks (via GitHub)" <gi...@apache.org>.
danielcweeks commented on code in PR #12808:
URL: https://github.com/apache/iceberg/pull/12808#discussion_r2067277897


##########
bigquery/src/main/java/org/apache/iceberg/gcp/bigquery/BigQueryMetastoreCatalog.java:
##########
@@ -0,0 +1,405 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.iceberg.gcp.bigquery;
+
+import com.google.api.services.bigquery.model.Dataset;
+import com.google.api.services.bigquery.model.DatasetList.Datasets;
+import com.google.api.services.bigquery.model.DatasetReference;
+import com.google.api.services.bigquery.model.ExternalCatalogDatasetOptions;
+import com.google.api.services.bigquery.model.TableReference;
+import com.google.cloud.ServiceOptions;
+import com.google.cloud.bigquery.BigQueryOptions;
+import java.io.IOException;
+import java.security.GeneralSecurityException;
+import java.util.List;
+import java.util.Locale;
+import java.util.Map;
+import java.util.Objects;
+import java.util.Set;
+import org.apache.hadoop.conf.Configurable;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.iceberg.BaseMetastoreCatalog;
+import org.apache.iceberg.CatalogProperties;
+import org.apache.iceberg.CatalogUtil;
+import org.apache.iceberg.TableMetadata;
+import org.apache.iceberg.TableOperations;
+import org.apache.iceberg.catalog.Namespace;
+import org.apache.iceberg.catalog.SupportsNamespaces;
+import org.apache.iceberg.catalog.TableIdentifier;
+import org.apache.iceberg.exceptions.AlreadyExistsException;
+import org.apache.iceberg.exceptions.NoSuchNamespaceException;
+import org.apache.iceberg.exceptions.NoSuchTableException;
+import org.apache.iceberg.exceptions.ServiceFailureException;
+import org.apache.iceberg.exceptions.ValidationException;
+import org.apache.iceberg.io.FileIO;
+import org.apache.iceberg.relocated.com.google.common.annotations.VisibleForTesting;
+import org.apache.iceberg.relocated.com.google.common.base.Preconditions;
+import org.apache.iceberg.relocated.com.google.common.base.Strings;
+import org.apache.iceberg.relocated.com.google.common.collect.ImmutableList;
+import org.apache.iceberg.relocated.com.google.common.collect.ImmutableMap;
+import org.apache.iceberg.relocated.com.google.common.collect.Maps;
+import org.apache.iceberg.util.LocationUtil;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/** Iceberg Bigquery Metastore Catalog implementation. */
+public class BigQueryMetastoreCatalog extends BaseMetastoreCatalog
+    implements SupportsNamespaces, Configurable {
+
+  // User provided properties.
+  public static final String PROJECT_ID = "gcp-project";

Review Comment:
   I feel like we should align these properties with the GCPProperties configuration.  We're using a different syntax and I'm not sure if this needs to align or needs to be separate from what is used for GCS access (e.g. `gcp.bigquery.project-id`)?



-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org

For queries about this service, please contact Infrastructure at:
users@infra.apache.org


---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For additional commands, e-mail: issues-help@iceberg.apache.org


Re: [PR] Catalog: Add BigQuery Metastore Catalog Support [iceberg]

Posted by "danielcweeks (via GitHub)" <gi...@apache.org>.
danielcweeks commented on code in PR #12808:
URL: https://github.com/apache/iceberg/pull/12808#discussion_r2067311677


##########
gradle/libs.versions.toml:
##########
@@ -51,6 +51,7 @@ guava = "33.4.7-jre"
 hadoop3 = "3.4.1"
 httpcomponents-httpclient5 = "5.4.3"
 hive2 = { strictly = "2.3.10"} # see rich version usage explanation above
+hive4 = { strictly = "4.0.1"}

Review Comment:
   Is there a specific dependency on this hive version?  If this is strictly for big query, I think we need to scope it so that there isn't confusion with the hive module and potential dependencies there (e.g. `bigquery-hive4`)



-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org

For queries about this service, please contact Infrastructure at:
users@infra.apache.org


---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For additional commands, e-mail: issues-help@iceberg.apache.org


Re: [PR] Catalog: Add BigQuery Metastore Catalog Support [iceberg]

Posted by "talatuyarer (via GitHub)" <gi...@apache.org>.
talatuyarer commented on code in PR #12808:
URL: https://github.com/apache/iceberg/pull/12808#discussion_r2067703309


##########
bigquery/src/main/java/org/apache/iceberg/gcp/bigquery/BigQueryMetastoreUtils.java:
##########
@@ -0,0 +1,95 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.iceberg.gcp.bigquery;
+
+import com.google.api.services.bigquery.model.ExternalCatalogDatasetOptions;
+import com.google.api.services.bigquery.model.ExternalCatalogTableOptions;
+import com.google.api.services.bigquery.model.SerDeInfo;
+import com.google.api.services.bigquery.model.StorageDescriptor;
+import java.util.Map;
+
+/** Shared utilities for BigQuery Metastore specific functions and constants. */
+public final class BigQueryMetastoreUtils {
+
+  private BigQueryMetastoreUtils() {}
+
+  public static final String HIVE_SERIALIZATION_LIBRARY =

Review Comment:
   No need. I changed to private



-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org

For queries about this service, please contact Infrastructure at:
users@infra.apache.org


---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For additional commands, e-mail: issues-help@iceberg.apache.org


Re: [PR] Catalog: Add BigQuery Metastore Catalog Support [iceberg]

Posted by "nastra (via GitHub)" <gi...@apache.org>.
nastra commented on code in PR #12808:
URL: https://github.com/apache/iceberg/pull/12808#discussion_r2066009039


##########
bigquery/src/main/java/org/apache/iceberg/gcp/bigquery/BigQueryMetastoreClientImpl.java:
##########
@@ -0,0 +1,594 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.iceberg.gcp.bigquery;
+
+import com.google.api.client.googleapis.javanet.GoogleNetHttpTransport;
+import com.google.api.client.http.HttpHeaders;
+import com.google.api.client.http.HttpResponse;
+import com.google.api.client.http.HttpResponseException;
+import com.google.api.client.http.HttpStatusCodes;
+import com.google.api.client.json.gson.GsonFactory;
+import com.google.api.client.util.Data;
+import com.google.api.services.bigquery.Bigquery;
+import com.google.api.services.bigquery.BigqueryScopes;
+import com.google.api.services.bigquery.model.Dataset;
+import com.google.api.services.bigquery.model.DatasetList;
+import com.google.api.services.bigquery.model.DatasetList.Datasets;
+import com.google.api.services.bigquery.model.DatasetReference;
+import com.google.api.services.bigquery.model.ExternalCatalogDatasetOptions;
+import com.google.api.services.bigquery.model.ExternalCatalogTableOptions;
+import com.google.api.services.bigquery.model.Table;
+import com.google.api.services.bigquery.model.TableList;
+import com.google.api.services.bigquery.model.TableList.Tables;
+import com.google.api.services.bigquery.model.TableReference;
+import com.google.api.services.bigquery.model.TableSchema;
+import com.google.auth.http.HttpCredentialsAdapter;
+import com.google.auth.oauth2.GoogleCredentials;
+import com.google.cloud.BaseServiceException;
+import com.google.cloud.ExceptionHandler;
+import com.google.cloud.bigquery.BigQueryErrorMessages;
+import com.google.cloud.bigquery.BigQueryOptions;
+import com.google.cloud.bigquery.BigQueryRetryConfig;
+import com.google.cloud.bigquery.BigQueryRetryHelper;
+import java.io.IOException;
+import java.nio.charset.StandardCharsets;
+import java.security.GeneralSecurityException;
+import java.util.List;
+import java.util.Locale;
+import java.util.Map;
+import java.util.Set;
+import java.util.stream.Collectors;
+import java.util.stream.Stream;
+import org.apache.iceberg.BaseMetastoreTableOperations;
+import org.apache.iceberg.exceptions.AlreadyExistsException;
+import org.apache.iceberg.exceptions.BadRequestException;
+import org.apache.iceberg.exceptions.ForbiddenException;
+import org.apache.iceberg.exceptions.NoSuchIcebergTableException;
+import org.apache.iceberg.exceptions.NoSuchNamespaceException;
+import org.apache.iceberg.exceptions.NoSuchTableException;
+import org.apache.iceberg.exceptions.NotAuthorizedException;
+import org.apache.iceberg.exceptions.RuntimeIOException;
+import org.apache.iceberg.exceptions.ServiceFailureException;
+import org.apache.iceberg.exceptions.ServiceUnavailableException;
+import org.apache.iceberg.exceptions.ValidationException;
+import org.apache.iceberg.relocated.com.google.common.base.Preconditions;
+import org.apache.iceberg.relocated.com.google.common.collect.Lists;
+import org.apache.iceberg.relocated.com.google.common.collect.Maps;
+
+/** A client of Google Bigquery Metastore functions over the BigQuery service. */
+public final class BigQueryMetastoreClientImpl implements BigQueryMetastoreClient {
+
+  public static final ExceptionHandler.Interceptor EXCEPTION_HANDLER_INTERCEPTOR =
+      new ExceptionHandler.Interceptor() {
+
+        private static final long serialVersionUID = -8429573486870467828L;
+
+        @Override
+        public RetryResult afterEval(Exception exception, RetryResult retryResult) {
+          return ExceptionHandler.Interceptor.RetryResult.CONTINUE_EVALUATION;
+        }
+
+        @Override
+        public RetryResult beforeEval(Exception exception) {
+          if (exception instanceof BaseServiceException) {
+            boolean retriable = ((BaseServiceException) exception).isRetryable();
+            return retriable
+                ? ExceptionHandler.Interceptor.RetryResult.RETRY
+                : ExceptionHandler.Interceptor.RetryResult.CONTINUE_EVALUATION;
+          }
+
+          return ExceptionHandler.Interceptor.RetryResult.CONTINUE_EVALUATION;
+        }
+      };
+
+  private final Bigquery client;
+  private final BigQueryOptions bigqueryOptions;
+
+  // Retry config with error messages and regex for rate limit exceeded errors.
+  private static final BigQueryRetryConfig DEFAULT_RETRY_CONFIG =
+      BigQueryRetryConfig.newBuilder()
+          .retryOnMessage(BigQueryErrorMessages.RATE_LIMIT_EXCEEDED_MSG)
+          .retryOnMessage(BigQueryErrorMessages.JOB_RATE_LIMIT_EXCEEDED_MSG)
+          .retryOnRegEx(BigQueryErrorMessages.RetryRegExPatterns.RATE_LIMIT_EXCEEDED_REGEX)
+          .build();
+
+  public static final ExceptionHandler BIGQUERY_EXCEPTION_HANDLER =
+      ExceptionHandler.newBuilder()
+          .abortOn(RuntimeException.class)
+          .retryOn(
+              java.net.ConnectException
+                  .class) // Retry on connection failures due to transient network issues.
+          .retryOn(
+              java.net.UnknownHostException
+                  .class) // Retry to recover from temporary DNS resolution failures.
+          .addInterceptors(EXCEPTION_HANDLER_INTERCEPTOR)
+          .build();
+
+  /** Constructs a client of the Google BigQuery service. */
+  public BigQueryMetastoreClientImpl(BigQueryOptions options)
+      throws IOException, GeneralSecurityException {
+    // Initialize client that will be used to send requests. This client only needs to be created
+    // once, and can be reused for multiple requests
+    HttpCredentialsAdapter httpCredentialsAdapter =
+        new HttpCredentialsAdapter(
+            GoogleCredentials.getApplicationDefault().createScoped(BigqueryScopes.all()));
+    this.client =
+        new Bigquery.Builder(
+                GoogleNetHttpTransport.newTrustedTransport(),
+                GsonFactory.getDefaultInstance(),
+                httpRequest -> {
+                  httpCredentialsAdapter.initialize(httpRequest);
+                  // Instead of throwing exceptions, analyze the HttpResponse object and inspect its
+                  // status code. This will allow BigQuery API errors to be converted into Iceberg
+                  // exceptions.
+                  httpRequest.setThrowExceptionOnExecuteError(false);
+                })
+            .setApplicationName("BigQuery Metastore Iceberg Catalog Plugin")
+            .build();
+    this.bigqueryOptions = options;
+  }
+
+  @Override
+  public Dataset create(Dataset dataset) {
+    Dataset response = null;
+    try {
+      response =
+          BigQueryRetryHelper.runWithRetries(
+              () -> internalCreate(dataset),
+              bigqueryOptions.getRetrySettings(),
+              BIGQUERY_EXCEPTION_HANDLER,
+              bigqueryOptions.getClock(),
+              DEFAULT_RETRY_CONFIG);
+    } catch (BigQueryRetryHelper.BigQueryRetryHelperException e) {
+      handleBigQueryRetryException(e);
+    }
+    return response;
+  }
+
+  @SuppressWarnings("FormatStringAnnotation")

Review Comment:
   no need to suppress this as you can fix it by updating the exception block to `throw new AlreadyExistsException("Namespace already exists: %s", dataset.getId());`



##########
bigquery/src/main/java/org/apache/iceberg/gcp/bigquery/BigQueryMetastoreClientImpl.java:
##########
@@ -0,0 +1,594 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.iceberg.gcp.bigquery;
+
+import com.google.api.client.googleapis.javanet.GoogleNetHttpTransport;
+import com.google.api.client.http.HttpHeaders;
+import com.google.api.client.http.HttpResponse;
+import com.google.api.client.http.HttpResponseException;
+import com.google.api.client.http.HttpStatusCodes;
+import com.google.api.client.json.gson.GsonFactory;
+import com.google.api.client.util.Data;
+import com.google.api.services.bigquery.Bigquery;
+import com.google.api.services.bigquery.BigqueryScopes;
+import com.google.api.services.bigquery.model.Dataset;
+import com.google.api.services.bigquery.model.DatasetList;
+import com.google.api.services.bigquery.model.DatasetList.Datasets;
+import com.google.api.services.bigquery.model.DatasetReference;
+import com.google.api.services.bigquery.model.ExternalCatalogDatasetOptions;
+import com.google.api.services.bigquery.model.ExternalCatalogTableOptions;
+import com.google.api.services.bigquery.model.Table;
+import com.google.api.services.bigquery.model.TableList;
+import com.google.api.services.bigquery.model.TableList.Tables;
+import com.google.api.services.bigquery.model.TableReference;
+import com.google.api.services.bigquery.model.TableSchema;
+import com.google.auth.http.HttpCredentialsAdapter;
+import com.google.auth.oauth2.GoogleCredentials;
+import com.google.cloud.BaseServiceException;
+import com.google.cloud.ExceptionHandler;
+import com.google.cloud.bigquery.BigQueryErrorMessages;
+import com.google.cloud.bigquery.BigQueryOptions;
+import com.google.cloud.bigquery.BigQueryRetryConfig;
+import com.google.cloud.bigquery.BigQueryRetryHelper;
+import java.io.IOException;
+import java.nio.charset.StandardCharsets;
+import java.security.GeneralSecurityException;
+import java.util.List;
+import java.util.Locale;
+import java.util.Map;
+import java.util.Set;
+import java.util.stream.Collectors;
+import java.util.stream.Stream;
+import org.apache.iceberg.BaseMetastoreTableOperations;
+import org.apache.iceberg.exceptions.AlreadyExistsException;
+import org.apache.iceberg.exceptions.BadRequestException;
+import org.apache.iceberg.exceptions.ForbiddenException;
+import org.apache.iceberg.exceptions.NoSuchIcebergTableException;
+import org.apache.iceberg.exceptions.NoSuchNamespaceException;
+import org.apache.iceberg.exceptions.NoSuchTableException;
+import org.apache.iceberg.exceptions.NotAuthorizedException;
+import org.apache.iceberg.exceptions.RuntimeIOException;
+import org.apache.iceberg.exceptions.ServiceFailureException;
+import org.apache.iceberg.exceptions.ServiceUnavailableException;
+import org.apache.iceberg.exceptions.ValidationException;
+import org.apache.iceberg.relocated.com.google.common.base.Preconditions;
+import org.apache.iceberg.relocated.com.google.common.collect.Lists;
+import org.apache.iceberg.relocated.com.google.common.collect.Maps;
+
+/** A client of Google Bigquery Metastore functions over the BigQuery service. */
+public final class BigQueryMetastoreClientImpl implements BigQueryMetastoreClient {
+
+  public static final ExceptionHandler.Interceptor EXCEPTION_HANDLER_INTERCEPTOR =
+      new ExceptionHandler.Interceptor() {
+
+        private static final long serialVersionUID = -8429573486870467828L;
+
+        @Override
+        public RetryResult afterEval(Exception exception, RetryResult retryResult) {
+          return ExceptionHandler.Interceptor.RetryResult.CONTINUE_EVALUATION;
+        }
+
+        @Override
+        public RetryResult beforeEval(Exception exception) {
+          if (exception instanceof BaseServiceException) {
+            boolean retriable = ((BaseServiceException) exception).isRetryable();
+            return retriable
+                ? ExceptionHandler.Interceptor.RetryResult.RETRY
+                : ExceptionHandler.Interceptor.RetryResult.CONTINUE_EVALUATION;
+          }
+
+          return ExceptionHandler.Interceptor.RetryResult.CONTINUE_EVALUATION;
+        }
+      };
+
+  private final Bigquery client;
+  private final BigQueryOptions bigqueryOptions;
+
+  // Retry config with error messages and regex for rate limit exceeded errors.
+  private static final BigQueryRetryConfig DEFAULT_RETRY_CONFIG =
+      BigQueryRetryConfig.newBuilder()
+          .retryOnMessage(BigQueryErrorMessages.RATE_LIMIT_EXCEEDED_MSG)
+          .retryOnMessage(BigQueryErrorMessages.JOB_RATE_LIMIT_EXCEEDED_MSG)
+          .retryOnRegEx(BigQueryErrorMessages.RetryRegExPatterns.RATE_LIMIT_EXCEEDED_REGEX)
+          .build();
+
+  public static final ExceptionHandler BIGQUERY_EXCEPTION_HANDLER =
+      ExceptionHandler.newBuilder()
+          .abortOn(RuntimeException.class)
+          .retryOn(
+              java.net.ConnectException
+                  .class) // Retry on connection failures due to transient network issues.
+          .retryOn(
+              java.net.UnknownHostException
+                  .class) // Retry to recover from temporary DNS resolution failures.
+          .addInterceptors(EXCEPTION_HANDLER_INTERCEPTOR)
+          .build();
+
+  /** Constructs a client of the Google BigQuery service. */
+  public BigQueryMetastoreClientImpl(BigQueryOptions options)
+      throws IOException, GeneralSecurityException {
+    // Initialize client that will be used to send requests. This client only needs to be created
+    // once, and can be reused for multiple requests
+    HttpCredentialsAdapter httpCredentialsAdapter =
+        new HttpCredentialsAdapter(
+            GoogleCredentials.getApplicationDefault().createScoped(BigqueryScopes.all()));
+    this.client =
+        new Bigquery.Builder(
+                GoogleNetHttpTransport.newTrustedTransport(),
+                GsonFactory.getDefaultInstance(),
+                httpRequest -> {
+                  httpCredentialsAdapter.initialize(httpRequest);
+                  // Instead of throwing exceptions, analyze the HttpResponse object and inspect its
+                  // status code. This will allow BigQuery API errors to be converted into Iceberg
+                  // exceptions.
+                  httpRequest.setThrowExceptionOnExecuteError(false);
+                })
+            .setApplicationName("BigQuery Metastore Iceberg Catalog Plugin")
+            .build();
+    this.bigqueryOptions = options;
+  }
+
+  @Override
+  public Dataset create(Dataset dataset) {
+    Dataset response = null;
+    try {
+      response =
+          BigQueryRetryHelper.runWithRetries(
+              () -> internalCreate(dataset),
+              bigqueryOptions.getRetrySettings(),
+              BIGQUERY_EXCEPTION_HANDLER,
+              bigqueryOptions.getClock(),
+              DEFAULT_RETRY_CONFIG);
+    } catch (BigQueryRetryHelper.BigQueryRetryHelperException e) {
+      handleBigQueryRetryException(e);
+    }
+    return response;
+  }
+
+  @SuppressWarnings("FormatStringAnnotation")
+  private Dataset internalCreate(Dataset dataset) {
+    try {
+      HttpResponse response =
+          client
+              .datasets()
+              .insert(dataset.getDatasetReference().getProjectId(), dataset)
+              .executeUnparsed();
+      return convertExceptionIfUnsuccessful(response).parseAs(Dataset.class);
+    } catch (IOException e) {
+      throw new RuntimeIOException(e);
+    } catch (AlreadyExistsException e) {
+      throw new AlreadyExistsException("Namespace already exists: " + dataset.getId());
+    }
+  }
+
+  @Override
+  @SuppressWarnings("FormatStringAnnotation")

Review Comment:
   same as above



-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org

For queries about this service, please contact Infrastructure at:
users@infra.apache.org


---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For additional commands, e-mail: issues-help@iceberg.apache.org