You are viewing a plain text version of this content. The canonical link for it is here.
Posted to issues@polaris.apache.org by "flyrain (via GitHub)" <gi...@apache.org> on 2026/08/18 05:49:31 UTC
Re: [PR] Support encrypted table operations and server-side purge [polaris]
flyrain commented on code in PR #5060:
URL: https://github.com/apache/polaris/pull/5060#discussion_r3797937174
##########
runtime/service/src/main/java/org/apache/polaris/service/catalog/io/CleanupTaskEncryption.java:
##########
@@ -0,0 +1,196 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.polaris.service.catalog.io;
+
+import com.fasterxml.jackson.core.JsonProcessingException;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.fasterxml.jackson.dataformat.smile.databind.SmileMapper;
+import java.io.Closeable;
+import java.io.IOException;
+import java.util.Base64;
+import java.util.List;
+import java.util.Map;
+import org.apache.iceberg.CatalogProperties;
+import org.apache.iceberg.EncryptedKeyParser;
+import org.apache.iceberg.TableMetadata;
+import org.apache.iceberg.TableProperties;
+import org.apache.iceberg.encryption.EncryptedKey;
+import org.apache.iceberg.encryption.EncryptingFileIO;
+import org.apache.iceberg.encryption.EncryptionManager;
+import org.apache.iceberg.encryption.EncryptionUtil;
+import org.apache.iceberg.encryption.KeyManagementClient;
+import org.apache.iceberg.encryption.StandardEncryptionManager;
+import org.apache.iceberg.io.FileIO;
+import org.apache.iceberg.util.PropertyUtil;
+import org.apache.polaris.core.entity.PolarisTaskConstants;
+
+/** Prepares and applies the encryption context used by Iceberg cleanup tasks. */
+public final class CleanupTaskEncryption {
+
+ private static final String CONTEXT_FORMAT_SMILE_V1 = "EC1";
+ private static final ObjectMapper SMILE_MAPPER = SmileMapper.builder().build();
+
+ private CleanupTaskEncryption() {}
+
+ /**
+ * Copies the catalog KMS configuration and the table's wrapped encryption keys into a cleanup
+ * task. Cleanup tasks run after the catalog request has ended, so they cannot reconstruct these
+ * values from the live table operations object.
+ */
+ public static void setupCleanupTaskEncryptionProperties(
+ Map<String, String> taskProperties,
+ Map<String, String> catalogProperties,
+ TableMetadata metadata) {
+ // Task properties start with table metadata. Always discard a table-controlled value for this
+ // reserved task field, including when the table is not encrypted.
+ taskProperties.remove(PolarisTaskConstants.ENCRYPTION_CONTEXT);
+
+ if (metadata == null
+ || metadata.properties().get(TableProperties.ENCRYPTION_TABLE_KEY) == null) {
+ return;
+ }
+
+ // Keep trusted catalog KMS settings and wrapped table keys in one task-only envelope. A custom
+ // KeyManagementClient may use additional catalog properties, but these values must not be
+ // merged into the FileIO and storage properties already resolved for the cleanup task.
+ CleanupTaskEncryptionContext context =
+ new CleanupTaskEncryptionContext(
+ catalogProperties,
Review Comment:
I don't think it's the right move to consider every Polaris catalog property or even any Polaris catalog property as `kmsProperties`. In fact, there are efforts(https://github.com/apache/iceberg/pull/17155/) in the Iceberg community to standardize kms client properties, including vended kms credentials. We may need to specify the kms config as well as kms credentials before this PR. cc @singhpk234
##########
runtime/service/src/main/java/org/apache/polaris/service/catalog/io/CleanupTaskEncryption.java:
##########
@@ -0,0 +1,196 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.polaris.service.catalog.io;
+
+import com.fasterxml.jackson.core.JsonProcessingException;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.fasterxml.jackson.dataformat.smile.databind.SmileMapper;
+import java.io.Closeable;
+import java.io.IOException;
+import java.util.Base64;
+import java.util.List;
+import java.util.Map;
+import org.apache.iceberg.CatalogProperties;
+import org.apache.iceberg.EncryptedKeyParser;
+import org.apache.iceberg.TableMetadata;
+import org.apache.iceberg.TableProperties;
+import org.apache.iceberg.encryption.EncryptedKey;
+import org.apache.iceberg.encryption.EncryptingFileIO;
+import org.apache.iceberg.encryption.EncryptionManager;
+import org.apache.iceberg.encryption.EncryptionUtil;
+import org.apache.iceberg.encryption.KeyManagementClient;
+import org.apache.iceberg.encryption.StandardEncryptionManager;
+import org.apache.iceberg.io.FileIO;
+import org.apache.iceberg.util.PropertyUtil;
+import org.apache.polaris.core.entity.PolarisTaskConstants;
+
+/** Prepares and applies the encryption context used by Iceberg cleanup tasks. */
+public final class CleanupTaskEncryption {
Review Comment:
Can we move this class to package `org.apache.polaris.service.task` given that the class is mainly used in the task related classes?
##########
runtime/service/src/main/java/org/apache/polaris/service/catalog/io/CleanupTaskEncryption.java:
##########
@@ -0,0 +1,196 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.polaris.service.catalog.io;
+
+import com.fasterxml.jackson.core.JsonProcessingException;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.fasterxml.jackson.dataformat.smile.databind.SmileMapper;
+import java.io.Closeable;
+import java.io.IOException;
+import java.util.Base64;
+import java.util.List;
+import java.util.Map;
+import org.apache.iceberg.CatalogProperties;
+import org.apache.iceberg.EncryptedKeyParser;
+import org.apache.iceberg.TableMetadata;
+import org.apache.iceberg.TableProperties;
+import org.apache.iceberg.encryption.EncryptedKey;
+import org.apache.iceberg.encryption.EncryptingFileIO;
+import org.apache.iceberg.encryption.EncryptionManager;
+import org.apache.iceberg.encryption.EncryptionUtil;
+import org.apache.iceberg.encryption.KeyManagementClient;
+import org.apache.iceberg.encryption.StandardEncryptionManager;
+import org.apache.iceberg.io.FileIO;
+import org.apache.iceberg.util.PropertyUtil;
+import org.apache.polaris.core.entity.PolarisTaskConstants;
+
+/** Prepares and applies the encryption context used by Iceberg cleanup tasks. */
+public final class CleanupTaskEncryption {
+
+ private static final String CONTEXT_FORMAT_SMILE_V1 = "EC1";
+ private static final ObjectMapper SMILE_MAPPER = SmileMapper.builder().build();
+
+ private CleanupTaskEncryption() {}
+
+ /**
+ * Copies the catalog KMS configuration and the table's wrapped encryption keys into a cleanup
+ * task. Cleanup tasks run after the catalog request has ended, so they cannot reconstruct these
+ * values from the live table operations object.
+ */
+ public static void setupCleanupTaskEncryptionProperties(
Review Comment:
Nit: shorten the method name to `setupEncryptionProperties`?
##########
runtime/service/src/main/java/org/apache/polaris/service/catalog/io/CleanupTaskEncryption.java:
##########
@@ -0,0 +1,196 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.polaris.service.catalog.io;
+
+import com.fasterxml.jackson.core.JsonProcessingException;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.fasterxml.jackson.dataformat.smile.databind.SmileMapper;
+import java.io.Closeable;
+import java.io.IOException;
+import java.util.Base64;
+import java.util.List;
+import java.util.Map;
+import org.apache.iceberg.CatalogProperties;
+import org.apache.iceberg.EncryptedKeyParser;
+import org.apache.iceberg.TableMetadata;
+import org.apache.iceberg.TableProperties;
+import org.apache.iceberg.encryption.EncryptedKey;
+import org.apache.iceberg.encryption.EncryptingFileIO;
+import org.apache.iceberg.encryption.EncryptionManager;
+import org.apache.iceberg.encryption.EncryptionUtil;
+import org.apache.iceberg.encryption.KeyManagementClient;
+import org.apache.iceberg.encryption.StandardEncryptionManager;
+import org.apache.iceberg.io.FileIO;
+import org.apache.iceberg.util.PropertyUtil;
+import org.apache.polaris.core.entity.PolarisTaskConstants;
+
+/** Prepares and applies the encryption context used by Iceberg cleanup tasks. */
+public final class CleanupTaskEncryption {
Review Comment:
Nit: may be a more descriptive class name like `CleanupTaskEncryptionUtils`
##########
runtime/service/src/main/java/org/apache/polaris/service/catalog/io/CleanupTaskEncryption.java:
##########
@@ -0,0 +1,196 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.polaris.service.catalog.io;
+
+import com.fasterxml.jackson.core.JsonProcessingException;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.fasterxml.jackson.dataformat.smile.databind.SmileMapper;
+import java.io.Closeable;
+import java.io.IOException;
+import java.util.Base64;
+import java.util.List;
+import java.util.Map;
+import org.apache.iceberg.CatalogProperties;
+import org.apache.iceberg.EncryptedKeyParser;
+import org.apache.iceberg.TableMetadata;
+import org.apache.iceberg.TableProperties;
+import org.apache.iceberg.encryption.EncryptedKey;
+import org.apache.iceberg.encryption.EncryptingFileIO;
+import org.apache.iceberg.encryption.EncryptionManager;
+import org.apache.iceberg.encryption.EncryptionUtil;
+import org.apache.iceberg.encryption.KeyManagementClient;
+import org.apache.iceberg.encryption.StandardEncryptionManager;
+import org.apache.iceberg.io.FileIO;
+import org.apache.iceberg.util.PropertyUtil;
+import org.apache.polaris.core.entity.PolarisTaskConstants;
+
+/** Prepares and applies the encryption context used by Iceberg cleanup tasks. */
+public final class CleanupTaskEncryption {
+
+ private static final String CONTEXT_FORMAT_SMILE_V1 = "EC1";
+ private static final ObjectMapper SMILE_MAPPER = SmileMapper.builder().build();
+
+ private CleanupTaskEncryption() {}
+
+ /**
+ * Copies the catalog KMS configuration and the table's wrapped encryption keys into a cleanup
+ * task. Cleanup tasks run after the catalog request has ended, so they cannot reconstruct these
+ * values from the live table operations object.
+ */
+ public static void setupCleanupTaskEncryptionProperties(
+ Map<String, String> taskProperties,
+ Map<String, String> catalogProperties,
+ TableMetadata metadata) {
+ // Task properties start with table metadata. Always discard a table-controlled value for this
+ // reserved task field, including when the table is not encrypted.
+ taskProperties.remove(PolarisTaskConstants.ENCRYPTION_CONTEXT);
Review Comment:
We may just throw in case of a key conflict.
##########
runtime/service/src/main/java/org/apache/polaris/service/catalog/iceberg/LocalIcebergCatalog.java:
##########
@@ -570,6 +571,12 @@ public boolean dropTable(TableIdentifier tableIdentifier, boolean purge) {
clone.put(CatalogProperties.FILE_IO_IMPL, ioImplClassName);
clone.putAll(properties);
clone.put(PolarisTaskConstants.STORAGE_LOCATION, lastMetadata.location());
+
+ // Polaris does not write encrypted manifests here. Preserve the encryption
+ // context only so the asynchronous server-side purge can read manifest lists
+ // and manifests that were written by the engine and enumerate files to delete.
Review Comment:
Nit: this comment seems unnecessary. The class name and method name should be good enough for code reader to understand.
##########
runtime/service/src/main/java/org/apache/polaris/service/catalog/io/CleanupTaskEncryption.java:
##########
@@ -0,0 +1,160 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.polaris.service.catalog.io;
+
+import java.io.Closeable;
+import java.io.IOException;
+import java.util.List;
+import java.util.Map;
+import org.apache.iceberg.CatalogProperties;
+import org.apache.iceberg.EncryptedKeyParser;
+import org.apache.iceberg.TableMetadata;
+import org.apache.iceberg.TableProperties;
+import org.apache.iceberg.encryption.EncryptedKey;
+import org.apache.iceberg.encryption.EncryptingFileIO;
+import org.apache.iceberg.encryption.EncryptionManager;
+import org.apache.iceberg.encryption.EncryptionUtil;
+import org.apache.iceberg.encryption.KeyManagementClient;
+import org.apache.iceberg.encryption.StandardEncryptionManager;
+import org.apache.iceberg.io.FileIO;
+import org.apache.iceberg.util.PropertyUtil;
+import org.apache.polaris.core.entity.PolarisTaskConstants;
+import org.apache.polaris.core.persistence.PolarisObjectMapperUtil;
+
+/** Utilities for connecting Polaris file operations to Iceberg table encryption. */
+public final class PolarisEncryptionUtil {
+
+ private PolarisEncryptionUtil() {}
+
+ /**
+ * Copies the catalog KMS configuration and the table's wrapped encryption keys into a cleanup
+ * task. Cleanup tasks run after the catalog request has ended, so they cannot reconstruct these
+ * values from the live table operations object.
+ */
+ public static void addCleanupTaskEncryptionProperties(
+ Map<String, String> taskProperties,
+ Map<String, String> catalogProperties,
+ TableMetadata metadata) {
+ // Task properties start with table metadata. Always discard a table-controlled value for this
+ // reserved task field, including when the table is not encrypted.
+ taskProperties.remove(PolarisTaskConstants.ENCRYPTION_CONTEXT);
+
+ if (metadata == null
+ || metadata.properties().get(TableProperties.ENCRYPTION_TABLE_KEY) == null) {
+ return;
Review Comment:
We may just throw in case of a key conflict.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: issues-unsubscribe@polaris.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org