You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by ay...@apache.org on 2023/09/14 14:51:44 UTC
[hive] branch master updated: HIVE-27322: Iceberg: metadata location overrides can cause data breach. (#4673). (Ayush Saxena, reviewed by Denys Kuzmenko, Attila Turoczy)
This is an automated email from the ASF dual-hosted git repository.
ayushsaxena pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hive.git
The following commit(s) were added to refs/heads/master by this push:
new 5b8daa4c993 HIVE-27322: Iceberg: metadata location overrides can cause data breach. (#4673). (Ayush Saxena, reviewed by Denys Kuzmenko, Attila Turoczy)
5b8daa4c993 is described below
commit 5b8daa4c99322e8f1947cbc25ef1846bec8bc8a8
Author: Ayush Saxena <ay...@apache.org>
AuthorDate: Thu Sep 14 20:21:36 2023 +0530
HIVE-27322: Iceberg: metadata location overrides can cause data breach. (#4673). (Ayush Saxena, reviewed by Denys Kuzmenko, Attila Turoczy)
---
.../iceberg/mr/hive/HiveIcebergStorageHandler.java | 38 +++++++++++++++-------
.../hive/TestHiveIcebergStorageHandlerNoScan.java | 33 +++++++++++++++++++
.../hive/ql/metadata/HiveStorageHandler.java | 8 +++++
.../hadoop/hive/ql/parse/SemanticAnalyzer.java | 8 +++++
4 files changed, 76 insertions(+), 11 deletions(-)
diff --git a/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/HiveIcebergStorageHandler.java b/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/HiveIcebergStorageHandler.java
index 06d62b0ef7b..1e61ebcc86b 100644
--- a/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/HiveIcebergStorageHandler.java
+++ b/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/HiveIcebergStorageHandler.java
@@ -931,18 +931,25 @@ public class HiveIcebergStorageHandler implements HiveStoragePredicateHandler, H
StringBuilder authURI =
new StringBuilder(ICEBERG_URI_PREFIX).append(encodeString(dbName)).append("/").append(encodeString(tableName))
.append("?snapshot=");
- Optional<String> locationProperty = SessionStateUtil.getProperty(conf, hive_metastoreConstants.META_TABLE_LOCATION);
- if (locationProperty.isPresent()) {
- Preconditions.checkArgument(locationProperty.get() != null,
- "Table location is not set in SessionState. Authorization URI cannot be supplied.");
- // this property is set during the create operation before the hive table was created
- // we are returning a dummy iceberg metadata file
- authURI.append(encodeString(URI.create(locationProperty.get()).getPath()))
- .append(encodeString("/metadata/dummy.metadata.json"));
+ // If metadata location is provided we should use that location for auth, since during create if the
+ // metadata_location is explicitly provided we register a table using that path.
+ Optional<String> metadataLocation =
+ SessionStateUtil.getProperty(conf, BaseMetastoreTableOperations.METADATA_LOCATION_PROP);
+ if (metadataLocation.isPresent()) {
+ authURI.append(encodeString(metadataLocation.get()));
} else {
- Table table = IcebergTableUtil.getTable(conf, hmsTable);
- authURI.append(
- encodeString(URI.create(((BaseTable) table).operations().current().metadataFileLocation()).getPath()));
+ Optional<String> locationProperty =
+ SessionStateUtil.getProperty(conf, hive_metastoreConstants.META_TABLE_LOCATION);
+ if (locationProperty.isPresent()) {
+ // this property is set during the create operation before the hive table was created
+ // we are returning a dummy iceberg metadata file
+ authURI.append(encodeString(URI.create(locationProperty.get()).getPath()))
+ .append(encodeString("/metadata/dummy.metadata.json"));
+ } else {
+ Table table = IcebergTableUtil.getTable(conf, hmsTable);
+ authURI.append(
+ encodeString(URI.create(((BaseTable) table).operations().current().metadataFileLocation()).getPath()));
+ }
}
LOG.debug("Iceberg storage handler authorization URI {}", authURI);
return new URI(authURI.toString());
@@ -1542,6 +1549,15 @@ public class HiveIcebergStorageHandler implements HiveStoragePredicateHandler, H
return COPY_ON_WRITE.equalsIgnoreCase(mode);
}
+ @Override
+ public void addResourcesForCreateTable(Map<String, String> tblProps, HiveConf hiveConf) {
+ String metadataLocation = tblProps.get(BaseMetastoreTableOperations.METADATA_LOCATION_PROP);
+ if (StringUtils.isNotEmpty(metadataLocation)) {
+ SessionStateUtil.addResourceOrThrow(hiveConf, BaseMetastoreTableOperations.METADATA_LOCATION_PROP,
+ metadataLocation);
+ }
+ }
+
@Override
public Boolean hasAppendsOnly(org.apache.hadoop.hive.ql.metadata.Table hmsTable, SnapshotContext since) {
TableDesc tableDesc = Utilities.getTableDesc(hmsTable);
diff --git a/iceberg/iceberg-handler/src/test/java/org/apache/iceberg/mr/hive/TestHiveIcebergStorageHandlerNoScan.java b/iceberg/iceberg-handler/src/test/java/org/apache/iceberg/mr/hive/TestHiveIcebergStorageHandlerNoScan.java
index e9b5106e3bf..933bd8ce67e 100644
--- a/iceberg/iceberg-handler/src/test/java/org/apache/iceberg/mr/hive/TestHiveIcebergStorageHandlerNoScan.java
+++ b/iceberg/iceberg-handler/src/test/java/org/apache/iceberg/mr/hive/TestHiveIcebergStorageHandlerNoScan.java
@@ -103,6 +103,7 @@ import static org.mockito.ArgumentMatchers.any;
import static org.mockito.ArgumentMatchers.isNull;
import static org.mockito.Mockito.doAnswer;
import static org.mockito.Mockito.spy;
+import static org.mockito.Mockito.times;
@RunWith(Parameterized.class)
public class TestHiveIcebergStorageHandlerNoScan {
@@ -1503,6 +1504,38 @@ public class TestHiveIcebergStorageHandlerNoScan {
}
+ @Test
+ public void testAuthzURIWithAuthEnabledWithMetadataLocation() throws HiveException {
+ shell.setHiveSessionValue("hive.security.authorization.enabled", true);
+ shell.setHiveSessionValue("hive.security.authorization.manager",
+ "org.apache.iceberg.mr.hive.CustomTestHiveAuthorizerFactory");
+ TableIdentifier source = TableIdentifier.of("default", "source");
+ Table sourceTable = testTables.createTable(shell, source.name(), HiveIcebergStorageHandlerTestUtils.CUSTOMER_SCHEMA,
+ PartitionSpec.unpartitioned(), FileFormat.PARQUET, ImmutableList.of());
+
+ String metadataFileLocation =
+ URI.create(((BaseTable) sourceTable).operations().current().metadataFileLocation()).getPath();
+ TableIdentifier target = TableIdentifier.of("default", "target");
+
+ Table targetTable = testTables.createTable(shell, target.name(), HiveIcebergStorageHandlerTestUtils.CUSTOMER_SCHEMA,
+ PartitionSpec.unpartitioned(), FileFormat.PARQUET, ImmutableList.of(), 1,
+ Collections.singletonMap(BaseMetastoreTableOperations.METADATA_LOCATION_PROP, metadataFileLocation));
+ HiveAuthorizer authorizer = CustomTestHiveAuthorizerFactory.getAuthorizer();
+ ArgumentCaptor<List<HivePrivilegeObject>> outputHObjsCaptor = ArgumentCaptor.forClass(List.class);
+ Mockito.verify(authorizer, times(2))
+ .checkPrivileges(Mockito.any(), Mockito.any(), outputHObjsCaptor.capture(), Mockito.any());
+ Optional<HivePrivilegeObject> hivePrivObject = outputHObjsCaptor.getValue().stream()
+ .filter(hpo -> hpo.getType().equals(HivePrivilegeObject.HivePrivilegeObjectType.STORAGEHANDLER_URI)).findAny();
+
+ // For the target table, validate the metadata file location is passed at Authorizer.
+ if (hivePrivObject.isPresent()) {
+ Assert.assertEquals("iceberg://" + target.namespace() + "/" + target.name() + "?snapshot=" + metadataFileLocation,
+ HiveConf.EncoderDecoderFactory.URL_ENCODER_DECODER.decode(hivePrivObject.get().getObjectName()));
+ } else {
+ Assert.fail("StorageHandler auth URI is not found");
+ }
+ }
+
@Test
public void testAuthzURIWithAuthEnabledAndMockCommandAuthorizer() throws HiveException {
shell.setHiveSessionValue("hive.security.authorization.enabled", true);
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/metadata/HiveStorageHandler.java b/ql/src/java/org/apache/hadoop/hive/ql/metadata/HiveStorageHandler.java
index b10b289e96b..0b6d7ecf8dd 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/metadata/HiveStorageHandler.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/metadata/HiveStorageHandler.java
@@ -392,6 +392,14 @@ public interface HiveStorageHandler extends Configurable {
return false;
}
+ /**
+ * Adds specific configurations to session for create table command.
+ * @param tblProps table properties
+ * @param hiveConf configuration
+ */
+ default void addResourcesForCreateTable(Map<String, String> tblProps, HiveConf hiveConf) {
+ }
+
enum AcidSupportType {
NONE,
WITH_TRANSACTIONS,
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java
index 9aa19f7dc67..81f1e55f6f8 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java
@@ -14095,6 +14095,14 @@ public class SemanticAnalyzer extends BaseSemanticAnalyzer {
throw new SemanticException(e);
}
}
+ try {
+ HiveStorageHandler storageHandler = HiveUtils.getStorageHandler(conf, storageFormat.getStorageHandler());
+ if (storageHandler != null) {
+ storageHandler.addResourcesForCreateTable(tblProps, conf);
+ }
+ } catch (HiveException e) {
+ throw new RuntimeException(e);
+ }
SessionStateUtil.addResourceOrThrow(conf, META_TABLE_LOCATION, tblLocation);
break;
case CTT: // CREATE TRANSACTIONAL TABLE