You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by ng...@apache.org on 2019/06/07 16:30:06 UTC
[hive] branch master updated: HIVE-21816: HMS Translation: Refactor
tests to work with ACID tables (Naveen Gangam, reviewed by Daniel Dai)
This is an automated email from the ASF dual-hosted git repository.
ngangam pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hive.git
The following commit(s) were added to refs/heads/master by this push:
new dc53d04 HIVE-21816: HMS Translation: Refactor tests to work with ACID tables (Naveen Gangam, reviewed by Daniel Dai)
dc53d04 is described below
commit dc53d04f43f6b4728425e9c9a0fa5899153489ab
Author: Naveen Gangam <ng...@apache.org>
AuthorDate: Fri Jun 7 12:29:51 2019 -0400
HIVE-21816: HMS Translation: Refactor tests to work with ACID tables (Naveen Gangam, reviewed by Daniel Dai)
---
.../metastore/TestHiveMetastoreTransformer.java | 977 +++++++++++++++++++++
.../hadoop/hive/metastore/HiveMetaStore.java | 40 +-
.../metastore/MetastoreDefaultTransformer.java | 192 +++-
.../hadoop/hive/metastore/TestHiveMetaStore.java | 817 -----------------
4 files changed, 1163 insertions(+), 863 deletions(-)
diff --git a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestHiveMetastoreTransformer.java b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestHiveMetastoreTransformer.java
new file mode 100644
index 0000000..fb676f2
--- /dev/null
+++ b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestHiveMetastoreTransformer.java
@@ -0,0 +1,977 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hive.metastore;
+
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+import org.apache.hadoop.hive.metastore.HiveMetaStoreClient.GetTablesRequestBuilder;
+import org.apache.hadoop.hive.metastore.TableType;
+import org.apache.hadoop.hive.metastore.client.builder.DatabaseBuilder;
+import org.apache.hadoop.hive.metastore.client.builder.PartitionBuilder;
+import org.apache.hadoop.hive.metastore.client.builder.TableBuilder;
+import org.apache.hadoop.hive.metastore.conf.MetastoreConf;
+import org.apache.hadoop.hive.metastore.conf.MetastoreConf.ConfVars;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hive.metastore.api.Database;
+import org.apache.hadoop.hive.metastore.api.ExtendedTableInfo;
+import org.apache.hadoop.hive.metastore.api.FieldSchema;
+import org.apache.hadoop.hive.metastore.api.GetTablesExtRequestFields;
+import org.apache.hadoop.hive.metastore.api.InvalidOperationException;
+import org.apache.hadoop.hive.metastore.api.NoSuchObjectException;
+import org.apache.hadoop.hive.metastore.api.Partition;
+import org.apache.hadoop.hive.metastore.api.Table;
+import org.apache.hadoop.hive.metastore.api.Type;
+import org.apache.hadoop.hive.metastore.api.hive_metastoreConstants;
+import static org.apache.hadoop.hive.metastore.api.hive_metastoreConstants.ACCESSTYPE_NONE;
+import static org.apache.hadoop.hive.metastore.api.hive_metastoreConstants.ACCESSTYPE_READONLY;
+import static org.apache.hadoop.hive.metastore.api.hive_metastoreConstants.ACCESSTYPE_READWRITE;
+import static org.apache.hadoop.hive.metastore.api.hive_metastoreConstants.ACCESSTYPE_WRITEONLY;
+import org.apache.hadoop.util.StringUtils;
+
+import com.google.common.collect.Lists;
+
+import org.apache.thrift.TException;
+import org.junit.Before;
+import org.junit.Test;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.assertNull;
+import static org.junit.Assert.fail;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+public class TestHiveMetastoreTransformer {
+ private static final Logger LOG = LoggerFactory.getLogger(TestHiveMetastoreTransformer.class);
+ protected static HiveMetaStoreClient client;
+ protected static Configuration conf;
+
+ protected static boolean isThriftClient = false;
+ private static final String CAPABILITIES_KEY = "OBJCAPABILITIES";
+
+ @Before
+ public void setUp() throws Exception {
+ conf = MetastoreConf.newMetastoreConf();
+
+ MetastoreConf.setVar(conf, ConfVars.METASTORE_METADATA_TRANSFORMER_CLASS, "org.apache.hadoop.hive.metastore.MetastoreDefaultTransformer");
+ client = new HiveMetaStoreClient(conf);
+ }
+
+ private static void silentDropDatabase(String dbName) throws TException {
+ try {
+ for (String tableName : client.getTables(dbName, "*")) {
+ client.dropTable(dbName, tableName);
+ }
+ client.dropDatabase(dbName);
+ } catch (NoSuchObjectException|InvalidOperationException e) {
+ // NOP
+ }
+ }
+
+ private void resetHMSClient() {
+ client.setProcessorIdentifier(null);
+ client.setProcessorCapabilities(null);
+ }
+
+ private void setHMSClient(String id, String[] caps) {
+ client.setProcessorIdentifier(id);
+ client.setProcessorCapabilities(caps);
+ }
+
+ @Test
+ public void testTransformerWithOldTables() throws Exception {
+ try {
+ resetHMSClient();
+ final String dbName = "db1";
+ String basetblName = "oldstyletable";
+ Map<String, Object> tProps = new HashMap<>();
+ int buckets = 32;
+
+ String tblName = basetblName;
+ tProps.put("DBNAME", dbName);
+ tProps.put("TBLNAME", tblName);
+ tProps.put("TBLTYPE", TableType.EXTERNAL_TABLE);
+ tProps.put("BUCKETS", buckets);
+ StringBuilder properties = new StringBuilder();
+ properties.append("EXTERNAL").append("=").append("TRUE");
+ properties.append(";");
+ tProps.put("PROPERTIES", properties.toString());
+ Table tbl = createTableWithCapabilities(tProps);
+
+ setHMSClient("testTranformerWithOldTables", (new String[] { "HIVEBUCKET2", "EXTREAD", "EXTWRITE"}));
+
+ Table tbl2 = client.getTable(dbName, tblName);
+ assertEquals("Created and retrieved tables do not match:" + tbl2.getTableName() + ":" + tblName,
+ tbl2.getTableName(), tblName);
+ LOG.info("Test execution complete:testTransformerWithOldTables");
+ } catch (Exception e) {
+ fail("testTransformerWithOldTables failed with " + e.getMessage());
+ } finally {
+ resetHMSClient();
+ }
+
+ }
+
+ /**
+ * EXTERNAL_TABLE
+ * 1) Old table with no capabilities
+ * 2a) New table with capabilities with no client requirements
+ * 2b) New table with capabilities with no matching client requirements
+ * 2c) New table with capabilities with partial match requirements
+ * 2d) New table with capabilities with full match requirements
+ */
+ @Test
+ public void testTransformerExternalTable() throws Exception {
+ try {
+ final String dbName = "db1";
+ String basetblName = "oldstyleexttable";
+ Map<String, Object> tProps = new HashMap<>();
+ int buckets = 32;
+
+ String tblName = basetblName;
+ tProps.put("DBNAME", dbName);
+ tProps.put("TBLNAME", tblName);
+ tProps.put("TBLTYPE", TableType.EXTERNAL_TABLE);
+ tProps.put("BUCKETS", buckets);
+ StringBuilder properties = new StringBuilder();
+ properties.append("EXTERNAL").append("=").append("TRUE");
+ tProps.put("PROPERTIES", properties.toString());
+ Table tbl = createTableWithCapabilities(tProps);
+
+ // retrieve the table
+ Table tbl2 = client.getTable(dbName, tblName);
+ LOG.info("Table=" + tblName + ",Access=" + tbl2.getAccessType());
+ assertEquals("Created and retrieved tables do not match:" + tbl2.getTableName() + ":" + tblName,
+ tbl2.getTableName(), tblName);
+ assertEquals("TableType mismatch", TableType.EXTERNAL_TABLE.name(), tbl2.getTableType());
+ assertEquals(buckets, tbl2.getSd().getNumBuckets()); // no transformation
+ assertEquals("Table access type does not match expected value:" + tblName,
+ 0, tbl2.getAccessType()); // old client, AccessType not set
+
+ setHMSClient("testTranformerExternalTable", (new String[] { "HIVEBUCKET2" }));
+ tbl2 = client.getTable(dbName, tblName);
+ assertEquals("Expected buckets does not match:", buckets, tbl2.getSd().getNumBuckets()); // no transformation
+ resetHMSClient();
+
+ tblName = "test_ext_bucketed_wc";
+ properties = new StringBuilder();
+ properties.append("EXTERNAL").append("=").append("TRUE");
+ properties.append(";");
+ properties.append(CAPABILITIES_KEY).append("=").append("HIVEBUCKET2,EXTREAD,EXTWRITE");
+ tProps.put("TBLNAME", tblName);
+ tProps.put("PROPERTIES", properties.toString());
+ tbl = createTableWithCapabilities(tProps);
+
+ tbl2 = client.getTable(dbName, tblName);
+ assertEquals("Table access type does not match expected value:" + tblName,
+ 0, tbl2.getAccessType());
+ assertEquals(buckets, tbl2.getSd().getNumBuckets()); // no tranformation
+
+ setHMSClient("testTranformerExternalTable", (new String[] { "HIVEBUCKET2", "EXTREAD", "EXTWRITE"}));
+ tbl2 = client.getTable(dbName, tblName);
+ assertEquals("Table access type does not match the expected value:" + tblName,
+ ACCESSTYPE_READWRITE, tbl2.getAccessType());
+ assertEquals(buckets, tbl2.getSd().getNumBuckets()); // client has the HIVEBUCKET2 capability, retain bucketing info
+
+ resetHMSClient();
+
+ setHMSClient("testTransformerExternalTableRO", (new String[] { "EXTREAD", "EXTWRITE"}));
+ tbl2 = client.getTable(dbName, tblName);
+ assertEquals("Table access type does not match the expected value:" + tblName,
+ ACCESSTYPE_READONLY, tbl2.getAccessType());
+ assertEquals(-1, tbl2.getSd().getNumBuckets()); // client has no HIVEBUCKET2 capability, remove bucketing info
+
+ resetHMSClient();
+
+ tblName = "test_ext_unbucketed_wc";
+ properties = new StringBuilder();
+ properties.append(CAPABILITIES_KEY).append("=").append("EXTREAD,EXTWRITE");
+ tProps.put("TBLNAME", tblName);
+ tProps.put("PROPERTIES", properties.toString());
+ tProps.remove("BUCKETS");
+ tbl = createTableWithCapabilities(tProps);
+
+ setHMSClient("TestTransformerExternalUnbucketed", (new String[] {"EXTREAD", "EXTWRITE"}));
+ tbl2 = client.getTable(dbName, tblName);
+ assertEquals("Table access type does not match the expected value:" + tblName,
+ ACCESSTYPE_READWRITE, tbl2.getAccessType());
+
+ resetHMSClient();
+
+ tblName = "test_ext_sparkbucketed_wc";
+ properties = new StringBuilder();
+ properties.append("EXTERNAL").append("=").append("TRUE");
+ properties.append(";");
+ properties.append(CAPABILITIES_KEY).append("=").append("SPARKDECIMAL,SPARKBUCKET,EXTREAD");
+ tProps.put("TBLNAME", tblName);
+ tProps.put("PROPERTIES", properties.toString());
+ tProps.put("BUCKETS", buckets);
+ tbl = createTableWithCapabilities(tProps);
+
+ setHMSClient("testTransformerExternalTableSpark", (new String[] { "HIVEFULLACIDREAD", "CONNECTORREAD"}));
+ tbl2 = client.getTable(dbName, tblName);
+ assertEquals("Table access type does not match the expected value:" + tblName,
+ ACCESSTYPE_NONE, tbl2.getAccessType()); // requires EXTREAD for RO
+ assertEquals(buckets, tbl2.getSd().getNumBuckets());
+
+ setHMSClient("testTransformerExternalTableSpark", (new String[] { "EXTREAD", "CONNECTORREAD"}));
+ tbl2 = client.getTable(dbName, tblName);
+ assertEquals("Table access type does not match the expected value:" + tblName,
+ ACCESSTYPE_READONLY, tbl2.getAccessType()); // requires EXTREAD for RO
+ assertEquals(buckets, tbl2.getSd().getNumBuckets());
+
+ setHMSClient("testTransformerExternalTableSpark", (new String[] { "SPARKBUCKET", "SPARKDECIMAL", "EXTREAD" }));
+ tbl2 = client.getTable(dbName, tblName);
+ assertEquals("Table access type does not match the expected value:" + tblName,
+ ACCESSTYPE_READWRITE, tbl2.getAccessType());
+ assertEquals(buckets, tbl2.getSd().getNumBuckets()); // client has SPARKBUCKET capability
+
+ resetHMSClient();
+
+ LOG.info("Test execution complete:testTransformerExternalTable");
+ } catch (Exception e) {
+ fail("testTransformerExternalTable failed with " + e.getMessage());
+ } finally {
+ resetHMSClient();
+ }
+ }
+
+ @Test
+ public void testTransformerManagedTable() throws Exception {
+ try {
+ resetHMSClient();
+
+ final String dbName = "db1";
+ String basetblName = "oldstylemgdtable";
+ Map<String, Object> tProps = new HashMap<>();
+
+ String tblName = basetblName;
+ tProps.put("DBNAME", dbName);
+ tProps.put("TBLNAME", tblName);
+ tProps.put("TBLTYPE", TableType.MANAGED_TABLE);
+ // create unbucketed managed table with no capabilities
+ Table tbl = createTableWithCapabilities(tProps);
+
+ // retrieve the table
+ Table tbl2 = client.getTable(dbName, tblName);
+ LOG.info("Table=" + tblName + ",Access=" + tbl2.getAccessType());
+ assertEquals("Created and retrieved tables do not match:" + tbl2.getTableName() + ":" + tblName,
+ tbl2.getTableName(), tblName);
+ assertEquals("TableType mismatch", TableType.MANAGED_TABLE.name(), tbl2.getTableType());
+ assertEquals(-1, tbl2.getSd().getNumBuckets());
+ assertEquals("Table access type does not match expected value:" + tblName,
+ 0, tbl2.getAccessType()); // no translation to be done, so accessType not set
+
+ // managed table with no capabilities
+ tblName = "test_mgd_insert_woc";
+ StringBuilder properties = new StringBuilder();
+ tProps.put("TBLNAME", tblName);
+ properties.append("transactional=true");
+ properties.append(";");
+ properties.append("transactional_properties=insert_only");
+ tProps.put("PROPERTIES", properties.toString());
+ tbl = createTableWithCapabilities(tProps);
+
+ tbl2 = client.getTable(dbName, tblName);
+ assertEquals("Table access type does not match expected value:" + tblName,
+ 0, tbl2.getAccessType()); // no transformation
+
+ setHMSClient("testMGDwithConnectorRead", new String[] {"CONNECTORREAD"});
+ tbl2 = client.getTable(dbName, tblName);
+ assertEquals("Table access type does not match expected value:" + tblName,
+ ACCESSTYPE_READONLY, tbl2.getAccessType());
+ resetHMSClient();
+
+ setHMSClient("testMGDwithInsertRead", new String[] {"HIVEMANAGEDINSERTREAD"});
+ tbl2 = client.getTable(dbName, tblName);
+ assertEquals("Table access type does not match expected value:" + tblName,
+ ACCESSTYPE_READONLY, tbl2.getAccessType());
+ resetHMSClient();
+
+ setHMSClient("testMGDwithInsertWrite", new String[] {"HIVEMANAGEDINSERTWRITE"});
+ tbl2 = client.getTable(dbName, tblName);
+ assertEquals("Table access type does not match expected value:" + tblName,
+ ACCESSTYPE_READWRITE, tbl2.getAccessType());
+ resetHMSClient();
+
+ setHMSClient("testMGDwithConnectorWrite", new String[] {"CONNECTORWRITE"});
+ tbl2 = client.getTable(dbName, tblName);
+ assertEquals("Table access type does not match expected value:" + tblName,
+ ACCESSTYPE_READWRITE, tbl2.getAccessType());
+ resetHMSClient();
+
+ // bucketed table with no capabilities
+ tblName = "test_mgd_insert_wc";
+ properties = new StringBuilder();
+ properties.append("transactional=true");
+ properties.append(";");
+ properties.append(CAPABILITIES_KEY).append("=").append("HIVEMANAGEDINSERTREAD,HIVEMANAGEDINSERTWRITE,HIVECACHEINVALIDATE,")
+ .append("HIVEMANAGEDSTATS,CONNECTORREAD,CONNECTORWRITE");
+ properties.append(";");
+ properties.append("transactional_properties=insert_only");
+ tProps.put("TBLNAME", tblName);
+ tProps.put("PROPERTIES", properties.toString());
+ tbl = createTableWithCapabilities(tProps);
+
+ tbl2 = client.getTable(dbName, tblName);
+ assertEquals("Table access type does not match expected value:" + tblName,
+ 0, tbl2.getAccessType());
+
+ setHMSClient("testMGDwithAllWrites", new String[] { "HIVEMANAGEDINSERTWRITE", "HIVECACHEINVALIDATE",
+ "HIVEMANAGEDSTATS" , "CONNECTORWRITE" });
+ tbl2 = client.getTable(dbName, tblName);
+ assertEquals("Table access type does not match expected value:" + tblName,
+ ACCESSTYPE_READWRITE, tbl2.getAccessType());
+ resetHMSClient();
+
+ setHMSClient("testMGDwith1MissingWrite", new String[] {"HIVEMANAGEDINSERTREAD", "HIVEMANAGEDINSERTWRITE",
+ "HIVECACHEINVALIDATE" });
+ tbl2 = client.getTable(dbName, tblName);
+ assertEquals("Table access type does not match expected value:" + tblName,
+ ACCESSTYPE_NONE, tbl2.getAccessType());
+ resetHMSClient();
+
+ setHMSClient("testMGDwith1Write", new String[] {"CONNECTORWRITE"});
+ tbl2 = client.getTable(dbName, tblName);
+ assertEquals("Table access type does not match expected value:" + tblName,
+ ACCESSTYPE_READWRITE, tbl2.getAccessType());
+ resetHMSClient();
+
+ setHMSClient("testMGDwithConnectorRead", new String[] {"CONNECTORREAD"});
+ tbl2 = client.getTable(dbName, tblName);
+ assertEquals("Table access type does not match expected value:" + tblName,
+ ACCESSTYPE_READONLY, tbl2.getAccessType());
+ resetHMSClient();
+
+ // managed tables with no capabilities
+ tblName = "test_mgd_full_woc";
+ properties = new StringBuilder();
+ properties.append("transactional=true");
+ properties.append(";");
+ tProps.put("TBLNAME", tblName);
+ tProps.put("TBLTYPE", TableType.MANAGED_TABLE);
+ tProps.put("PROPERTIES", properties.toString());
+ tbl = createTableWithCapabilities(tProps);
+
+ tbl2 = client.getTable(dbName, tblName);
+ assertEquals("Table access type does not match expected value:" + tblName,
+ 0, tbl2.getAccessType()); // translation skipped due to no capabilities
+
+ setHMSClient("testMGDwithRAWMETADATA", new String[] {"MANAGERAWMETADATA"});
+ tbl2 = client.getTable(dbName, tblName);
+ assertEquals("Table access type does not match expected value:" + tblName,
+ 0, tbl2.getAccessType());
+ resetHMSClient();
+
+ setHMSClient("testMGDwithConnectorRead", new String[] {"CONNECTORREAD"});
+ tbl2 = client.getTable(dbName, tblName);
+ assertEquals("Table access type does not match expected value:" + tblName,
+ ACCESSTYPE_READONLY, tbl2.getAccessType());
+ resetHMSClient();
+
+ setHMSClient("testMGDwithFullACIDRead", new String[] {"HIVEFULLACIDREAD"});
+ tbl2 = client.getTable(dbName, tblName);
+ assertEquals("Table access type does not match expected value:" + tblName,
+ ACCESSTYPE_READONLY, tbl2.getAccessType());
+ resetHMSClient();
+
+ setHMSClient("testMGDwith1Write", new String[] {"HIVEFULLACIDWRITE"});
+ tbl2 = client.getTable(dbName, tblName);
+ assertEquals("Table access type does not match expected value:" + tblName,
+ ACCESSTYPE_READWRITE, tbl2.getAccessType());
+ resetHMSClient();
+
+ setHMSClient("testMGDwithConnectorWrite", new String[] {"CONNECTORWRITE"});
+ tbl2 = client.getTable(dbName, tblName);
+ assertEquals("Table access type does not match expected value:" + tblName,
+ ACCESSTYPE_READWRITE, tbl2.getAccessType());
+ resetHMSClient();
+
+ // bucketed table with no capabilities
+ tblName = "test_mgd_full_wc";
+ properties = new StringBuilder();
+ properties.append("transactional=true");
+ properties.append(";");
+ properties.append(CAPABILITIES_KEY).append("=").append("HIVEFULLACIDREAD,HIVEFULLACIDWRITE,HIVECACHEINVALIDATE,")
+ .append("HIVEMANAGEDSTATS,CONNECTORREAD,CONNECTORWRITE");
+ tProps.put("TBLNAME", tblName);
+ tProps.put("TBLTYPE", TableType.MANAGED_TABLE);
+ tProps.put("PROPERTIES", properties.toString());
+ tbl = createTableWithCapabilities(tProps);
+
+ tbl2 = client.getTable(dbName, tblName);
+ assertEquals("Table access type does not match expected value:" + tblName,
+ 0, tbl2.getAccessType());
+
+ setHMSClient("testMGDwithRAWMETADATA", new String[] {"MANAGERAWMETADATA"});
+ tbl2 = client.getTable(dbName, tblName);
+ assertEquals("Table access type does not match expected value:" + tblName,
+ 0, tbl2.getAccessType());
+ resetHMSClient();
+
+ setHMSClient("testMGDFULLwithAllWrites", new String[] { "HIVEFULLACIDWRITE", "HIVECACHEINVALIDATE",
+ "HIVEMANAGEDSTATS", "CONNECTORWRITE" });
+ tbl2 = client.getTable(dbName, tblName);
+ assertEquals("Table access type does not match expected value:" + tblName,
+ ACCESSTYPE_READWRITE, tbl2.getAccessType());
+ resetHMSClient();
+
+ setHMSClient("testMGDFULLwith1MissingWrite", new String[] { "HIVECACHEINVALIDATE",
+ "HIVEMANAGEDSTATS", "CONNECTORWRITE" });
+ tbl2 = client.getTable(dbName, tblName);
+ assertEquals("Table access type does not match expected value:" + tblName,
+ ACCESSTYPE_READWRITE, tbl2.getAccessType());
+ resetHMSClient();
+
+ setHMSClient("testMGDFULLwith1Write", new String[] { "CONNECTORWRITE" });
+ tbl2 = client.getTable(dbName, tblName);
+ assertEquals("Table access type does not match expected value:" + tblName,
+ ACCESSTYPE_READWRITE, tbl2.getAccessType());
+ resetHMSClient();
+
+ setHMSClient("testMGDwith1MissingWrite", new String[] {"HIVEFULLACIDREAD", "HIVEFULLACIDWRITE",
+ "HIVECACHEINVALIDATE" });
+ tbl2 = client.getTable(dbName, tblName);
+ assertEquals("Table access type does not match expected value:" + tblName,
+ ACCESSTYPE_READWRITE, tbl2.getAccessType());
+ resetHMSClient();
+
+ setHMSClient("testMGDwithConnectorRead", new String[] {"CONNECTORREAD"});
+ tbl2 = client.getTable(dbName, tblName);
+ assertEquals("Table access type does not match expected value:" + tblName,
+ ACCESSTYPE_READONLY, tbl2.getAccessType());
+ resetHMSClient();
+
+ setHMSClient("testTranformerExternalTable", (new String[] { "HIVEBUCKET2", "EXTREAD", "EXTWRITE"}));
+ tbl2 = client.getTable(dbName, tblName);
+ assertEquals("Table access type does not match the expected value:" + tblName,
+ ACCESSTYPE_NONE, tbl2.getAccessType());
+
+ resetHMSClient();
+
+ LOG.info("Test execution complete:testTransformerManagedTable");
+ } catch (Exception e) {
+ e.printStackTrace();
+ fail("testTransformerManagedTable failed with " + e.getMessage());
+ } finally {
+ resetHMSClient();
+ }
+ }
+
+ @Test
+ public void testTransformerVirtualView() throws Exception {
+ try {
+ final String dbName = "db1";
+ String basetblName = "oldstyleview";
+ Map<String, Object> tProps = new HashMap<>();
+
+ String tblName = basetblName;
+ tProps.put("DBNAME", dbName);
+ tProps.put("TBLNAME", tblName);
+ tProps.put("TBLTYPE", TableType.VIRTUAL_VIEW);
+ StringBuilder properties = new StringBuilder();
+ tProps.put("PROPERTIES", properties.toString());
+ Table tbl = createTableWithCapabilities(tProps);
+
+ // retrieve the table
+ Table tbl2 = client.getTable(dbName, tblName);
+ LOG.info("View=" + tblName + ",Access=" + tbl2.getAccessType());
+ assertEquals("Created and retrieved views do not match:" + tbl2.getTableName() + ":" + tblName,
+ tbl2.getTableName(), tblName);
+ assertEquals("TableType mismatch", TableType.VIRTUAL_VIEW.name(), tbl2.getTableType());
+ assertEquals("Table access type does not match expected value:" + tblName,
+ 0, tbl2.getAccessType()); // old client, AccessType not set
+
+ // table has capabilities
+ tblName = "test_view_wc";
+ properties = new StringBuilder();
+ properties.append(CAPABILITIES_KEY).append("=").append("HIVESQL");
+ tProps.put("TBLNAME", tblName);
+ tProps.put("PROPERTIES", properties.toString());
+ tbl = createTableWithCapabilities(tProps);
+
+ tbl2 = client.getTable(dbName, tblName);
+ assertEquals("Table access type does not match expected value:" + tblName,
+ 0, tbl2.getAccessType()); // old client, no transformation
+
+ setHMSClient("testTransformerVirtualView", (new String[] { "HIVESQL" }));
+ tbl2 = client.getTable(dbName, tblName);
+ assertEquals("Table access type does not match the expected value:" + tblName,
+ ACCESSTYPE_READONLY, tbl2.getAccessType()); // RO accessonly for all views
+
+ setHMSClient("testTransformerVirtualView", (new String[] { "EXTWRITE", "HIVEFULLACIDWRITE" }));
+ tbl2 = client.getTable(dbName, tblName);
+ assertEquals("Table access type does not match the expected value:" + tblName,
+ ACCESSTYPE_NONE, tbl2.getAccessType()); // missing HIVESQL
+
+ resetHMSClient();
+
+ // table does not capabilities but client is newer
+ tblName = "test_view_woc";
+ properties = new StringBuilder();
+ tProps.put("TBLNAME", tblName);
+ tProps.put("PROPERTIES", properties.toString());
+ tbl = createTableWithCapabilities(tProps);
+
+ setHMSClient("testTransformerVirtualView", (new String[] { "HIVESQL" }));
+ tbl2 = client.getTable(dbName, tblName);
+ assertEquals("Table access type does not match the expected value:" + tblName,
+ ACCESSTYPE_READONLY, tbl2.getAccessType()); // RO accessonly for all views
+
+ setHMSClient("testTransformerVirtualView", (new String[] { "EXTWRITE", "HIVEFULLACIDWRITE" }));
+ tbl2 = client.getTable(dbName, tblName);
+ assertEquals("Table access type does not match the expected value:" + tblName,
+ ACCESSTYPE_NONE, tbl2.getAccessType()); // missing HIVESQL
+
+ LOG.info("Test execution complete:testTransformerVirtualView");
+ } catch (Exception e) {
+ fail("testTransformerVirtualView failed with " + e.getMessage());
+ } finally {
+ resetHMSClient();
+ }
+ }
+
+ @Test
+ public void testTransformerMaterializedView() throws Exception {
+ try {
+ resetHMSClient();
+
+ final String dbName = "db1";
+ String basetblName = "oldstylemqtview";
+ Map<String, Object> tProps = new HashMap<>();
+
+ String tblName = basetblName;
+ tProps.put("DBNAME", dbName);
+ tProps.put("TBLNAME", tblName);
+ tProps.put("TBLTYPE", TableType.MATERIALIZED_VIEW);
+ StringBuilder properties = new StringBuilder();
+ tProps.put("PROPERTIES", properties.toString());
+ Table tbl = createTableWithCapabilities(tProps);
+
+ // retrieve the table
+ Table tbl2 = client.getTable(dbName, tblName);
+ LOG.info("View=" + tblName + ",Access=" + tbl2.getAccessType());
+ assertEquals("Created and retrieved views do not match:" + tbl2.getTableName() + ":" + tblName,
+ tbl2.getTableName(), tblName);
+ assertEquals("TableType mismatch", TableType.MATERIALIZED_VIEW.name(), tbl2.getTableType());
+ assertEquals("Table access type does not match expected value:" + tblName,
+ 0, tbl2.getAccessType()); // old client, AccessType not set
+
+ // table has capabilities
+ tblName = "test_mqtview_wc";
+ properties = new StringBuilder();
+ properties.append(CAPABILITIES_KEY).append("=").append("HIVEFULLACIDREAD,HIVEONLYMQTWRITE,HIVEMANAGESTATS,HIVEMQT,CONNECTORREAD");
+ tProps.put("TBLNAME", tblName);
+ tProps.put("PROPERTIES", properties.toString());
+ tbl = createTableWithCapabilities(tProps);
+
+ tbl2 = client.getTable(dbName, tblName);
+ assertEquals("Table access type does not match expected value:" + tblName,
+ 0, tbl2.getAccessType()); // View has capabilities, processor doesnt, no tranformation
+
+ setHMSClient("testTransformerMQTFullSet", (new String[] { "HIVEFULLACIDREAD", "HIVEONLYMQTWRITE",
+ "HIVEMANAGESTATS", "HIVEMQT", "CONNECTORREAD" }));
+ tbl2 = client.getTable(dbName, tblName);
+ assertEquals("Table access type does not match the expected value:" + tblName,
+ ACCESSTYPE_READONLY, tbl2.getAccessType()); // RO accessonly for all views
+
+ setHMSClient("testTransformerMQTFullRead", (new String[] { "HIVEFULLACIDREAD", "HIVEMQT" }));
+ tbl2 = client.getTable(dbName, tblName);
+ assertEquals("Table access type does not match the expected value:" + tblName,
+ ACCESSTYPE_NONE, tbl2.getAccessType());
+
+ setHMSClient("testTransformerMQTConnRead", (new String[] { "CONNECTORREAD", "HIVEMQT" }));
+ tbl2 = client.getTable(dbName, tblName);
+ assertEquals("Table access type does not match the expected value:" + tblName,
+ ACCESSTYPE_NONE, tbl2.getAccessType()); // RO accessonly for all views
+
+ setHMSClient("testTransformerMQTDummySet", (new String[] { "EXTWRITE", "HIVEFULLACIDWRITE" }));
+ tbl2 = client.getTable(dbName, tblName);
+ assertEquals("Table access type does not match the expected value:" + tblName,
+ ACCESSTYPE_NONE, tbl2.getAccessType()); // missing HIVEMQT + *READ
+
+ resetHMSClient();
+
+ // table does not capabilities but client is newer
+ tblName = "test_mqtview_woc";
+ properties = new StringBuilder();
+ tProps.put("TBLNAME", tblName);
+ tProps.put("PROPERTIES", properties.toString());
+ tbl = createTableWithCapabilities(tProps);
+
+ setHMSClient("testTransformerVirtualView", (new String[] { "HIVEFULLACIDREAD", "HIVEMQT" }));
+ tbl2 = client.getTable(dbName, tblName);
+ assertEquals("Table access type does not match the expected value:" + tblName,
+ ACCESSTYPE_READONLY, tbl2.getAccessType()); // RO accessonly for all views
+
+ setHMSClient("testTransformerMQTConnRead", (new String[] { "CONNECTORREAD", "HIVEMQT" }));
+ tbl2 = client.getTable(dbName, tblName);
+ assertEquals("Table access type does not match the expected value:" + tblName,
+ ACCESSTYPE_READONLY, tbl2.getAccessType()); // RO accessonly for all views
+
+ setHMSClient("testTransformerVirtualView", (new String[] { "EXTWRITE, HIVEFULLACIDWRITE" }));
+ tbl2 = client.getTable(dbName, tblName);
+ assertEquals("Table access type does not match the expected value:" + tblName,
+ ACCESSTYPE_NONE, tbl2.getAccessType()); // missing HIVEMQT + *READ
+
+ LOG.info("Test execution complete:testTransformerMaterializedView");
+ } catch (Exception e) {
+ fail("testTransformerMaterializedVirtualView failed with " + e.getMessage());
+ } finally {
+ resetHMSClient();
+ }
+ }
+
+ @Test
+ public void testGetTablesExt() throws Exception {
+ try {
+ resetHMSClient();
+
+ final String dbName = "dbext";
+ String tblName = "test_get_tables_ext";
+ int count = 10;
+ Map<String, Object> tProps = new HashMap<>();
+
+ tProps.put("DBNAME", dbName);
+ tProps.put("TBLNAME", tblName);
+
+ StringBuilder table_params = new StringBuilder();
+ List<String> capabilities = new ArrayList<>();
+ TableType type = TableType.EXTERNAL_TABLE;
+
+ capabilities.add("HIVEFULLACIDWRITE");
+ capabilities.add("HIVEFULLACIDREAD");
+ capabilities.add("HIVECACHEINVALIDATE");
+ capabilities.add("CONNECTORREAD");
+ capabilities.add("CONNECTORWRITE");
+
+ tProps.put("CAPABILITIES", capabilities);
+ tProps.put("TABLECOUNT", count);
+ tProps.put("TBLTYPE", type);
+
+ setHMSClient("test_get_tables_ext", (String[])(capabilities.toArray(new String[0])));
+
+ List<String> tables = createTables(tProps);
+ int requestedFields = (new GetTablesRequestBuilder().with(GetTablesExtRequestFields.PROCESSOR_CAPABILITIES)).bitValue();
+ List<ExtendedTableInfo> extTables = client.getTablesExt(null, dbName, "*", requestedFields, 2000);
+ LOG.debug("Return list size=" + extTables.size() + ",bitValue=" + requestedFields);
+ assertEquals("Return list size does not match expected size:extTables", count, extTables.size());
+ for (ExtendedTableInfo tableInfo : extTables) {
+ assertEquals("Capability set size does not match", capabilities.size(), tableInfo.getProcessorCapabilities().size());
+ assertEquals("AccessType not expected to be set", 0, tableInfo.getAccessType());
+ }
+
+ requestedFields = (new GetTablesRequestBuilder().with(GetTablesExtRequestFields.ACCESS_TYPE)).bitValue();
+ extTables = client.getTablesExt(null, dbName, "*", requestedFields, 2000);
+ LOG.debug("Return list size=" + extTables.size() + ",bitValue=" + requestedFields);
+ assertEquals("Return list size does not match expected size", count, extTables.size());
+ for (ExtendedTableInfo tableInfo : extTables) {
+ assertNull("Return value should not contain capabilities", tableInfo.getProcessorCapabilities());
+ assertTrue("AccessType expected to be set", tableInfo.getAccessType() > 0);
+ }
+
+ requestedFields = (new GetTablesRequestBuilder().with(GetTablesExtRequestFields.ALL)).bitValue();
+ extTables = client.getTablesExt(null, dbName, "*", requestedFields, 2000);
+ LOG.debug("Return list size=" + extTables.size() + ",bitValue=" + requestedFields);
+ assertEquals("Return list size does not match expected size", count, extTables.size());
+ for (ExtendedTableInfo tableInfo : extTables) {
+ assertEquals("Capability set size does not match", capabilities.size(), tableInfo.getProcessorCapabilities().size());
+ assertTrue("AccessType expected to be set", tableInfo.getAccessType() > 0);
+ }
+
+ extTables = client.getTablesExt(null, dbName, "*", requestedFields, (count - 3));
+ LOG.debug("Return list size=" + extTables.size() + ",bitValue=" + requestedFields);
+ assertEquals("Return list size does not match expected size", (count - 3), extTables.size());
+ for (ExtendedTableInfo tableInfo : extTables) {
+ assertEquals("Capability set size does not match", capabilities.size(), tableInfo.getProcessorCapabilities().size());
+ assertTrue("AccessType expected to be set", tableInfo.getAccessType() > 0);
+ }
+
+ extTables = client.getTablesExt(null, dbName, "*", requestedFields, -1);
+ LOG.debug("Return list size=" + extTables.size() + ",bitValue=" + requestedFields);
+ assertEquals("Return list size does not match expected size", count, extTables.size());
+
+ count = 300;
+ tProps.put("TBLNAME", "test_limit");
+ tProps.put("TABLECOUNT", count);
+ tables = createTables(tProps);
+ assertEquals("Unexpected number of tables created", count, tables.size());
+
+ extTables = client.getTablesExt(null, dbName, "test_limit*", requestedFields, count);
+ assertEquals("Unexpected number of tables returned", count, extTables.size());
+
+ extTables = client.getTablesExt(null, dbName, "test_limit*", requestedFields, (count/2));
+ assertEquals("Unexpected number of tables returned", (count/2), extTables.size());
+
+ extTables = client.getTablesExt(null, dbName, "test_limit*", requestedFields, 1);
+ assertEquals("Unexpected number of tables returned", 1, extTables.size());
+
+ LOG.info("Test execution complete:testTablesExt");
+ } catch (Exception e) {
+ System.err.println(StringUtils.stringifyException(e));
+ System.err.println("testGetTablesExt() failed.");
+ fail("testGetTablesExt failed with " + e.getMessage());
+ } finally {
+ resetHMSClient();
+ }
+ }
+
+ @Test
+ public void testGetPartitionsByNames() throws Exception {
+ try {
+ resetHMSClient();
+
+ final String dbName = "db1";
+ String tblName = "test_get_parts_by_names";
+ int count = 1;
+ int pCount = 10;
+ int bucketCount = 100;
+ Map<String, Object> tProps = new HashMap<>();
+
+ tProps.put("DBNAME", dbName);
+ tProps.put("TBLNAME", tblName);
+
+ List<String> capabilities = new ArrayList<>();
+ TableType type = TableType.MANAGED_TABLE;
+
+ capabilities.add("HIVEFULLACIDWRITE");
+ capabilities.add("HIVEFULLACIDREAD");
+ capabilities.add("HIVECACHEINVALIDATE");
+ capabilities.add("CONNECTORREAD");
+ capabilities.add("CONNECTORWRITE");
+
+ // tProps.put("CAPABILITIES", capabilities);
+ tProps.put("TBLTYPE", type);
+ tProps.put("PARTITIONS", pCount);
+
+ setHMSClient("TestGetPartitionByNames#1", (String[])(capabilities.toArray(new String[0])));
+
+ Table table = createTableWithCapabilities(tProps);
+
+ List<String> partNames = new ArrayList<>();
+ List<String> partValues = new ArrayList<>();
+ for (int i = 1; i <= pCount; i++) {
+ partValues.add("partcol=" + i);
+ }
+ List<Partition> parts = client.getPartitionsByNames(dbName, tblName, partValues, false);
+ assertEquals("Return list size does not match expected size", pCount, parts.size());
+
+ tblName = "test_gp_ext_bucketed_wc";
+ StringBuilder properties = new StringBuilder();
+ properties.append("EXTERNAL").append("=").append("TRUE");
+ properties.append(";");
+ properties.append(CAPABILITIES_KEY).append("=").append("HIVEBUCKET2,EXTREAD,EXTWRITE");
+ tProps.put("TBLNAME", tblName);
+ tProps.put("TBLTYPE", TableType.EXTERNAL_TABLE);
+ tProps.put("BUCKETS", bucketCount);
+ tProps.put("PROPERTIES", properties.toString());
+ table = createTableWithCapabilities(tProps);
+
+ parts = client.getPartitionsByNames(dbName, tblName, partValues, false);
+ LOG.debug("Return list size=" + parts.size());
+
+ for (Partition part : parts) {
+ assertEquals("Partition bucket count does not match", -1, part.getSd().getNumBuckets());
+ }
+
+ // processor has capabilities
+ capabilities.clear();
+ capabilities.add("HIVEBUCKET2");
+ setHMSClient("TestGetPartitionByNames#2", (String[])(capabilities.toArray(new String[0])));
+ parts = client.getPartitionsByNames(dbName, tblName, partValues, false);
+
+ for (Partition part : parts) {
+ assertEquals("Partition bucket count does not match", -1, part.getSd().getNumBuckets());
+ }
+
+ tblName = "test_parts_mgd_insert_wc";
+ properties = new StringBuilder();
+ properties.append("transactional=true");
+ properties.append(";");
+ properties.append(CAPABILITIES_KEY).append("=").append("HIVEMANAGEDINSERTREAD,HIVEMANAGEDINSERTWRITE,HIVECACHEINVALIDATE,")
+ .append("HIVEMANAGEDSTATS,CONNECTORREAD,CONNECTORWRITE");
+ properties.append(";");
+ properties.append("transactional_properties=insert_only");
+ tProps.put("TBLNAME", tblName);
+ tProps.put("PROPERTIES", properties.toString());
+ table = createTableWithCapabilities(tProps);
+
+ capabilities.clear();
+ capabilities.add("CONNECTORREAD");
+ setHMSClient("TestGetPartitionByNames#3", (String[])(capabilities.toArray(new String[0])));
+ parts = client.getPartitionsByNames(dbName, tblName, partValues, false);
+ assertEquals("Partition count does not match", pCount, parts.size());
+
+ LOG.info("Test execution complete:testGetPartitionsByNames");
+ } catch (Exception e) {
+ System.err.println(StringUtils.stringifyException(e));
+ System.err.println("testGetPartitionsByNames() failed.");
+ fail("testGetPartitions failed:" + e.getMessage());
+ } finally {
+ resetHMSClient();
+ }
+ }
+
+ private List<String> createTables(Map<String, Object> props) throws Exception {
+ int count = ((Integer)props.get("TABLECOUNT")).intValue();
+ String tblName = (String)props.get("TBLNAME");
+ List<String> caps = (List<String>)props.get("CAPABILITIES");
+ StringBuilder table_params = new StringBuilder();
+ if (caps != null)
+ table_params.append(CAPABILITIES_KEY).append("=").append(String.join(",", caps));
+ props.put("PROPERTIES", table_params.toString());
+ List<String> ret = new ArrayList<>();
+ String newtblName = null;
+
+ try {
+ for (int i = 0; i < count; i++) {
+ newtblName = tblName + i;
+ props.put("TBLNAME", newtblName);
+ createTableWithCapabilities(props);
+ props.put("DROPDB", Boolean.FALSE);
+ ret.add(newtblName);
+ }
+ } catch (Exception e) {
+ LOG.warn("Create table failed for " + newtblName);
+ }
+
+ return ret;
+ }
+
+ private Table createTableWithCapabilities(Map<String, Object> props) throws Exception {
+ String catalog = (String)props.getOrDefault("CATALOG", "testcat");
+ String dbName = (String)props.getOrDefault("DBNAME", "simpdb");
+ String tblName = (String)props.getOrDefault("TBLNAME", "test_table");
+ TableType type = (TableType)props.getOrDefault("TBLTYPE", TableType.MANAGED_TABLE);
+ int buckets = ((Integer)props.getOrDefault("BUCKETS", -1)).intValue();
+ String properties = (String)props.getOrDefault("PROPERTIES", "");
+ boolean dropDb = ((Boolean)props.getOrDefault("DROPDB", Boolean.TRUE)).booleanValue();
+ int partitionCount = ((Integer)props.getOrDefault("PARTITIONS", 0)).intValue();
+
+ final String typeName = "Person";
+
+ Map<String,String> table_params = new HashMap();
+ if (properties.length() > 0) {
+ String[] propArray = properties.split(";");
+ for (String prop : propArray) {
+ String[] keyValue = prop.split("=");
+ table_params.put(keyValue[0], keyValue[1]);
+ }
+ }
+
+ try {
+ client.dropTable(dbName, tblName);
+ } catch (Exception e) {
+ LOG.info("Drop table failed for " + dbName + "." + tblName);
+ }
+
+ try {
+ if (dropDb)
+ silentDropDatabase(dbName);
+ } catch (Exception e) {
+ LOG.info("Drop database failed for " + dbName);
+ }
+
+ if (dropDb)
+ new DatabaseBuilder()
+ .setName(dbName)
+ .create(client, conf);
+
+ try {
+ client.dropType(typeName);
+ } catch (Exception e) {
+ LOG.info("Drop type failed for " + typeName);
+ }
+
+ Type typ1 = new Type();
+ typ1.setName(typeName);
+ typ1.setFields(new ArrayList<>(2));
+ typ1.getFields().add(
+ new FieldSchema("name", ColumnType.STRING_TYPE_NAME, ""));
+ typ1.getFields().add(
+ new FieldSchema("income", ColumnType.INT_TYPE_NAME, ""));
+ client.createType(typ1);
+
+ TableBuilder builder = new TableBuilder()
+ .setDbName(dbName)
+ .setTableName(tblName)
+ .setCols(typ1.getFields())
+ .setType(type.name())
+ .setNumBuckets(buckets)
+ .setTableParams(table_params)
+ .addBucketCol("name")
+ .addStorageDescriptorParam("test_param_1", "Use this for comments etc");
+
+ if (buckets > 0)
+ builder.setNumBuckets(buckets).addBucketCol("name");
+
+ if (partitionCount > 0) {
+ builder.addPartCol("partcol", "string");
+ }
+
+ if (type == TableType.MANAGED_TABLE) {
+ if (properties.contains("transactional=true") && !properties.contains("transactional_properties=insert_only")) {
+ builder.setInputFormat("org.apache.hadoop.hive.ql.io.orc.OrcInputFormat");
+ builder.setOutputFormat("org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat");
+ builder.setSerdeLib("org.apache.hadoop.hive.ql.io.orc.OrcSerde");
+ builder.addStorageDescriptorParam("inputFormat", "org.apache.hadoop.hive.ql.io.orc.OrcInputFormat");
+ builder.addStorageDescriptorParam("outputFormat", "org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat");
+ }
+ }
+
+ Table tbl = builder.create(client, conf);
+ LOG.info("Table " + tblName + " created:type=" + type.name());
+
+ if (partitionCount > 0) {
+ List<Partition> partitions = new ArrayList<>();
+
+ List<List<String>> partValues = new ArrayList<>();
+ for (int i = 1; i <= partitionCount; i++) {
+ partValues.add(Lists.newArrayList("" + i));
+ }
+
+ for(List<String> vals : partValues){
+ addPartition(client, tbl, vals);
+ }
+ }
+
+ if (isThriftClient) {
+ // the createTable() above does not update the location in the 'tbl'
+ // object when the client is a thrift client and the code below relies
+ // on the location being present in the 'tbl' object - so get the table
+ // from the metastore
+ tbl = client.getTable(dbName, tblName);
+ }
+ return tbl;
+ }
+
+ private void addPartition(IMetaStoreClient client, Table table, List<String> values)
+ throws TException {
+ PartitionBuilder partitionBuilder = new PartitionBuilder().inTable(table);
+ values.forEach(val -> partitionBuilder.addValue(val));
+ Partition p = partitionBuilder.build(conf);
+ p.getSd().setNumBuckets(-1); // PartitionBuilder uses 0 as default whereas we use -1 for Tables.
+ client.add_partition(p);
+ }
+}
+
diff --git a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java
index d2c9c1b..a5286b5 100644
--- a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java
+++ b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java
@@ -3073,11 +3073,15 @@ public class HiveMetaStore extends ThriftHiveMetastore {
if (tables.size() > 0) {
tObjects = getMS().getTableObjectsByName(catalog, database, tables);
LOG.debug("get_tables_ext:getTableObjectsByName() returned " + tObjects.size());
- if (processorCapabilities != null && processorCapabilities.size() > 0) {
+ if (processorCapabilities == null || processorCapabilities.size() == 0 ||
+ processorCapabilities.contains("MANAGERAWMETADATA")) {
+ LOG.info("Skipping translation for processor with " + processorId);
+ } else {
if (transformer != null) {
Map<Table, List<String>> retMap = transformer.transform(tObjects, processorCapabilities, processorId);
for (Map.Entry<Table, List<String>> entry : retMap.entrySet()) {
+ LOG.debug("Table " + entry.getKey().getTableName() + " requires " + Arrays.toString((entry.getValue()).toArray()));
ret.add(convertTableToExtendedTable(entry.getKey(), entry.getValue(), req.getRequestedFields()));
}
} else {
@@ -3146,15 +3150,21 @@ public class HiveMetaStore extends ThriftHiveMetastore {
"insert-only tables", "get_table_req");
}
- if (processorCapabilities != null && processorCapabilities.size() > 0 && transformer != null) {
- List<Table> tList = new ArrayList<>();
- tList.add(t);
- Map<Table, List<String>> ret = transformer.transform(tList, processorCapabilities, processorId);
- if (ret.size() > 1) {
- LOG.warn("Unexpected resultset size:" + ret.size());
- throw new MetaException("Unexpected result from metadata transformer:return list size=" + ret.size());
+ if (processorCapabilities == null || processorCapabilities.size() == 0 ||
+ processorCapabilities.contains("MANAGERAWMETADATA")) {
+ LOG.info("Skipping translation for processor with " + processorId);
+ } else {
+ if (transformer != null) {
+ List<Table> tList = new ArrayList<>();
+ tList.add(t);
+ Map<Table, List<String>> ret = transformer.transform(tList, processorCapabilities, processorId);
+ if (ret.size() > 1) {
+ LOG.warn("Unexpected resultset size:" + ret.size());
+ throw new MetaException("Unexpected result from metadata transformer:return list size is " + ret.size());
+ }
+ t = (Table)(ret.keySet().iterator().next());
+ LOG.debug("Table " + t.getTableName() + " requires " + Arrays.toString((ret.get(t)).toArray()));
}
- t = (Table)(ret.keySet().iterator().next());
}
firePreEvent(new PreReadTableEvent(t, this));
@@ -5028,7 +5038,10 @@ public class HiveMetaStore extends ThriftHiveMetastore {
request.getFilterSpec());
List<String> processorCapabilities = request.getProcessorCapabilities();
String processorId = request.getProcessorIdentifier();
- if (processorCapabilities != null && processorCapabilities.size() > 0) {
+ if (processorCapabilities == null || processorCapabilities.size() == 0 ||
+ processorCapabilities.contains("MANAGERAWMETADATA")) {
+ LOG.info("Skipping translation for processor with " + processorId);
+ } else {
if (transformer != null) {
partitions = transformer.transformPartitions(partitions, processorCapabilities, processorId);
}
@@ -6614,8 +6627,11 @@ public class HiveMetaStore extends ThriftHiveMetastore {
}
}
- if (transformer != null) {
- if (processorCapabilities != null && processorCapabilities.size() > 0) {
+ if (processorCapabilities == null || processorCapabilities.size() == 0 ||
+ processorCapabilities.contains("MANAGERAWMETADATA")) {
+ LOG.info("Skipping translation for processor with " + processorId);
+ } else {
+ if (transformer != null) {
ret = transformer.transformPartitions(ret, processorCapabilities, processorId);
}
}
diff --git a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/MetastoreDefaultTransformer.java b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/MetastoreDefaultTransformer.java
index b07c1cd..7ecbef8 100644
--- a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/MetastoreDefaultTransformer.java
+++ b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/MetastoreDefaultTransformer.java
@@ -45,7 +45,6 @@ public class MetastoreDefaultTransformer implements IMetaStoreMetadataTransforme
private static final String CONNECTORWRITE = "CONNECTORWRITE".intern();
private static final String EXTWRITE = "EXTWRITE".intern();
private static final String EXTREAD = "EXTREAD".intern();
- private static final String HIVEACIDWRITE = "HIVEACIDWRITE".intern();
private static final String HIVEBUCKET2 = "HIVEBUCKET2".intern();
private static final String HIVECACHEINVALIDATE = "HIVECACHEINVALIDATE".intern();
private static final String HIVEFULLACIDREAD = "HIVEFULLACIDREAD".intern();
@@ -58,8 +57,30 @@ public class MetastoreDefaultTransformer implements IMetaStoreMetadataTransforme
private static final String HIVESQL = "HIVESQL".intern();
private static final String OBJCAPABILITIES = "OBJCAPABILITIES".intern();
+ private static final List<String> ACIDCOMMONWRITELIST = new ArrayList(Arrays.asList(
+ HIVEMANAGESTATS,
+ HIVECACHEINVALIDATE,
+ CONNECTORWRITE));
+
+ private static List<String> ACIDWRITELIST = new ArrayList<>(Arrays.asList(HIVEFULLACIDWRITE));
+ private static List<String> INSERTONLYWRITELIST = new ArrayList<>(Arrays.asList(HIVEMANAGEDINSERTWRITE));
+ private static List<String> MQTLIST = new ArrayList<>(Arrays.asList(HIVEFULLACIDREAD, HIVEONLYMQTWRITE,
+ HIVEMANAGESTATS, HIVEMQT, CONNECTORREAD));
+
+ private static List<String> ACIDLIST = new ArrayList<>();
+ private static List<String> INSERTONLYLIST = new ArrayList<>();
public MetastoreDefaultTransformer(IHMSHandler handler) throws HiveMetaException {
this.hmsHandler = handler;
+
+ ACIDWRITELIST.addAll(ACIDCOMMONWRITELIST);
+ ACIDLIST.addAll(ACIDWRITELIST);
+ ACIDLIST.add(HIVEFULLACIDREAD);
+ ACIDLIST.add(CONNECTORREAD);
+
+ INSERTONLYWRITELIST.addAll(ACIDCOMMONWRITELIST);
+ INSERTONLYLIST.addAll(INSERTONLYWRITELIST);
+ INSERTONLYLIST.add(HIVEMANAGEDINSERTREAD);
+ INSERTONLYLIST.add(CONNECTORREAD);
}
@Override
@@ -96,6 +117,7 @@ public class MetastoreDefaultTransformer implements IMetaStoreMetadataTransforme
StorageDescriptor newSd = new StorageDescriptor(table.getSd());
newSd.setNumBuckets(-1); // remove bucketing info
newTable.setSd(newSd);
+ LOG.info("Bucketed table without HIVEBUCKET2 capability, removed bucketing info from table");
}
} else { // Unbucketed, so RW for all
newTable.setAccessType(ACCESSTYPE_READWRITE);
@@ -106,8 +128,8 @@ public class MetastoreDefaultTransformer implements IMetaStoreMetadataTransforme
case "MANAGED_TABLE":
String txnal = params.get("transactional");
if (txnal == null || txnal.equalsIgnoreCase("FALSE")) { // non-ACID MANAGED table
- table.setAccessType(ACCESSTYPE_READWRITE);
- // table does not require any capabilities for full RW
+ table.setAccessType(ACCESSTYPE_READONLY);
+ generated.addAll(ACIDWRITELIST);
}
if (txnal != null && txnal.equalsIgnoreCase("TRUE")) { // ACID table
@@ -115,61 +137,64 @@ public class MetastoreDefaultTransformer implements IMetaStoreMetadataTransforme
processorCapabilities.contains(HIVEFULLACIDREAD) ||
processorCapabilities.contains(HIVEMANAGEDINSERTREAD))) {
table.setAccessType(ACCESSTYPE_NONE); // clients have no access to ACID tables without capabilities
- generated.add(CONNECTORREAD);
- generated.add(CONNECTORWRITE);
- generated.add(HIVECACHEINVALIDATE);
- generated.add(HIVEMANAGESTATS);
+ generated.addAll(ACIDLIST);
}
String txntype = params.get("transactional_properties");
if (txntype != null && txntype.equalsIgnoreCase("insert_only")) { // MICRO_MANAGED Tables
- generated.add(HIVEMANAGEDINSERTREAD);
- generated.add(HIVEMANAGEDINSERTWRITE);
// MGD table is insert only, not full ACID
if (processorCapabilities.contains(HIVEMANAGEDINSERTWRITE) || processorCapabilities.contains(CONNECTORWRITE)) {
table.setAccessType(ACCESSTYPE_READWRITE); // clients have RW access to INSERT-ONLY ACID tables
+ processorCapabilities.retainAll(INSERTONLYWRITELIST);
+ generated.addAll(processorCapabilities);
} else if (processorCapabilities.contains(HIVEMANAGEDINSERTREAD) || processorCapabilities.contains(CONNECTORREAD)) {
table.setAccessType(ACCESSTYPE_READONLY); // clients have RO access to INSERT-ONLY ACID tables
+ generated.addAll(INSERTONLYWRITELIST);
+ processorCapabilities.retainAll(getReads(INSERTONLYLIST));
+ generated.addAll(processorCapabilities);
} else {
table.setAccessType(ACCESSTYPE_NONE); // clients have NO access to INSERT-ONLY ACID tables
+ generated.addAll(INSERTONLYLIST);
}
} else { // FULL ACID MANAGED TABLE
- generated.add(HIVEFULLACIDWRITE);
- generated.add(HIVEFULLACIDREAD);
if (processorCapabilities.contains(HIVEFULLACIDWRITE) || processorCapabilities.contains(CONNECTORWRITE)) {
table.setAccessType(ACCESSTYPE_READWRITE); // clients have RW access to IUD ACID tables
+ processorCapabilities.retainAll(ACIDWRITELIST);
+ generated.addAll(processorCapabilities);
} else if (processorCapabilities.contains(HIVEFULLACIDREAD) || processorCapabilities.contains(CONNECTORREAD)) {
table.setAccessType(ACCESSTYPE_READONLY); // clients have RO access to IUD ACID tables
+ generated.addAll(ACIDWRITELIST);
+ processorCapabilities.retainAll(getReads(ACIDLIST));
+ generated.addAll(processorCapabilities);
} else {
table.setAccessType(ACCESSTYPE_NONE); // clients have NO access to IUD ACID tables
+ generated.addAll(ACIDLIST);
}
}
}
ret.put(table, generated);
break;
case "VIRTUAL_VIEW":
- generated.add("HIVESQL");
if (processorCapabilities.contains(HIVESQL) ||
processorCapabilities.contains(CONNECTORREAD)) {
table.setAccessType(ACCESSTYPE_READONLY);
} else {
table.setAccessType(ACCESSTYPE_NONE);
+ generated.add(HIVESQL);
+ generated.add(CONNECTORREAD);
}
ret.put(table, generated);
break;
case "MATERIALIZED_VIEW":
- generated.add(CONNECTORREAD);
- generated.add(HIVEFULLACIDREAD);
- generated.add(HIVEONLYMQTWRITE);
- generated.add(HIVEMANAGESTATS);
- generated.add(HIVEMQT);
if ((processorCapabilities.contains(CONNECTORREAD) ||
processorCapabilities.contains(HIVEFULLACIDREAD)) && processorCapabilities.contains(HIVEMQT)) {
LOG.info("Processor has one of the READ abilities and HIVEMQT, AccessType=RO");
table.setAccessType(ACCESSTYPE_READONLY);
+ generated.addAll(diff(MQTLIST, processorCapabilities));
} else {
LOG.info("Processor has no READ abilities or HIVEMQT, AccessType=None");
table.setAccessType(ACCESSTYPE_NONE);
+ generated.addAll(MQTLIST);
}
ret.put(table, generated);
break;
@@ -223,6 +248,13 @@ public class MetastoreDefaultTransformer implements IMetaStoreMetadataTransforme
ret.put(newTable, requiredCapabilities);
break;
case "MANAGED_TABLE":
+ if (processorCapabilities.size() == 0) { // processor has no capabilities
+ LOG.info("Client has no capabilities for type " + tableType + ",accesstype is NONE");
+ table.setAccessType(ACCESSTYPE_NONE);
+ ret.put(table, requiredCapabilities);
+ continue;
+ }
+
if (processorCapabilities.containsAll(requiredCapabilities)) {
// AccessType is RW
LOG.info("Abilities for match: Table type=" + tableType + ",accesstype is RW");
@@ -233,56 +265,94 @@ public class MetastoreDefaultTransformer implements IMetaStoreMetadataTransforme
String txnal = params.get("transactional");
if (txnal == null || txnal.equalsIgnoreCase("FALSE")) { // non-ACID MANAGED table
- LOG.info("Table is non ACID: ,accesstype is RW");
- table.setAccessType(ACCESSTYPE_READWRITE);
- // table does not require any capabilities for full RW
+ LOG.info("Table is non ACID, accesstype is RO");
+ table.setAccessType(ACCESSTYPE_READONLY);
+ List<String> missing = diff(requiredCapabilities, processorCapabilities);
+ ret.put(table, missing);
+ continue;
}
if (txnal != null && txnal.equalsIgnoreCase("TRUE")) { // ACID table
String txntype = params.get("transactional_properties");
+ List<String> hintList = new ArrayList<>();
if (txntype != null && txntype.equalsIgnoreCase("insert_only")) { // MICRO_MANAGED Tables
- LOG.info("Table is INSERTONLY ACID: ,accesstype is RW");
+ LOG.info("Table is INSERTONLY ACID");
// MGD table is insert only, not full ACID
- if ((processorCapabilities.contains(HIVEMANAGEDINSERTWRITE) && requiredCapabilities.contains(HIVEMANAGEDINSERTWRITE))
- || (processorCapabilities.contains(CONNECTORWRITE) && requiredCapabilities.contains(CONNECTORWRITE))) {
+ if (processorCapabilities.containsAll(getWrites(requiredCapabilities)) // contains all writes on table
+ || processorCapabilities.contains(HIVEFULLACIDWRITE)) {
+ LOG.info("Processor has all writes or full acid write, access is RW");
table.setAccessType(ACCESSTYPE_READWRITE); // clients have RW access to INSERT-ONLY ACID tables
- } else if ((processorCapabilities.contains(HIVEMANAGEDINSERTREAD) && requiredCapabilities.contains(HIVEMANAGEDINSERTREAD))
- || (processorCapabilities.contains(CONNECTORREAD) && requiredCapabilities.contains(CONNECTORREAD))) {
+ ret.put(table, hintList);
+ continue;
+ }
+
+ if (processorCapabilities.contains(CONNECTORWRITE)) {
+ table.setAccessType(ACCESSTYPE_READWRITE); // clients have RW access to INSERT-ONLY ACID tables with CONNWRITE
+ hintList.add(CONNECTORWRITE);
+ hintList.addAll(getReads(requiredCapabilities));
+ ret.put(table, hintList);
+ continue;
+ } else if (processorCapabilities.containsAll(getReads(requiredCapabilities))
+ || processorCapabilities.contains(CONNECTORREAD)) {
table.setAccessType(ACCESSTYPE_READONLY);
+ hintList.add(CONNECTORREAD);
+ hintList.addAll(diff(getWrites(requiredCapabilities), processorCapabilities));
+ ret.put(table, hintList);
+ continue;
} else {
table.setAccessType(ACCESSTYPE_NONE);
+ ret.put(table, requiredCapabilities);
+ continue;
}
} else { // MANAGED FULL ACID TABLES
- LOG.info("Table is FULL ACID:");
- if ((processorCapabilities.contains(HIVEFULLACIDWRITE) && requiredCapabilities.contains(HIVEFULLACIDWRITE))
- || (processorCapabilities.contains(CONNECTORWRITE) && requiredCapabilities.contains(CONNECTORWRITE))) {
+ LOG.info("Table is FULLACID");
+ if (processorCapabilities.containsAll(getWrites(requiredCapabilities)) // contains all writes on table
+ || processorCapabilities.contains(HIVEFULLACIDWRITE)) {
+ LOG.info("Processor has all writes or atleast " + HIVEFULLACIDWRITE + ", access is RW");
+ table.setAccessType(ACCESSTYPE_READWRITE); // clients have RW access to ACID tables
+ ret.put(table, hintList);
+ continue;
+ }
+
+ if(processorCapabilities.contains(CONNECTORWRITE)) {
table.setAccessType(ACCESSTYPE_READWRITE); // clients have RW access to IUD ACID tables
- } else if ((processorCapabilities.contains(HIVEFULLACIDREAD) && requiredCapabilities.contains(HIVEFULLACIDREAD))
- || (processorCapabilities.contains(CONNECTORREAD) && requiredCapabilities.contains(CONNECTORREAD))) {
+ hintList.add(CONNECTORWRITE);
+ hintList.addAll(getReads(requiredCapabilities));
+ ret.put(table, hintList);
+ continue;
+ } else if (processorCapabilities.contains(HIVEFULLACIDREAD)
+ || (processorCapabilities.contains(CONNECTORREAD) )) {
table.setAccessType(ACCESSTYPE_READONLY); // clients have RO access to IUD ACID tables
+ hintList.add(CONNECTORREAD);
+ hintList.addAll(diff(getWrites(requiredCapabilities), processorCapabilities));
+ ret.put(table, hintList);
+ continue;
} else {
table.setAccessType(ACCESSTYPE_NONE); // clients have NO access to IUD ACID tables
+ ret.put(table, requiredCapabilities);
+ continue;
}
}
}
- LOG.info("setting required to " + requiredCapabilities + ",MANAGED:Access=" + table.getAccessType());
- ret.put(table, requiredCapabilities);
+ LOG.info("setting required to " + ret.get(table) + ",MANAGED:Access=" + table.getAccessType());
break;
case "VIRTUAL_VIEW":
if (processorCapabilities.containsAll(requiredCapabilities)) {
table.setAccessType(ACCESSTYPE_READONLY);
+ ret.put(table, new ArrayList<String>());
} else {
table.setAccessType(ACCESSTYPE_NONE);
+ ret.put(table, requiredCapabilities);
}
- ret.put(table, requiredCapabilities);
break;
case "MATERIALIZED_VIEW":
if (processorCapabilities.containsAll(requiredCapabilities)) {
table.setAccessType(ACCESSTYPE_READONLY);
+ ret.put(table, new ArrayList<String>());
} else {
table.setAccessType(ACCESSTYPE_NONE);
+ ret.put(table, requiredCapabilities);
}
- ret.put(table, requiredCapabilities);
break;
default:
table.setAccessType(ACCESSTYPE_NONE);
@@ -401,4 +471,58 @@ public class MetastoreDefaultTransformer implements IMetaStoreMetadataTransforme
LOG.info("Returning partition set of size " + ret.size());
return ret;
}
+
+ // returns the elements contained in list1 but missing in list2
+ private static List<String> diff(final List<String> list1, final List<String> list2) {
+ List<String> diffList = new ArrayList<>();
+
+ if (list2 == null || list2.size() == 0)
+ return list1;
+
+ if (list1 == null || list1.size() == 0)
+ return new ArrayList<String>();
+
+ if (list2.containsAll(list1))
+ return new ArrayList<String>();
+
+ diffList.addAll(list2);
+ LOG.debug("diffList=" + Arrays.toString(diffList.toArray()) + ",master list=" + Arrays.toString(list1.toArray()));
+ if (diffList.retainAll(list1)) {
+ LOG.debug("diffList=" + Arrays.toString(diffList.toArray()));
+ if (diffList.size() == list1.size()) { // lists match
+ return new ArrayList<String>(); // return empty list indicating no missing elements
+ } else {
+ list1.removeAll(diffList);
+ LOG.debug("list1.size():" + list1.size());
+ return list1;
+ }
+ } else {
+ list1.removeAll(diffList);
+ LOG.debug("diff returning " + Arrays.toString(diffList.toArray()) + ",full list=" + Arrays.toString(list1.toArray()));
+ return list1;
+ }
+ }
+
+ private static List<String> getWrites(List<String> capabilities) {
+ List<String> writes = new ArrayList<>();
+ for (String capability : capabilities) {
+ if (capability.toUpperCase().endsWith("WRITE") ||
+ capability.toUpperCase().endsWith("STATS") ||
+ capability.toUpperCase().endsWith("INVALIDATE")) {
+ writes.add(capability);
+ }
+ }
+ return writes;
+ }
+
+ private static List<String> getReads(List<String> capabilities) {
+ List<String> reads = new ArrayList<>();
+ for (String capability : capabilities) {
+ if (capability.toUpperCase().endsWith("READ") ||
+ capability.toUpperCase().endsWith("SQL")) {
+ reads.add(capability);
+ }
+ }
+ return reads;
+ }
}
diff --git a/standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/TestHiveMetaStore.java b/standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/TestHiveMetaStore.java
index 0f2b857..0c4c84c 100644
--- a/standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/TestHiveMetaStore.java
+++ b/standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/TestHiveMetaStore.java
@@ -44,25 +44,14 @@ import java.lang.reflect.*;
import static org.mockito.Mockito.mock;
import com.google.common.collect.Sets;
-import org.apache.hadoop.hive.metastore.HiveMetaStoreClient.GetTablesRequestBuilder;
-import org.apache.hadoop.hive.metastore.TableType;
-import org.apache.hadoop.hive.metastore.api.ExtendedTableInfo;
import org.apache.hadoop.hive.metastore.api.GetPartitionsFilterSpec;
import org.apache.hadoop.hive.metastore.api.GetPartitionsProjectionSpec;
import org.apache.hadoop.hive.metastore.api.GetPartitionsRequest;
import org.apache.hadoop.hive.metastore.api.GetPartitionsResponse;
-import org.apache.hadoop.hive.metastore.api.GetTableRequest;
-import org.apache.hadoop.hive.metastore.api.GetTablesExtRequest;
-import org.apache.hadoop.hive.metastore.api.GetTablesExtRequestFields;
-import static org.apache.hadoop.hive.metastore.api.hive_metastoreConstants.ACCESSTYPE_NONE;
-import static org.apache.hadoop.hive.metastore.api.hive_metastoreConstants.ACCESSTYPE_READONLY;
-import static org.apache.hadoop.hive.metastore.api.hive_metastoreConstants.ACCESSTYPE_READWRITE;
-import static org.apache.hadoop.hive.metastore.api.hive_metastoreConstants.ACCESSTYPE_WRITEONLY;
import org.apache.hadoop.hive.metastore.api.PartitionSpec;
import org.apache.hadoop.hive.metastore.api.PartitionSpecWithSharedSD;
import org.apache.hadoop.hive.metastore.api.PartitionWithoutSD;
import org.apache.hadoop.hive.metastore.client.builder.DatabaseBuilder;
-import org.apache.hadoop.hive.metastore.client.builder.PartitionBuilder;
import org.apache.hadoop.hive.metastore.client.builder.TableBuilder;
import org.apache.hadoop.hive.metastore.conf.MetastoreConf;
import org.apache.hadoop.hive.metastore.conf.MetastoreConf.ConfVars;
@@ -132,7 +121,6 @@ public abstract class TestHiveMetaStore {
private static final String TEST_DB1_NAME = "testdb1";
private static final String TEST_DB2_NAME = "testdb2";
- private static final String CAPABILITIES_KEY = "OBJCAPABILITIES";
private static final int DEFAULT_LIMIT_PARTITION_REQUEST = 100;
@@ -155,8 +143,6 @@ public abstract class TestHiveMetaStore {
MetastoreConf.setLongVar(conf, ConfVars.BATCH_RETRIEVE_MAX, 2);
MetastoreConf.setLongVar(conf, ConfVars.LIMIT_PARTITION_REQUEST, DEFAULT_LIMIT_PARTITION_REQUEST);
MetastoreConf.setVar(conf, ConfVars.STORAGE_SCHEMA_READER_IMPL, "no.such.class");
-
- MetastoreConf.setVar(conf, ConfVars.METASTORE_METADATA_TRANSFORMER_CLASS, "org.apache.hadoop.hive.metastore.MetastoreDefaultTransformer");
}
@Test
@@ -3365,807 +3351,4 @@ public abstract class TestHiveMetaStore {
m.invoke(hms, tbl, part, false, null);
verify(wh, never()).getFileStatusesForLocation(part.getSd().getLocation());
}
-
- private void resetHMSClient() {
- client.setProcessorIdentifier(null);
- client.setProcessorCapabilities(null);
- }
-
- private void setHMSClient(String id, String[] caps) {
- client.setProcessorIdentifier(id);
- client.setProcessorCapabilities(caps);
- }
-
- /**
- * EXTERNAL_TABLE
- * 1) Old table with no capabilities
- * 2a) New table with capabilities with no client requirements
- * 2b) New table with capabilities with no matching client requirements
- * 2c) New table with capabilities with partial match requirements
- * 2d) New table with capabilities with full match requirements
- */
- @Test
- public void testTransformerExternalTable() throws Exception {
- try {
- final String dbName = "db1";
- String basetblName = "oldstyletable";
- Map<String, Object> tProps = new HashMap<>();
- int buckets = 32;
-
- String tblName = basetblName;
- tProps.put("DBNAME", dbName);
- tProps.put("TBLNAME", tblName);
- tProps.put("TBLTYPE", TableType.EXTERNAL_TABLE);
- tProps.put("BUCKETS", buckets);
- StringBuilder properties = new StringBuilder();
- properties.append("EXTERNAL").append("=").append("TRUE");
- tProps.put("PROPERTIES", properties.toString());
- Table tbl = createTableWithCapabilities(tProps);
-
- // retrieve the table
- Table tbl2 = client.getTable(dbName, tblName);
- LOG.info("Table=" + tblName + ",Access=" + tbl2.getAccessType());
- assertEquals("Created and retrieved tables do not match:" + tbl2.getTableName() + ":" + tblName,
- tbl2.getTableName(), tblName);
- assertEquals("TableType mismatch", TableType.EXTERNAL_TABLE.name(), tbl2.getTableType());
- assertEquals(buckets, tbl2.getSd().getNumBuckets()); // no transformation
- assertEquals("Table access type does not match expected value:" + tblName,
- 0, tbl2.getAccessType()); // old client, AccessType not set
-
- tblName = "test_ext_bucketed_wc";
- properties = new StringBuilder();
- properties.append("EXTERNAL").append("=").append("TRUE");
- properties.append(";");
- properties.append(CAPABILITIES_KEY).append("=").append("HIVEBUCKET2,EXTREAD,EXTWRITE");
- tProps.put("TBLNAME", tblName);
- tProps.put("PROPERTIES", properties.toString());
- tbl = createTableWithCapabilities(tProps);
-
- tbl2 = client.getTable(dbName, tblName);
- assertEquals("Table access type does not match expected value:" + tblName,
- 0, tbl2.getAccessType());
- assertEquals(buckets, tbl2.getSd().getNumBuckets()); // no tranformation
-
- setHMSClient("testTranformerExternalTable", (new String[] { "HIVEBUCKET2", "EXTREAD", "EXTWRITE"}));
- tbl2 = client.getTable(dbName, tblName);
- assertEquals("Table access type does not match the expected value:" + tblName,
- ACCESSTYPE_READWRITE, tbl2.getAccessType());
- assertEquals(buckets, tbl2.getSd().getNumBuckets()); // client has the HIVEBUCKET2 capability, retain bucketing info
-
- resetHMSClient();
-
- setHMSClient("testTransformerExternalTableRO", (new String[] { "EXTREAD", "EXTWRITE"}));
- tbl2 = client.getTable(dbName, tblName);
- assertEquals("Table access type does not match the expected value:" + tblName,
- ACCESSTYPE_READONLY, tbl2.getAccessType());
- assertEquals(-1, tbl2.getSd().getNumBuckets()); // client has no HIVEBUCKET2 capability, remove bucketing info
-
- resetHMSClient();
-
- tblName = "test_ext_unbucketed_wc";
- properties = new StringBuilder();
- properties.append(CAPABILITIES_KEY).append("=").append("EXTREAD,EXTWRITE");
- tProps.put("TBLNAME", tblName);
- tProps.put("PROPERTIES", properties.toString());
- tProps.remove("BUCKETS");
- tbl = createTableWithCapabilities(tProps);
-
- setHMSClient("TestTransformerExternalUnbucketed", (new String[] {"EXTREAD", "EXTWRITE"}));
- tbl2 = client.getTable(dbName, tblName);
- assertEquals("Table access type does not match the expected value:" + tblName,
- ACCESSTYPE_READWRITE, tbl2.getAccessType());
-
- resetHMSClient();
-
- tblName = "test_ext_sparkbucketed_wc";
- properties = new StringBuilder();
- properties.append("EXTERNAL").append("=").append("TRUE");
- properties.append(";");
- properties.append(CAPABILITIES_KEY).append("=").append("SPARKDECIMAL,SPARKBUCKET,EXTREAD");
- tProps.put("TBLNAME", tblName);
- tProps.put("PROPERTIES", properties.toString());
- tProps.put("BUCKETS", buckets);
- tbl = createTableWithCapabilities(tProps);
-
- setHMSClient("testTransformerExternalTableSpark", (new String[] { "HIVEFULLACIDREAD", "CONNECTORREAD"}));
- tbl2 = client.getTable(dbName, tblName);
- assertEquals("Table access type does not match the expected value:" + tblName,
- ACCESSTYPE_NONE, tbl2.getAccessType()); // requires EXTREAD for RO
- assertEquals(buckets, tbl2.getSd().getNumBuckets());
-
- setHMSClient("testTransformerExternalTableSpark", (new String[] { "EXTREAD", "CONNECTORREAD"}));
- tbl2 = client.getTable(dbName, tblName);
- assertEquals("Table access type does not match the expected value:" + tblName,
- ACCESSTYPE_READONLY, tbl2.getAccessType()); // requires EXTREAD for RO
- assertEquals(buckets, tbl2.getSd().getNumBuckets());
-
- setHMSClient("testTransformerExternalTableSpark", (new String[] { "SPARKBUCKET", "SPARKDECIMAL", "EXTREAD" }));
- tbl2 = client.getTable(dbName, tblName);
- assertEquals("Table access type does not match the expected value:" + tblName,
- ACCESSTYPE_READWRITE, tbl2.getAccessType());
- assertEquals(buckets, tbl2.getSd().getNumBuckets()); // client has SPARKBUCKET capability
-
- resetHMSClient();
-
- } catch (Exception e) {
- } finally {
- resetHMSClient();
- }
- }
-
- @Test
- public void testTransformerManagedTable() throws Exception {
- try {
- resetHMSClient();
-
- final String dbName = "db1";
- String basetblName = "oldstylemgdtable";
- Map<String, Object> tProps = new HashMap<>();
-
- String tblName = basetblName;
- tProps.put("DBNAME", dbName);
- tProps.put("TBLNAME", tblName);
- tProps.put("TBLTYPE", TableType.MANAGED_TABLE);
- // create unbucketed managed table with no capabilities
- Table tbl = createTableWithCapabilities(tProps);
-
- // retrieve the table
- Table tbl2 = client.getTable(dbName, tblName);
- LOG.info("Table=" + tblName + ",Access=" + tbl2.getAccessType());
- assertEquals("Created and retrieved tables do not match:" + tbl2.getTableName() + ":" + tblName,
- tbl2.getTableName(), tblName);
- assertEquals("TableType mismatch", TableType.MANAGED_TABLE.name(), tbl2.getTableType());
- assertEquals(-1, tbl2.getSd().getNumBuckets());
- assertEquals("Table access type does not match expected value:" + tblName,
- 0, tbl2.getAccessType()); // no translation to be done, so accessType not set
-
- // managed table with no capabilities
- tblName = "test_mgd_insert_woc";
- StringBuilder properties = new StringBuilder();
- tProps.put("TBLNAME", tblName);
- properties.append("transactional=true");
- properties.append(";");
- properties.append("transactional_properties='insert_only'");
- tProps.put("PROPERTIES", properties.toString());
- tbl = createTableWithCapabilities(tProps);
-
- tbl2 = client.getTable(dbName, tblName);
- assertEquals("Table access type does not match expected value:" + tblName,
- 0, tbl2.getAccessType()); // no transformation
-
- setHMSClient("testMGDwithConnectorRead", new String[] {"CONNECTORREAD"});
- tbl2 = client.getTable(dbName, tblName);
- assertEquals("Table access type does not match expected value:" + tblName,
- ACCESSTYPE_READONLY, tbl2.getAccessType());
- resetHMSClient();
-
- setHMSClient("testMGDwithInsertRead", new String[] {"HIVEMANAGEDINSERTREAD"});
- tbl2 = client.getTable(dbName, tblName);
- assertEquals("Table access type does not match expected value:" + tblName,
- ACCESSTYPE_READONLY, tbl2.getAccessType());
- resetHMSClient();
-
- setHMSClient("testMGDwithInsertWrite", new String[] {"HIVEMANAGEDINSERTWRITE"});
- tbl2 = client.getTable(dbName, tblName);
- assertEquals("Table access type does not match expected value:" + tblName,
- ACCESSTYPE_READWRITE, tbl2.getAccessType());
- resetHMSClient();
-
- setHMSClient("testMGDwithConnectorWrite", new String[] {"CONNECTORWRITE"});
- tbl2 = client.getTable(dbName, tblName);
- assertEquals("Table access type does not match expected value:" + tblName,
- ACCESSTYPE_READWRITE, tbl2.getAccessType());
- resetHMSClient();
-
- // bucketed table with no capabilities
- tblName = "test_mgd_insert_wc";
- properties = new StringBuilder();
- properties.append(CAPABILITIES_KEY).append("=").append("HIVEMANAGEDINSERTREAD,HIVEMANAGEDINSERTWRITE,HIVECACHEINVALIDATE")
- .append("HIVEMANAGEDSTATS,CONNECTORREAD,CONNECTORWRITE");
- properties.append(";");
- properties.append("transactional_properties='insert_only'");
- tProps.put("TBLNAME", tblName);
- tProps.put("PROPERTIES", properties.toString());
- tbl = createTableWithCapabilities(tProps);
-
- tbl2 = client.getTable(dbName, tblName);
- assertEquals("Table access type does not match expected value:" + tblName,
- ACCESSTYPE_NONE, tbl2.getAccessType());
-
- setHMSClient("testMGDwithAllWrites", new String[] { "HIVEMANAGEDINSERTWRITE", "HIVECACHEINVALIDATE",
- "HIVEMANAGEDSTATS" , "CONNECTORWRITE" });
- tbl2 = client.getTable(dbName, tblName);
- assertEquals("Table access type does not match expected value:" + tblName,
- ACCESSTYPE_READWRITE, tbl2.getAccessType());
- resetHMSClient();
-
- setHMSClient("testMGDwith1MissingWrite", new String[] {"HIVEMANAGEDINSERTREAD", "HIVEMANAGEDINSERTWRITE",
- "HIVECACHEINVALIDATE" });
- tbl2 = client.getTable(dbName, tblName);
- assertEquals("Table access type does not match expected value:" + tblName,
- ACCESSTYPE_READWRITE, tbl2.getAccessType());
- resetHMSClient();
-
- setHMSClient("testMGDwith1Write", new String[] {"CONNECTORWRITE"});
- tbl2 = client.getTable(dbName, tblName);
- assertEquals("Table access type does not match expected value:" + tblName,
- ACCESSTYPE_READWRITE, tbl2.getAccessType());
- resetHMSClient();
-
- setHMSClient("testMGDwithConnectorRead", new String[] {"CONNECTORREAD"});
- tbl2 = client.getTable(dbName, tblName);
- assertEquals("Table access type does not match expected value:" + tblName,
- ACCESSTYPE_READONLY, tbl2.getAccessType());
- resetHMSClient();
-
- // managed tables with no capabilities
- tblName = "test_mgd_full_woc";
- properties = new StringBuilder();
- tProps.put("TBLNAME", tblName);
- tProps.put("TBLTYPE", TableType.MANAGED_TABLE);
- tProps.put("PROPERTIES", properties.toString());
- tbl = createTableWithCapabilities(tProps);
-
- tbl2 = client.getTable(dbName, tblName);
- assertEquals("Table access type does not match expected value:" + tblName,
- ACCESSTYPE_NONE, tbl2.getAccessType());
-
- setHMSClient("testMGDwithConnectorRead", new String[] {"CONNECTORREAD"});
- tbl2 = client.getTable(dbName, tblName);
- assertEquals("Table access type does not match expected value:" + tblName,
- ACCESSTYPE_READONLY, tbl2.getAccessType());
- resetHMSClient();
-
- setHMSClient("testMGDwithFullACIDRead", new String[] {"HIVEFULLACIDREAD"});
- tbl2 = client.getTable(dbName, tblName);
- assertEquals("Table access type does not match expected value:" + tblName,
- ACCESSTYPE_READONLY, tbl2.getAccessType());
- resetHMSClient();
-
- setHMSClient("testMGDwith1Write", new String[] {"HIVEFULLACIDWRITE"});
- tbl2 = client.getTable(dbName, tblName);
- assertEquals("Table access type does not match expected value:" + tblName,
- ACCESSTYPE_READWRITE, tbl2.getAccessType());
- resetHMSClient();
-
- setHMSClient("testMGDwithConnectorWrite", new String[] {"CONNECTORWRITE"});
- tbl2 = client.getTable(dbName, tblName);
- assertEquals("Table access type does not match expected value:" + tblName,
- ACCESSTYPE_READWRITE, tbl2.getAccessType());
- resetHMSClient();
-
- // bucketed table with no capabilities
- tblName = "test_mgd_full_wc";
- properties = new StringBuilder();
- properties.append(CAPABILITIES_KEY).append("=").append("HIVEFULLACIDREAD,HIVEFULLACIDWRITE,HIVECACHEINVALIDATE")
- .append("HIVEMANAGEDSTATS,CONNECTORREAD,CONNECTORWRITE");
- tProps.put("TBLNAME", tblName);
- tProps.put("PROPERTIES", properties.toString());
- tbl = createTableWithCapabilities(tProps);
-
- tbl2 = client.getTable(dbName, tblName);
- assertEquals("Table access type does not match expected value:" + tblName,
- ACCESSTYPE_NONE, tbl2.getAccessType());
-
- setHMSClient("testMGDFULLwithAllWrites", new String[] { "HIVEFULLACIDWRITE", "HIVECACHEINVALIDATE",
- "HIVEMANAGEDSTATS", "CONNECTORWRITE" });
- tbl2 = client.getTable(dbName, tblName);
- assertEquals("Table access type does not match expected value:" + tblName,
- ACCESSTYPE_READWRITE, tbl2.getAccessType());
- resetHMSClient();
-
- setHMSClient("testMGDFULLwith1MissingWrite", new String[] { "HIVECACHEINVALIDATE",
- "HIVEMANAGEDSTATS", "CONNECTORWRITE" });
- tbl2 = client.getTable(dbName, tblName);
- assertEquals("Table access type does not match expected value:" + tblName,
- ACCESSTYPE_READWRITE, tbl2.getAccessType());
- resetHMSClient();
-
- setHMSClient("testMGDFULLwith1Write", new String[] { "CONNECTORWRITE" });
- tbl2 = client.getTable(dbName, tblName);
- assertEquals("Table access type does not match expected value:" + tblName,
- ACCESSTYPE_READWRITE, tbl2.getAccessType());
- resetHMSClient();
-
- setHMSClient("testMGDwith1MissingWrite", new String[] {"HIVEFULLACIDREAD", "HIVEFULLACIDWRITE",
- "HIVECACHEINVALIDATE" });
- tbl2 = client.getTable(dbName, tblName);
- assertEquals("Table access type does not match expected value:" + tblName,
- ACCESSTYPE_READONLY, tbl2.getAccessType());
- resetHMSClient();
-
- setHMSClient("testMGDwithConnectorRead", new String[] {"CONNECTORREAD"});
- tbl2 = client.getTable(dbName, tblName);
- assertEquals("Table access type does not match expected value:" + tblName,
- ACCESSTYPE_READONLY, tbl2.getAccessType());
- resetHMSClient();
-
- setHMSClient("testTranformerExternalTable", (new String[] { "HIVEBUCKET2", "EXTREAD", "EXTWRITE"}));
- tbl2 = client.getTable(dbName, tblName);
- assertEquals("Table access type does not match the expected value:" + tblName,
- ACCESSTYPE_READWRITE, tbl2.getAccessType());
- assertEquals(100, tbl2.getSd().getNumBuckets());
-
- resetHMSClient();
-
- } catch (Exception e) {
- } finally {
- resetHMSClient();
- }
- }
-
- @Test
- public void testTransformerVirtualView() throws Exception {
- try {
- final String dbName = "db1";
- String basetblName = "oldstyleview";
- Map<String, Object> tProps = new HashMap<>();
-
- String tblName = basetblName;
- tProps.put("DBNAME", dbName);
- tProps.put("TBLNAME", tblName);
- tProps.put("TBLTYPE", TableType.VIRTUAL_VIEW);
- StringBuilder properties = new StringBuilder();
- tProps.put("PROPERTIES", properties.toString());
- Table tbl = createTableWithCapabilities(tProps);
-
- // retrieve the table
- Table tbl2 = client.getTable(dbName, tblName);
- LOG.info("View=" + tblName + ",Access=" + tbl2.getAccessType());
- assertEquals("Created and retrieved views do not match:" + tbl2.getTableName() + ":" + tblName,
- tbl2.getTableName(), tblName);
- assertEquals("TableType mismatch", TableType.VIRTUAL_VIEW.name(), tbl2.getTableType());
- assertEquals("Table access type does not match expected value:" + tblName,
- 0, tbl2.getAccessType()); // old client, AccessType not set
-
- // table has capabilities
- tblName = "test_view_wc";
- properties = new StringBuilder();
- properties.append(CAPABILITIES_KEY).append("=").append("HIVESQL");
- tProps.put("TBLNAME", tblName);
- tProps.put("PROPERTIES", properties.toString());
- tbl = createTableWithCapabilities(tProps);
-
- tbl2 = client.getTable(dbName, tblName);
- assertEquals("Table access type does not match expected value:" + tblName,
- 0, tbl2.getAccessType()); // old client, no transformation
-
- setHMSClient("testTransformerVirtualView", (new String[] { "HIVESQL" }));
- tbl2 = client.getTable(dbName, tblName);
- assertEquals("Table access type does not match the expected value:" + tblName,
- ACCESSTYPE_READONLY, tbl2.getAccessType()); // RO accessonly for all views
-
- setHMSClient("testTransformerVirtualView", (new String[] { "EXTWRITE", "HIVEFULLACIDWRITE" }));
- tbl2 = client.getTable(dbName, tblName);
- assertEquals("Table access type does not match the expected value:" + tblName,
- ACCESSTYPE_NONE, tbl2.getAccessType()); // missing HIVESQL
-
- resetHMSClient();
-
- // table does not capabilities but client is newer
- tblName = "test_view_woc";
- properties = new StringBuilder();
- tProps.put("TBLNAME", tblName);
- tProps.put("PROPERTIES", properties.toString());
- tbl = createTableWithCapabilities(tProps);
-
- setHMSClient("testTransformerVirtualView", (new String[] { "HIVESQL" }));
- tbl2 = client.getTable(dbName, tblName);
- assertEquals("Table access type does not match the expected value:" + tblName,
- ACCESSTYPE_READONLY, tbl2.getAccessType()); // RO accessonly for all views
-
- setHMSClient("testTransformerVirtualView", (new String[] { "EXTWRITE", "HIVEFULLACIDWRITE" }));
- tbl2 = client.getTable(dbName, tblName);
- assertEquals("Table access type does not match the expected value:" + tblName,
- ACCESSTYPE_NONE, tbl2.getAccessType()); // missing HIVESQL
- } catch (Exception e) {
- } finally {
- resetHMSClient();
- }
- }
-
- @Test
- public void testTransformerMaterializedView() throws Exception {
- try {
- resetHMSClient();
-
- final String dbName = "db1";
- String basetblName = "oldstylemqtview";
- Map<String, Object> tProps = new HashMap<>();
-
- String tblName = basetblName;
- tProps.put("DBNAME", dbName);
- tProps.put("TBLNAME", tblName);
- tProps.put("TBLTYPE", TableType.MATERIALIZED_VIEW);
- StringBuilder properties = new StringBuilder();
- tProps.put("PROPERTIES", properties.toString());
- Table tbl = createTableWithCapabilities(tProps);
-
- // retrieve the table
- Table tbl2 = client.getTable(dbName, tblName);
- LOG.info("View=" + tblName + ",Access=" + tbl2.getAccessType());
- assertEquals("Created and retrieved views do not match:" + tbl2.getTableName() + ":" + tblName,
- tbl2.getTableName(), tblName);
- assertEquals("TableType mismatch", TableType.MATERIALIZED_VIEW.name(), tbl2.getTableType());
- assertEquals("Table access type does not match expected value:" + tblName,
- 0, tbl2.getAccessType()); // old client, AccessType not set
-
- // table has capabilities
- tblName = "test_mqtview_wc";
- properties = new StringBuilder();
- properties.append(CAPABILITIES_KEY).append("=").append("HIVEFULLACIDREAD,HIVEONLYMQTWRITE,HIVEMANAGESTATS,HIVEMQT,CONNECTORREAD");
- tProps.put("TBLNAME", tblName);
- tProps.put("PROPERTIES", properties.toString());
- tbl = createTableWithCapabilities(tProps);
-
- tbl2 = client.getTable(dbName, tblName);
- assertEquals("Table access type does not match expected value:" + tblName,
- 0, tbl2.getAccessType()); // View has capabilities, processor doesnt, no tranformation
-
- setHMSClient("testTransformerMQTFullSet", (new String[] { "HIVEFULLACIDREAD", "HIVEONLYMQTWRITE",
- "HIVEMANAGESTATS", "HIVEMQT", "CONNECTORREAD" }));
- tbl2 = client.getTable(dbName, tblName);
- assertEquals("Table access type does not match the expected value:" + tblName,
- ACCESSTYPE_READONLY, tbl2.getAccessType()); // RO accessonly for all views
-
- setHMSClient("testTransformerMQTFullRead", (new String[] { "HIVEFULLACIDREAD", "HIVEMQT" }));
- tbl2 = client.getTable(dbName, tblName);
- assertEquals("Table access type does not match the expected value:" + tblName,
- ACCESSTYPE_NONE, tbl2.getAccessType());
-
- setHMSClient("testTransformerMQTConnRead", (new String[] { "CONNECTORREAD", "HIVEMQT" }));
- tbl2 = client.getTable(dbName, tblName);
- assertEquals("Table access type does not match the expected value:" + tblName,
- ACCESSTYPE_NONE, tbl2.getAccessType()); // RO accessonly for all views
-
- setHMSClient("testTransformerMQTDummySet", (new String[] { "EXTWRITE", "HIVEFULLACIDWRITE" }));
- tbl2 = client.getTable(dbName, tblName);
- assertEquals("Table access type does not match the expected value:" + tblName,
- ACCESSTYPE_NONE, tbl2.getAccessType()); // missing HIVEMQT + *READ
-
- resetHMSClient();
-
- // table does not capabilities but client is newer
- tblName = "test_mqtview_woc";
- properties = new StringBuilder();
- tProps.put("TBLNAME", tblName);
- tProps.put("PROPERTIES", properties.toString());
- tbl = createTableWithCapabilities(tProps);
-
- setHMSClient("testTransformerVirtualView", (new String[] { "HIVEFULLACIDREAD", "HIVEMQT" }));
- tbl2 = client.getTable(dbName, tblName);
- assertEquals("Table access type does not match the expected value:" + tblName,
- ACCESSTYPE_READONLY, tbl2.getAccessType()); // RO accessonly for all views
-
- setHMSClient("testTransformerMQTConnRead", (new String[] { "CONNECTORREAD", "HIVEMQT" }));
- tbl2 = client.getTable(dbName, tblName);
- assertEquals("Table access type does not match the expected value:" + tblName,
- ACCESSTYPE_READONLY, tbl2.getAccessType()); // RO accessonly for all views
-
- setHMSClient("testTransformerVirtualView", (new String[] { "EXTWRITE, HIVEFULLACIDWRITE" }));
- tbl2 = client.getTable(dbName, tblName);
- assertEquals("Table access type does not match the expected value:" + tblName,
- ACCESSTYPE_NONE, tbl2.getAccessType()); // missing HIVEMQT + *READ
- } catch (Exception e) {
- } finally {
- resetHMSClient();
- }
- }
-
- @Test
- public void testGetTablesExt() throws Exception {
- try {
- resetHMSClient();
-
- final String dbName = "db1";
- String tblName = "test_get_tables_ext";
- int count = 10;
- Map<String, Object> tProps = new HashMap<>();
-
- tProps.put("DBNAME", dbName);
- tProps.put("TBLNAME", tblName);
-
- StringBuilder table_params = new StringBuilder();
- List<String> capabilities = new ArrayList<>();
- TableType type = TableType.EXTERNAL_TABLE;
-
- capabilities.add("HIVEFULLACIDWRITE");
- capabilities.add("HIVEFULLACIDREAD");
- capabilities.add("HIVECACHEINVALIDATE");
- capabilities.add("CONNECTORREAD");
- capabilities.add("CONNECTORWRITE");
-
- tProps.put("CAPABILITIES", capabilities);
- tProps.put("TABLECOUNT", count);
- tProps.put("TBLTYPE", type);
-
- setHMSClient("test_get_tables_ext", (String[])(capabilities.toArray(new String[0])));
-
- List<String> tables = createTables(tProps);
- int requestedFields = (new GetTablesRequestBuilder().with(GetTablesExtRequestFields.PROCESSOR_CAPABILITIES)).bitValue();
- List<ExtendedTableInfo> extTables = client.getTablesExt(null, dbName, "*", requestedFields, 2000);
- LOG.debug("Return list size=" + extTables.size() + ",bitValue=" + requestedFields);
- assertEquals("Return list size does not match expected size:extTables", count, extTables.size());
- for (ExtendedTableInfo tableInfo : extTables) {
- assertEquals("Capability set size does not match", capabilities.size(), tableInfo.getProcessorCapabilities().size());
- assertEquals("AccessType not expected to be set", 0, tableInfo.getAccessType());
- }
-
- requestedFields = (new GetTablesRequestBuilder().with(GetTablesExtRequestFields.ACCESS_TYPE)).bitValue();
- extTables = client.getTablesExt(null, dbName, "*", requestedFields, 2000);
- LOG.debug("Return list size=" + extTables.size() + ",bitValue=" + requestedFields);
- assertEquals("Return list size does not match expected size", count, extTables.size());
- for (ExtendedTableInfo tableInfo : extTables) {
- assertNull("Return value should not contain capabilities", tableInfo.getProcessorCapabilities());
- assertTrue("AccessType expected to be set", tableInfo.getAccessType() > 0);
- }
-
- requestedFields = (new GetTablesRequestBuilder().with(GetTablesExtRequestFields.ALL)).bitValue();
- extTables = client.getTablesExt(null, dbName, "*", requestedFields, 2000);
- LOG.debug("Return list size=" + extTables.size() + ",bitValue=" + requestedFields);
- assertEquals("Return list size does not match expected size", count, extTables.size());
- for (ExtendedTableInfo tableInfo : extTables) {
- assertEquals("Capability set size does not match", capabilities.size(), tableInfo.getProcessorCapabilities().size());
- assertTrue("AccessType expected to be set", tableInfo.getAccessType() > 0);
- }
-
- extTables = client.getTablesExt(null, dbName, "*", requestedFields, (count - 3));
- LOG.debug("Return list size=" + extTables.size() + ",bitValue=" + requestedFields);
- assertEquals("Return list size does not match expected size", (count - 3), extTables.size());
- for (ExtendedTableInfo tableInfo : extTables) {
- assertEquals("Capability set size does not match", capabilities.size(), tableInfo.getProcessorCapabilities().size());
- assertTrue("AccessType expected to be set", tableInfo.getAccessType() > 0);
- }
-
- extTables = client.getTablesExt(null, dbName, "*", requestedFields, -1);
- LOG.debug("Return list size=" + extTables.size() + ",bitValue=" + requestedFields);
- assertEquals("Return list size does not match expected size", count, extTables.size());
-
- count = 300;
- tProps.put("TBLNAME", "test_limit");
- tProps.put("TABLECOUNT", count);
- tables = createTables(tProps);
- assertEquals("Unexpected number of tables created", count, tables.size());
-
- extTables = client.getTablesExt(null, dbName, "test_limit*", requestedFields, count);
- assertEquals("Unexpected number of tables returned", count, extTables.size());
-
- extTables = client.getTablesExt(null, dbName, "test_limit*", requestedFields, (count/2));
- assertEquals("Unexpected number of tables returned", (count/2), extTables.size());
-
- extTables = client.getTablesExt(null, dbName, "test_limit*", requestedFields, 1);
- assertEquals("Unexpected number of tables returned", 1, extTables.size());
-
- } catch (Exception e) {
- System.err.println(StringUtils.stringifyException(e));
- System.err.println("testGetTablesExt() failed.");
- throw e;
- } finally {
- resetHMSClient();
- }
- }
-
- @Test
- public void testGetPartitionsByNames() throws Exception {
- try {
- resetHMSClient();
-
- final String dbName = "db1";
- String tblName = "test_get_parts_by_names";
- int count = 1;
- int pCount = 10;
- int bucketCount = 100;
- Map<String, Object> tProps = new HashMap<>();
-
- tProps.put("DBNAME", dbName);
- tProps.put("TBLNAME", tblName);
-
- List<String> capabilities = new ArrayList<>();
- TableType type = TableType.MANAGED_TABLE;
-
- capabilities.add("HIVEFULLACIDWRITE");
- capabilities.add("HIVEFULLACIDREAD");
- capabilities.add("HIVECACHEINVALIDATE");
- capabilities.add("CONNECTORREAD");
- capabilities.add("CONNECTORWRITE");
-
- // tProps.put("CAPABILITIES", capabilities);
- tProps.put("TBLTYPE", type);
- tProps.put("PARTITIONS", pCount);
-
- setHMSClient("TestGetPartitionByNames#1", (String[])(capabilities.toArray(new String[0])));
-
- Table table = createTableWithCapabilities(tProps);
-
- List<String> partNames = new ArrayList<>();
- List<String> partValues = new ArrayList<>();
- for (int i = 1; i <= pCount; i++) {
- partValues.add("partcol=" + i);
- }
- List<Partition> parts = client.getPartitionsByNames(dbName, tblName, partValues, false);
- assertEquals("Return list size does not match expected size", pCount, parts.size());
-
- tblName = "test_gp_ext_bucketed_wc";
- StringBuilder properties = new StringBuilder();
- properties.append("EXTERNAL").append("=").append("TRUE");
- properties.append(";");
- properties.append(CAPABILITIES_KEY).append("=").append("HIVEBUCKET2,EXTREAD,EXTWRITE");
- tProps.put("TBLNAME", tblName);
- tProps.put("TBLTYPE", TableType.EXTERNAL_TABLE);
- tProps.put("BUCKETS", bucketCount);
- tProps.put("PROPERTIES", properties.toString());
- table = createTableWithCapabilities(tProps);
-
- parts = client.getPartitionsByNames(dbName, tblName, partValues, false);
- LOG.debug("Return list size=" + parts.size());
-
- for (Partition part : parts) {
- assertEquals("Partition bucket count does not match", -1, part.getSd().getNumBuckets());
- }
-
- // processor has capabilities
- capabilities.clear();
- capabilities.add("HIVEBUCKET2");
- setHMSClient("TestGetPartitionByNames#2", (String[])(capabilities.toArray(new String[0])));
- parts = client.getPartitionsByNames(dbName, tblName, partValues, false);
-
- for (Partition part : parts) {
- assertEquals("Partition bucket count does not match", -1, part.getSd().getNumBuckets());
- }
-
- tblName = "test_parts_mgd_insert_wc";
- properties = new StringBuilder();
- properties.append(CAPABILITIES_KEY).append("=").append("HIVEMANAGEDINSERTREAD,HIVEMANAGEDINSERTWRITE,HIVECACHEINVALIDATE")
- .append("HIVEMANAGEDSTATS,CONNECTORREAD,CONNECTORWRITE");
- properties.append(";");
- properties.append("transactional_properties='insert_only'");
- tProps.put("TBLNAME", tblName);
- tProps.put("PROPERTIES", properties.toString());
- table = createTableWithCapabilities(tProps);
-
- capabilities.clear();
- capabilities.add("CONNECTORREAD");
- setHMSClient("TestGetPartitionByNames#3", (String[])(capabilities.toArray(new String[0])));
- parts = client.getPartitionsByNames(dbName, tblName, partValues, false);
- assertEquals("Partition count does not match", pCount, parts.size());
-
- } catch (Exception e) {
- System.err.println(StringUtils.stringifyException(e));
- System.err.println("testGetPartitionsByNames() failed.");
- throw e;
- } finally {
- resetHMSClient();
- }
- }
-
- private List<String> createTables(Map<String, Object> props) throws Exception {
- int count = ((Integer)props.get("TABLECOUNT")).intValue();
- String tblName = (String)props.get("TBLNAME");
- List<String> caps = (List<String>)props.get("CAPABILITIES");
- StringBuilder table_params = new StringBuilder();
- table_params.append(CAPABILITIES_KEY).append("=").append(String.join(",", caps));
- props.put("PROPERTIES", table_params.toString());
- List<String> ret = new ArrayList<>();
- String newtblName = null;
-
- try {
- for (int i = 0; i < count; i++) {
- newtblName = tblName + i;
- props.put("TBLNAME", newtblName);
- props.put("DROPDB", Boolean.FALSE);
- createTableWithCapabilities(props);
- ret.add(newtblName);
- }
- } catch (Exception e) {
- LOG.warn("Create table failed for " + newtblName);
- }
-
- return ret;
- }
-
- private Table createTableWithCapabilities(Map<String, Object> props) throws Exception {
- String catalog = (String)props.getOrDefault("CATALOG", "testcat");
- String dbName = (String)props.getOrDefault("DBNAME", "simpdb");
- String tblName = (String)props.getOrDefault("TBLNAME", "test_table");
- TableType type = (TableType)props.getOrDefault("TBLTYPE", TableType.MANAGED_TABLE);
- int buckets = ((Integer)props.getOrDefault("BUCKETS", -1)).intValue();
- String properties = (String)props.getOrDefault("PROPERTIES", "");
- boolean dropDb = ((Boolean)props.getOrDefault("DROPDB", Boolean.TRUE)).booleanValue();
- int partitionCount = ((Integer)props.getOrDefault("PARTITIONS", 0)).intValue();
-
- final String typeName = "Person";
-
- Map<String,String> table_params = new HashMap();
- if (properties.length() > 0) {
- String[] propArray = properties.split(";");
- for (String prop : propArray) {
- String[] keyValue = prop.split("=");
- table_params.put(keyValue[0], keyValue[1]);
- }
- }
-
- try {
- client.dropTable(dbName, tblName);
- } catch (Exception e) {
- LOG.info("Drop table failed for " + dbName + "." + tblName);
- }
-
- try {
- if (dropDb)
- silentDropDatabase(dbName);
- } catch (Exception e) {
- LOG.info("Drop database failed for " + dbName);
- }
-
- if (dropDb)
- new DatabaseBuilder()
- .setName(dbName)
- .create(client, conf);
-
- try {
- client.dropType(typeName);
- } catch (Exception e) {
- LOG.info("Drop type failed for " + typeName);
- }
-
- Type typ1 = new Type();
- typ1.setName(typeName);
- typ1.setFields(new ArrayList<>(2));
- typ1.getFields().add(
- new FieldSchema("name", ColumnType.STRING_TYPE_NAME, ""));
- typ1.getFields().add(
- new FieldSchema("income", ColumnType.INT_TYPE_NAME, ""));
- client.createType(typ1);
-
- TableBuilder builder = new TableBuilder()
- .setDbName(dbName)
- .setTableName(tblName)
- .setCols(typ1.getFields())
- .setType(type.name())
- .setNumBuckets(buckets)
- .setTableParams(table_params)
- .addBucketCol("name")
- .addStorageDescriptorParam("test_param_1", "Use this for comments etc");
-
- if (buckets > 0)
- builder.setNumBuckets(buckets).addBucketCol("name");
-
- if (partitionCount > 0) {
- builder.addPartCol("partcol", "string");
- }
-
- Table tbl = builder.create(client, conf);
- LOG.info("Table " + tblName + " created:type=" + type.name());
-
- if (partitionCount > 0) {
- List<Partition> partitions = new ArrayList<>();
-
- List<List<String>> partValues = new ArrayList<>();
- for (int i = 1; i <= partitionCount; i++) {
- partValues.add(Lists.newArrayList("" + i));
- }
-
- for(List<String> vals : partValues){
- addPartition(client, tbl, vals);
- }
- }
-
- if (isThriftClient) {
- // the createTable() above does not update the location in the 'tbl'
- // object when the client is a thrift client and the code below relies
- // on the location being present in the 'tbl' object - so get the table
- // from the metastore
- tbl = client.getTable(dbName, tblName);
- }
- return tbl;
- }
-
- private void addPartition(IMetaStoreClient client, Table table, List<String> values)
- throws TException {
- PartitionBuilder partitionBuilder = new PartitionBuilder().inTable(table);
- values.forEach(val -> partitionBuilder.addValue(val));
- Partition p = partitionBuilder.build(conf);
- p.getSd().setNumBuckets(-1); // PartitionBuilder uses 0 as default whereas we use -1 for Tables.
- client.add_partition(p);
- }
}