You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by pv...@apache.org on 2018/06/07 08:46:56 UTC
hive git commit: HIVE-19503: Create a test that checks for
dropPartitions with directSql (Peter Vary, reviewed by Vihang Karajgaonkar)
Repository: hive
Updated Branches:
refs/heads/master ec0636c06 -> cfd57348c
HIVE-19503: Create a test that checks for dropPartitions with directSql (Peter Vary, reviewed by Vihang Karajgaonkar)
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/cfd57348
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/cfd57348
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/cfd57348
Branch: refs/heads/master
Commit: cfd57348c1ac188e0ba131d5636a62ff7b7c27be
Parents: ec0636c
Author: Peter Vary <pv...@cloudera.com>
Authored: Thu Jun 7 10:46:08 2018 +0200
Committer: Peter Vary <pv...@cloudera.com>
Committed: Thu Jun 7 10:46:08 2018 +0200
----------------------------------------------------------------------
.../hive/metastore/MetaStoreDirectSql.java | 4 +
.../hadoop/hive/metastore/ObjectStore.java | 9 +-
.../hadoop/hive/metastore/TestObjectStore.java | 155 +++++++++++++++++++
3 files changed, 167 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/cfd57348/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/MetaStoreDirectSql.java
----------------------------------------------------------------------
diff --git a/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/MetaStoreDirectSql.java b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/MetaStoreDirectSql.java
index 3be94e8..285ca3f 100644
--- a/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/MetaStoreDirectSql.java
+++ b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/MetaStoreDirectSql.java
@@ -77,7 +77,9 @@ import org.apache.hadoop.hive.metastore.model.MCreationMetadata;
import org.apache.hadoop.hive.metastore.model.MDatabase;
import org.apache.hadoop.hive.metastore.model.MNotificationLog;
import org.apache.hadoop.hive.metastore.model.MNotificationNextId;
+import org.apache.hadoop.hive.metastore.model.MPartitionColumnPrivilege;
import org.apache.hadoop.hive.metastore.model.MPartitionColumnStatistics;
+import org.apache.hadoop.hive.metastore.model.MPartitionPrivilege;
import org.apache.hadoop.hive.metastore.model.MTableColumnStatistics;
import org.apache.hadoop.hive.metastore.model.MWMResourcePlan;
import org.apache.hadoop.hive.metastore.parser.ExpressionTree;
@@ -236,6 +238,8 @@ class MetaStoreDirectSql {
initQueries.add(pm.newQuery(MNotificationNextId.class, "nextEventId < -1"));
initQueries.add(pm.newQuery(MWMResourcePlan.class, "name == ''"));
initQueries.add(pm.newQuery(MCreationMetadata.class, "dbName == ''"));
+ initQueries.add(pm.newQuery(MPartitionPrivilege.class, "principalName == ''"));
+ initQueries.add(pm.newQuery(MPartitionColumnPrivilege.class, "principalName == ''"));
Query q;
while ((q = initQueries.peekFirst()) != null) {
q.execute();
http://git-wip-us.apache.org/repos/asf/hive/blob/cfd57348/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/ObjectStore.java
----------------------------------------------------------------------
diff --git a/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/ObjectStore.java b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/ObjectStore.java
index e3665b2..b3a8dd0 100644
--- a/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/ObjectStore.java
+++ b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/ObjectStore.java
@@ -2648,10 +2648,17 @@ public class ObjectStore implements RawStore, Configurable {
@Override
public void dropPartitions(String catName, String dbName, String tblName, List<String> partNames)
throws MetaException, NoSuchObjectException {
+ dropPartitionsInternal(catName, dbName, tblName, partNames, true, true);
+ }
+
+ @VisibleForTesting
+ void dropPartitionsInternal(String catName, String dbName, String tblName,
+ List<String> partNames, boolean allowSql, boolean allowJdo)
+ throws MetaException, NoSuchObjectException {
if (CollectionUtils.isEmpty(partNames)) {
return;
}
- new GetListHelper<Void>(catName, dbName, tblName, true, true) {
+ new GetListHelper<Void>(catName, dbName, tblName, allowSql, allowJdo) {
@Override
protected List<Void> getSqlResult(GetHelper<List<Void>> ctx) throws MetaException {
directSql.dropPartitionsViaSqlFilter(catName, dbName, tblName, partNames);
http://git-wip-us.apache.org/repos/asf/hive/blob/cfd57348/standalone-metastore/src/test/java/org/apache/hadoop/hive/metastore/TestObjectStore.java
----------------------------------------------------------------------
diff --git a/standalone-metastore/src/test/java/org/apache/hadoop/hive/metastore/TestObjectStore.java b/standalone-metastore/src/test/java/org/apache/hadoop/hive/metastore/TestObjectStore.java
index 9912213..7984af6 100644
--- a/standalone-metastore/src/test/java/org/apache/hadoop/hive/metastore/TestObjectStore.java
+++ b/standalone-metastore/src/test/java/org/apache/hadoop/hive/metastore/TestObjectStore.java
@@ -45,6 +45,8 @@ import org.apache.hadoop.hive.metastore.api.StorageDescriptor;
import org.apache.hadoop.hive.metastore.api.Table;
import org.apache.hadoop.hive.metastore.client.builder.CatalogBuilder;
import org.apache.hadoop.hive.metastore.client.builder.DatabaseBuilder;
+import org.apache.hadoop.hive.metastore.client.builder.PartitionBuilder;
+import org.apache.hadoop.hive.metastore.client.builder.TableBuilder;
import org.apache.hadoop.hive.metastore.conf.MetastoreConf;
import org.apache.hadoop.hive.metastore.messaging.EventMessage;
import org.apache.hadoop.hive.metastore.metrics.Metrics;
@@ -62,6 +64,11 @@ import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import javax.jdo.Query;
+import java.sql.Connection;
+import java.sql.DriverManager;
+import java.sql.ResultSet;
+import java.sql.SQLException;
+import java.sql.Statement;
import java.util.Arrays;
import java.util.HashMap;
import java.util.HashSet;
@@ -104,6 +111,7 @@ public class TestObjectStore {
@Before
public void setUp() throws Exception {
conf = MetastoreConf.newMetastoreConf();
+ MetastoreConf.setBoolVar(conf, MetastoreConf.ConfVars.HIVE_IN_TEST, true);
MetaStoreTestUtils.setConfForStandloneMode(conf);
objectStore = new ObjectStore();
@@ -340,6 +348,153 @@ public class TestObjectStore {
}
/**
+ * Checks if the JDO cache is able to handle directSQL partition drops in one session.
+ * @throws MetaException
+ * @throws InvalidObjectException
+ * @throws NoSuchObjectException
+ * @throws SQLException
+ */
+ @Test
+ public void testDirectSQLDropPartitionsCacheInSession()
+ throws MetaException, InvalidObjectException, NoSuchObjectException {
+ createPartitionedTable();
+ // query the partitions with JDO
+ Deadline.startTimer("getPartition");
+ List<Partition> partitions = objectStore.getPartitionsInternal(DEFAULT_CATALOG_NAME, DB1, TABLE1,
+ 10, false, true);
+ Assert.assertEquals(3, partitions.size());
+
+ // drop partitions with directSql
+ objectStore.dropPartitionsInternal(DEFAULT_CATALOG_NAME, DB1, TABLE1,
+ Arrays.asList("test_part_col=a0", "test_part_col=a1"), true, false);
+
+ // query the partitions with JDO, checking the cache is not causing any problem
+ partitions = objectStore.getPartitionsInternal(DEFAULT_CATALOG_NAME, DB1, TABLE1,
+ 10, false, true);
+ Assert.assertEquals(1, partitions.size());
+ }
+
+ /**
+ * Checks if the JDO cache is able to handle directSQL partition drops cross sessions.
+ * @throws MetaException
+ * @throws InvalidObjectException
+ * @throws NoSuchObjectException
+ * @throws SQLException
+ */
+ @Test
+ public void testDirectSQLDropPartitionsCacheCrossSession()
+ throws MetaException, InvalidObjectException, NoSuchObjectException {
+ ObjectStore objectStore2 = new ObjectStore();
+ objectStore2.setConf(conf);
+
+ createPartitionedTable();
+ // query the partitions with JDO in the 1st session
+ Deadline.startTimer("getPartition");
+ List<Partition> partitions = objectStore.getPartitionsInternal(DEFAULT_CATALOG_NAME, DB1, TABLE1,
+ 10, false, true);
+ Assert.assertEquals(3, partitions.size());
+
+ // query the partitions with JDO in the 2nd session
+ partitions = objectStore2.getPartitionsInternal(DEFAULT_CATALOG_NAME, DB1, TABLE1, 10,
+ false, true);
+ Assert.assertEquals(3, partitions.size());
+
+ // drop partitions with directSql in the 1st session
+ objectStore.dropPartitionsInternal(DEFAULT_CATALOG_NAME, DB1, TABLE1,
+ Arrays.asList("test_part_col=a0", "test_part_col=a1"), true, false);
+
+ // query the partitions with JDO in the 2nd session, checking the cache is not causing any
+ // problem
+ partitions = objectStore2.getPartitionsInternal(DEFAULT_CATALOG_NAME, DB1, TABLE1,
+ 10, false, true);
+ Assert.assertEquals(1, partitions.size());
+ }
+
+ /**
+ * Checks if the directSQL partition drop removes every connected data from the RDBMS tables.
+ * @throws MetaException
+ * @throws InvalidObjectException
+ * @throws NoSuchObjectException
+ * @throws SQLException
+ */
+ @Test
+ public void testDirectSQLDropParitionsCleanup() throws MetaException, InvalidObjectException,
+ NoSuchObjectException, SQLException {
+
+ createPartitionedTable();
+
+ // drop the partitions
+ Deadline.startTimer("dropPartitions");
+ objectStore.dropPartitionsInternal(DEFAULT_CATALOG_NAME, DB1, TABLE1,
+ Arrays.asList("test_part_col=a0", "test_part_col=a1", "test_part_col=a2"), true, false);
+
+ // Check, if every data is dropped connected to the partitions
+ checkBackendTableSize("PARTITIONS", 0);
+ checkBackendTableSize("PART_PRIVS", 0);
+ checkBackendTableSize("PART_COL_PRIVS", 0);
+ checkBackendTableSize("PART_COL_STATS", 0);
+ checkBackendTableSize("PARTITION_PARAMS", 0);
+ checkBackendTableSize("PARTITION_KEY_VALS", 0);
+ checkBackendTableSize("SD_PARAMS", 0);
+ checkBackendTableSize("BUCKETING_COLS", 0);
+ checkBackendTableSize("SKEWED_COL_NAMES", 0);
+ checkBackendTableSize("SDS", 1); // Table has an SDS
+ checkBackendTableSize("SORT_COLS", 0);
+ checkBackendTableSize("SERDE_PARAMS", 0);
+ checkBackendTableSize("SERDES", 1); // Table has a serde
+ }
+
+ /**
+ * Creates DB1 database, TABLE1 table with 3 partitions.
+ * @throws MetaException
+ * @throws InvalidObjectException
+ */
+ private void createPartitionedTable() throws MetaException, InvalidObjectException {
+ Database db1 = new DatabaseBuilder()
+ .setName(DB1)
+ .setDescription("description")
+ .setLocation("locationurl")
+ .build(conf);
+ objectStore.createDatabase(db1);
+ Table tbl1 =
+ new TableBuilder()
+ .setDbName(DB1)
+ .setTableName(TABLE1)
+ .addCol("test_col1", "int")
+ .addCol("test_col2", "int")
+ .addPartCol("test_part_col", "int")
+ .build(conf);
+ objectStore.createTable(tbl1);
+
+ // Create partitions for the partitioned table
+ for(int i=0; i < 3; i++) {
+ Partition part = new PartitionBuilder()
+ .inTable(tbl1)
+ .addValue("a" + i)
+ .build(conf);
+ objectStore.addPartition(part);
+ }
+ }
+
+ /**
+ * Checks if the HMS backend db row number is as expected. If they are not, an
+ * {@link AssertionError} is thrown.
+ * @param tableName The table in which we count the rows
+ * @param size The expected row number
+ * @throws SQLException If there is a problem connecting to / querying the backend DB
+ */
+ private void checkBackendTableSize(String tableName, int size) throws SQLException {
+ String connectionStr = MetastoreConf.getVar(conf, MetastoreConf.ConfVars.CONNECT_URL_KEY);
+ Connection conn = DriverManager.getConnection(connectionStr);
+ Statement stmt = conn.createStatement();
+
+ ResultSet rs = stmt.executeQuery("SELECT COUNT(1) FROM " + tableName);
+ rs.next();
+ Assert.assertEquals(tableName + " table should contain " + size + " rows", size,
+ rs.getLong(1));
+ }
+
+ /**
* Test master keys operation
*/
@Test