You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by se...@apache.org on 2018/07/13 01:40:56 UTC
[54/91] [abbrv] hive git commit: HIVE-19416 : merge master into
branch (Sergey Shelukhin) 0712
http://git-wip-us.apache.org/repos/asf/hive/blob/93b9cdd6/standalone-metastore/metastore-common/src/test/java/org/apache/hadoop/hive/metastore/InjectableBehaviourObjectStore.java
----------------------------------------------------------------------
diff --cc standalone-metastore/metastore-common/src/test/java/org/apache/hadoop/hive/metastore/InjectableBehaviourObjectStore.java
index 0000000,abbcda3..c5977b2
mode 000000,100644..100644
--- a/standalone-metastore/metastore-common/src/test/java/org/apache/hadoop/hive/metastore/InjectableBehaviourObjectStore.java
+++ b/standalone-metastore/metastore-common/src/test/java/org/apache/hadoop/hive/metastore/InjectableBehaviourObjectStore.java
@@@ -1,0 -1,211 +1,218 @@@
+ /*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+ package org.apache.hadoop.hive.metastore;
+
+ import java.util.List;
++
+ import org.apache.hadoop.hive.metastore.api.Function;
+ import org.apache.hadoop.hive.metastore.api.InvalidObjectException;
+ import org.apache.hadoop.hive.metastore.api.MetaException;
+ import org.apache.hadoop.hive.metastore.api.NotificationEventRequest;
+ import org.apache.hadoop.hive.metastore.api.NotificationEventResponse;
+ import org.apache.hadoop.hive.metastore.api.NoSuchObjectException;
+ import org.apache.hadoop.hive.metastore.api.Partition;
+ import org.apache.hadoop.hive.metastore.api.SQLForeignKey;
+ import org.apache.hadoop.hive.metastore.api.SQLPrimaryKey;
+ import org.apache.hadoop.hive.metastore.api.Table;
+
+ import static org.junit.Assert.assertEquals;
+
+
+ /**
+ * A wrapper around {@link ObjectStore} that allows us to inject custom behaviour
+ * on to some of the methods for testing.
+ */
+ public class InjectableBehaviourObjectStore extends ObjectStore {
+ public InjectableBehaviourObjectStore() {
+ super();
+ }
+
+ /**
+ * A utility class that allows people injecting behaviour to determine if their injections occurred.
+ */
+ public static abstract class BehaviourInjection<T, F>
+ implements com.google.common.base.Function<T, F>{
+ protected boolean injectionPathCalled = false;
+ protected boolean nonInjectedPathCalled = false;
+
+ public void assertInjectionsPerformed(
+ boolean expectedInjectionCalled, boolean expectedNonInjectedPathCalled){
+ assertEquals(expectedInjectionCalled, injectionPathCalled);
+ assertEquals(expectedNonInjectedPathCalled, nonInjectedPathCalled);
+ }
+ }
+
+ /**
+ * A utility class to pass the arguments of the caller to the stub method.
+ */
+ public class CallerArguments {
+ public String dbName;
+ public String tblName;
+ public String funcName;
+ public String constraintTblName;
+
+ public CallerArguments(String dbName) {
+ this.dbName = dbName;
+ }
+ }
+
+ private static com.google.common.base.Function<Table, Table> getTableModifier =
+ com.google.common.base.Functions.identity();
+ private static com.google.common.base.Function<Partition, Partition> getPartitionModifier =
+ com.google.common.base.Functions.identity();
+ private static com.google.common.base.Function<List<String>, List<String>> listPartitionNamesModifier =
+ com.google.common.base.Functions.identity();
+ private static com.google.common.base.Function<NotificationEventResponse, NotificationEventResponse>
+ getNextNotificationModifier = com.google.common.base.Functions.identity();
+
+ private static com.google.common.base.Function<CallerArguments, Boolean> callerVerifier = null;
+
+ // Methods to set/reset getTable modifier
+ public static void setGetTableBehaviour(com.google.common.base.Function<Table, Table> modifier){
+ getTableModifier = (modifier == null) ? com.google.common.base.Functions.identity() : modifier;
+ }
+
+ public static void resetGetTableBehaviour(){
+ setGetTableBehaviour(null);
+ }
+
+ // Methods to set/reset getPartition modifier
+ public static void setGetPartitionBehaviour(com.google.common.base.Function<Partition, Partition> modifier){
+ getPartitionModifier = (modifier == null) ? com.google.common.base.Functions.identity() : modifier;
+ }
+
+ public static void resetGetPartitionBehaviour(){
+ setGetPartitionBehaviour(null);
+ }
+
+ // Methods to set/reset listPartitionNames modifier
+ public static void setListPartitionNamesBehaviour(com.google.common.base.Function<List<String>, List<String>> modifier){
+ listPartitionNamesModifier = (modifier == null)? com.google.common.base.Functions.identity() : modifier;
+ }
+
+ public static void resetListPartitionNamesBehaviour(){
+ setListPartitionNamesBehaviour(null);
+ }
+
+ // Methods to set/reset getNextNotification modifier
+ public static void setGetNextNotificationBehaviour(
+ com.google.common.base.Function<NotificationEventResponse,NotificationEventResponse> modifier){
+ getNextNotificationModifier = (modifier == null)? com.google.common.base.Functions.identity() : modifier;
+ }
+
+ public static void resetGetNextNotificationBehaviour(){
+ setGetNextNotificationBehaviour(null);
+ }
+
+ // Methods to set/reset caller checker
+ public static void setCallerVerifier(com.google.common.base.Function<CallerArguments, Boolean> verifier){
+ callerVerifier = verifier;
+ }
+
+ public static void resetCallerVerifier(){
+ setCallerVerifier(null);
+ }
+
+ // ObjectStore methods to be overridden with injected behavior
+ @Override
+ public Table getTable(String catName, String dbName, String tableName) throws MetaException {
+ return getTableModifier.apply(super.getTable(catName, dbName, tableName));
+ }
+
+ @Override
++ public Table getTable(String catName, String dbName, String tableName,
++ long txnId, String writeIdList) throws MetaException {
++ return getTableModifier.apply(super.getTable(catName, dbName, tableName, txnId, writeIdList));
++ }
++
++ @Override
+ public Partition getPartition(String catName, String dbName, String tableName,
+ List<String> partVals) throws NoSuchObjectException, MetaException {
+ return getPartitionModifier.apply(super.getPartition(catName, dbName, tableName, partVals));
+ }
+
+ @Override
+ public List<String> listPartitionNames(String catName, String dbName, String tableName, short max)
+ throws MetaException {
+ return listPartitionNamesModifier.apply(super.listPartitionNames(catName, dbName, tableName, max));
+ }
+
+ @Override
+ public NotificationEventResponse getNextNotification(NotificationEventRequest rqst) {
+ return getNextNotificationModifier.apply(super.getNextNotification(rqst));
+ }
+
+ @Override
+ public void createTable(Table tbl) throws InvalidObjectException, MetaException {
+ if (callerVerifier != null) {
+ CallerArguments args = new CallerArguments(tbl.getDbName());
+ args.tblName = tbl.getTableName();
+ Boolean success = callerVerifier.apply(args);
+ if ((success != null) && !success) {
+ throw new MetaException("InjectableBehaviourObjectStore: Invalid Create Table operation on DB: "
+ + args.dbName + " table: " + args.tblName);
+ }
+ }
+ super.createTable(tbl);
+ }
+
+ @Override
+ public void createFunction(Function func) throws InvalidObjectException, MetaException {
+ if (callerVerifier != null) {
+ CallerArguments args = new CallerArguments(func.getDbName());
+ args.funcName = func.getFunctionName();
+ Boolean success = callerVerifier.apply(args);
+ if ((success != null) && !success) {
+ throw new MetaException("InjectableBehaviourObjectStore: Invalid Create Function operation on DB: "
+ + args.dbName + " function: " + args.funcName);
+ }
+ }
+ super.createFunction(func);
+ }
+
+ @Override
+ public List<String> addPrimaryKeys(List<SQLPrimaryKey> pks) throws InvalidObjectException,
+ MetaException {
+ if (callerVerifier != null) {
+ CallerArguments args = new CallerArguments(pks.get(0).getTable_db());
+ args.constraintTblName = pks.get(0).getTable_name();
+ Boolean success = callerVerifier.apply(args);
+ if ((success != null) && !success) {
+ throw new MetaException("InjectableBehaviourObjectStore: Invalid Add Primary Key operation on DB: "
+ + args.dbName + " table: " + args.constraintTblName);
+ }
+ }
+ return super.addPrimaryKeys(pks);
+ }
+
+ @Override
+ public List<String> addForeignKeys(List<SQLForeignKey> fks) throws InvalidObjectException,
+ MetaException {
+ if (callerVerifier != null) {
+ CallerArguments args = new CallerArguments(fks.get(0).getFktable_db());
+ args.constraintTblName = fks.get(0).getFktable_name();
+ Boolean success = callerVerifier.apply(args);
+ if ((success != null) && !success) {
+ throw new MetaException("InjectableBehaviourObjectStore: Invalid Add Foreign Key operation on DB: "
+ + args.dbName + " table: " + args.constraintTblName);
+ }
+ }
+ return super.addForeignKeys(fks);
+ }
+ }
http://git-wip-us.apache.org/repos/asf/hive/blob/93b9cdd6/standalone-metastore/metastore-common/src/test/java/org/apache/hadoop/hive/metastore/TestHiveAlterHandler.java
----------------------------------------------------------------------
diff --cc standalone-metastore/metastore-common/src/test/java/org/apache/hadoop/hive/metastore/TestHiveAlterHandler.java
index 0000000,adc82b0..d9dd954
mode 000000,100644..100644
--- a/standalone-metastore/metastore-common/src/test/java/org/apache/hadoop/hive/metastore/TestHiveAlterHandler.java
+++ b/standalone-metastore/metastore-common/src/test/java/org/apache/hadoop/hive/metastore/TestHiveAlterHandler.java
@@@ -1,0 -1,121 +1,121 @@@
+ /*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+ package org.apache.hadoop.hive.metastore;
+
+ import org.apache.hadoop.conf.Configuration;
+ import org.apache.hadoop.hive.metastore.annotation.MetastoreUnitTest;
+ import org.apache.hadoop.hive.metastore.api.*;
+ import org.apache.hadoop.hive.metastore.conf.MetastoreConf;
+ import org.apache.hadoop.hive.metastore.utils.MetaStoreUtils;
+ import org.junit.Test;
+ import org.junit.experimental.categories.Category;
+ import org.mockito.Mockito;
+
+ import java.util.Arrays;
+
+ import static org.apache.hadoop.hive.metastore.utils.MetaStoreUtils.getDefaultCatalog;
+
+ @Category(MetastoreUnitTest.class)
+ public class TestHiveAlterHandler {
+
+ private Configuration conf = MetastoreConf.newMetastoreConf();
+
+ @Test
+ public void testAlterTableAddColNotUpdateStats() throws MetaException, InvalidObjectException, NoSuchObjectException {
+ FieldSchema col1 = new FieldSchema("col1", "string", "col1 comment");
+ FieldSchema col2 = new FieldSchema("col2", "string", "col2 comment");
+ FieldSchema col3 = new FieldSchema("col3", "string", "col3 comment");
+ FieldSchema col4 = new FieldSchema("col4", "string", "col4 comment");
+
+ StorageDescriptor oldSd = new StorageDescriptor();
+ oldSd.setCols(Arrays.asList(col1, col2, col3));
+ Table oldTable = new Table();
+ oldTable.setDbName("default");
+ oldTable.setTableName("test_table");
+ oldTable.setSd(oldSd);
+
+ StorageDescriptor newSd = new StorageDescriptor(oldSd);
+ newSd.setCols(Arrays.asList(col1, col2, col3, col4));
+ Table newTable = new Table(oldTable);
+ newTable.setSd(newSd);
+
+ RawStore msdb = Mockito.mock(RawStore.class);
+ Mockito.doThrow(new RuntimeException("shouldn't be called")).when(msdb).getTableColumnStatistics(
+ getDefaultCatalog(conf), oldTable.getDbName(), oldTable.getTableName(), Arrays.asList("col1", "col2", "col3"));
+ HiveAlterHandler handler = new HiveAlterHandler();
+ handler.setConf(conf);
- handler.alterTableUpdateTableColumnStats(msdb, oldTable, newTable);
++ handler.alterTableUpdateTableColumnStats(msdb, oldTable, newTable, null);
+ }
+
+ @Test
+ public void testAlterTableDelColUpdateStats() throws MetaException, InvalidObjectException, NoSuchObjectException {
+ FieldSchema col1 = new FieldSchema("col1", "string", "col1 comment");
+ FieldSchema col2 = new FieldSchema("col2", "string", "col2 comment");
+ FieldSchema col3 = new FieldSchema("col3", "string", "col3 comment");
+ FieldSchema col4 = new FieldSchema("col4", "string", "col4 comment");
+
+ StorageDescriptor oldSd = new StorageDescriptor();
+ oldSd.setCols(Arrays.asList(col1, col2, col3, col4));
+ Table oldTable = new Table();
+ oldTable.setDbName("default");
+ oldTable.setTableName("test_table");
+ oldTable.setSd(oldSd);
+
+ StorageDescriptor newSd = new StorageDescriptor(oldSd);
+ newSd.setCols(Arrays.asList(col1, col2, col3));
+ Table newTable = new Table(oldTable);
+ newTable.setSd(newSd);
+
+ RawStore msdb = Mockito.mock(RawStore.class);
+ HiveAlterHandler handler = new HiveAlterHandler();
+ handler.setConf(conf);
- handler.alterTableUpdateTableColumnStats(msdb, oldTable, newTable);
++ handler.alterTableUpdateTableColumnStats(msdb, oldTable, newTable, null);
+ Mockito.verify(msdb, Mockito.times(1)).getTableColumnStatistics(
+ getDefaultCatalog(conf), oldTable.getDbName(), oldTable.getTableName(), Arrays.asList("col1", "col2", "col3", "col4")
+ );
+ }
+
+ @Test
+ public void testAlterTableChangePosNotUpdateStats() throws MetaException, InvalidObjectException, NoSuchObjectException {
+ FieldSchema col1 = new FieldSchema("col1", "string", "col1 comment");
+ FieldSchema col2 = new FieldSchema("col2", "string", "col2 comment");
+ FieldSchema col3 = new FieldSchema("col3", "string", "col3 comment");
+ FieldSchema col4 = new FieldSchema("col4", "string", "col4 comment");
+
+ StorageDescriptor oldSd = new StorageDescriptor();
+ oldSd.setCols(Arrays.asList(col1, col2, col3, col4));
+ Table oldTable = new Table();
+ oldTable.setDbName("default");
+ oldTable.setTableName("test_table");
+ oldTable.setSd(oldSd);
+
+ StorageDescriptor newSd = new StorageDescriptor(oldSd);
+ newSd.setCols(Arrays.asList(col1, col4, col2, col3));
+ Table newTable = new Table(oldTable);
+ newTable.setSd(newSd);
+
+ RawStore msdb = Mockito.mock(RawStore.class);
+ Mockito.doThrow(new RuntimeException("shouldn't be called")).when(msdb).getTableColumnStatistics(
+ getDefaultCatalog(conf), oldTable.getDbName(), oldTable.getTableName(), Arrays.asList("col1", "col2", "col3", "col4"));
+ HiveAlterHandler handler = new HiveAlterHandler();
+ handler.setConf(conf);
- handler.alterTableUpdateTableColumnStats(msdb, oldTable, newTable);
++ handler.alterTableUpdateTableColumnStats(msdb, oldTable, newTable, null);
+ }
+
+ }
http://git-wip-us.apache.org/repos/asf/hive/blob/93b9cdd6/standalone-metastore/metastore-common/src/test/java/org/apache/hadoop/hive/metastore/TestObjectStore.java
----------------------------------------------------------------------
diff --cc standalone-metastore/metastore-common/src/test/java/org/apache/hadoop/hive/metastore/TestObjectStore.java
index 0000000,833e2bd..c40d45d
mode 000000,100644..100644
--- a/standalone-metastore/metastore-common/src/test/java/org/apache/hadoop/hive/metastore/TestObjectStore.java
+++ b/standalone-metastore/metastore-common/src/test/java/org/apache/hadoop/hive/metastore/TestObjectStore.java
@@@ -1,0 -1,904 +1,904 @@@
+ /*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+ package org.apache.hadoop.hive.metastore;
+
+ import com.codahale.metrics.Counter;
+ import com.google.common.base.Supplier;
+ import com.google.common.collect.ImmutableList;
+ import org.apache.hadoop.hive.metastore.ObjectStore.RetryingExecutor;
+ import org.apache.hadoop.conf.Configuration;
+ import org.apache.hadoop.hive.metastore.annotation.MetastoreUnitTest;
+ import org.apache.hadoop.hive.metastore.api.BooleanColumnStatsData;
+ import org.apache.hadoop.hive.metastore.api.Catalog;
+ import org.apache.hadoop.hive.metastore.api.ColumnStatistics;
+ import org.apache.hadoop.hive.metastore.api.ColumnStatisticsData;
+ import org.apache.hadoop.hive.metastore.api.ColumnStatisticsDesc;
+ import org.apache.hadoop.hive.metastore.api.ColumnStatisticsObj;
+ import org.apache.hadoop.hive.metastore.api.CurrentNotificationEventId;
+ import org.apache.hadoop.hive.metastore.api.Database;
+ import org.apache.hadoop.hive.metastore.api.FieldSchema;
+ import org.apache.hadoop.hive.metastore.api.Function;
+ import org.apache.hadoop.hive.metastore.api.HiveObjectPrivilege;
+ import org.apache.hadoop.hive.metastore.api.HiveObjectRef;
+ import org.apache.hadoop.hive.metastore.api.InvalidInputException;
+ import org.apache.hadoop.hive.metastore.api.InvalidObjectException;
+ import org.apache.hadoop.hive.metastore.api.MetaException;
+ import org.apache.hadoop.hive.metastore.api.NoSuchObjectException;
+ import org.apache.hadoop.hive.metastore.api.NotificationEvent;
+ import org.apache.hadoop.hive.metastore.api.NotificationEventRequest;
+ import org.apache.hadoop.hive.metastore.api.NotificationEventResponse;
+ import org.apache.hadoop.hive.metastore.api.Partition;
+ import org.apache.hadoop.hive.metastore.api.PrincipalType;
+ import org.apache.hadoop.hive.metastore.api.PrivilegeBag;
+ import org.apache.hadoop.hive.metastore.api.PrivilegeGrantInfo;
+ import org.apache.hadoop.hive.metastore.api.Role;
+ import org.apache.hadoop.hive.metastore.api.SQLForeignKey;
+ import org.apache.hadoop.hive.metastore.api.SQLPrimaryKey;
+ import org.apache.hadoop.hive.metastore.api.SerDeInfo;
+ import org.apache.hadoop.hive.metastore.api.StorageDescriptor;
+ import org.apache.hadoop.hive.metastore.api.Table;
+ import org.apache.hadoop.hive.metastore.client.builder.CatalogBuilder;
+ import org.apache.hadoop.hive.metastore.client.builder.DatabaseBuilder;
+ import org.apache.hadoop.hive.metastore.client.builder.PartitionBuilder;
+ import org.apache.hadoop.hive.metastore.client.builder.HiveObjectPrivilegeBuilder;
+ import org.apache.hadoop.hive.metastore.client.builder.HiveObjectRefBuilder;
+ import org.apache.hadoop.hive.metastore.client.builder.PrivilegeGrantInfoBuilder;
+ import org.apache.hadoop.hive.metastore.client.builder.TableBuilder;
+ import org.apache.hadoop.hive.metastore.conf.MetastoreConf;
+ import org.apache.hadoop.hive.metastore.messaging.EventMessage;
+ import org.apache.hadoop.hive.metastore.metrics.Metrics;
+ import org.apache.hadoop.hive.metastore.metrics.MetricsConstants;
+ import org.apache.hadoop.hive.metastore.model.MNotificationLog;
+ import org.apache.hadoop.hive.metastore.model.MNotificationNextId;
+ import org.junit.Assert;
+ import org.junit.Assume;
+ import org.junit.Before;
+ import org.junit.Ignore;
+ import org.junit.Test;
+ import org.junit.experimental.categories.Category;
+ import org.mockito.Mockito;
+ import org.slf4j.Logger;
+ import org.slf4j.LoggerFactory;
+
+ import javax.jdo.Query;
+ import java.sql.Connection;
+ import java.sql.DriverManager;
+ import java.sql.ResultSet;
+ import java.sql.SQLException;
+ import java.sql.Statement;
+ import java.util.ArrayList;
+ import java.util.Arrays;
+ import java.util.HashMap;
+ import java.util.HashSet;
+ import java.util.List;
+ import java.util.Set;
+ import java.util.concurrent.BrokenBarrierException;
+ import java.util.concurrent.CyclicBarrier;
+ import java.util.concurrent.ExecutionException;
+ import java.util.concurrent.ExecutorService;
+ import java.util.concurrent.Executors;
+ import java.util.concurrent.TimeUnit;
+
+ import static org.apache.hadoop.hive.metastore.Warehouse.DEFAULT_CATALOG_NAME;
+
+ @Category(MetastoreUnitTest.class)
+ public class TestObjectStore {
+ private ObjectStore objectStore = null;
+ private Configuration conf;
+
+ private static final String DB1 = "testobjectstoredb1";
+ private static final String DB2 = "testobjectstoredb2";
+ private static final String TABLE1 = "testobjectstoretable1";
+ private static final String KEY1 = "testobjectstorekey1";
+ private static final String KEY2 = "testobjectstorekey2";
+ private static final String OWNER = "testobjectstoreowner";
+ private static final String USER1 = "testobjectstoreuser1";
+ private static final String ROLE1 = "testobjectstorerole1";
+ private static final String ROLE2 = "testobjectstorerole2";
+ private static final Logger LOG = LoggerFactory.getLogger(TestObjectStore.class.getName());
+
+ private static final class LongSupplier implements Supplier<Long> {
+ public long value = 0;
+
+ @Override
+ public Long get() {
+ return value;
+ }
+ }
+
+ @Before
+ public void setUp() throws Exception {
+ conf = MetastoreConf.newMetastoreConf();
+ MetastoreConf.setBoolVar(conf, MetastoreConf.ConfVars.HIVE_IN_TEST, true);
+ MetaStoreTestUtils.setConfForStandloneMode(conf);
+
+ objectStore = new ObjectStore();
+ objectStore.setConf(conf);
+ dropAllStoreObjects(objectStore);
+ HiveMetaStore.HMSHandler.createDefaultCatalog(objectStore, new Warehouse(conf));
+ }
+
+ @Test
+ public void catalogs() throws MetaException, NoSuchObjectException {
+ final String names[] = {"cat1", "cat2"};
+ final String locations[] = {"loc1", "loc2"};
+ final String descriptions[] = {"description 1", "description 2"};
+
+ for (int i = 0; i < names.length; i++) {
+ Catalog cat = new CatalogBuilder()
+ .setName(names[i])
+ .setLocation(locations[i])
+ .setDescription(descriptions[i])
+ .build();
+ objectStore.createCatalog(cat);
+ }
+
+ List<String> fetchedNames = objectStore.getCatalogs();
+ Assert.assertEquals(3, fetchedNames.size());
+ for (int i = 0; i < names.length - 1; i++) {
+ Assert.assertEquals(names[i], fetchedNames.get(i));
+ Catalog cat = objectStore.getCatalog(fetchedNames.get(i));
+ Assert.assertEquals(names[i], cat.getName());
+ Assert.assertEquals(descriptions[i], cat.getDescription());
+ Assert.assertEquals(locations[i], cat.getLocationUri());
+ }
+ Catalog cat = objectStore.getCatalog(fetchedNames.get(2));
+ Assert.assertEquals(DEFAULT_CATALOG_NAME, cat.getName());
+ Assert.assertEquals(Warehouse.DEFAULT_CATALOG_COMMENT, cat.getDescription());
+ // Location will vary by system.
+
+ for (int i = 0; i < names.length; i++) objectStore.dropCatalog(names[i]);
+ fetchedNames = objectStore.getCatalogs();
+ Assert.assertEquals(1, fetchedNames.size());
+ }
+
+ @Test(expected = NoSuchObjectException.class)
+ public void getNoSuchCatalog() throws MetaException, NoSuchObjectException {
+ objectStore.getCatalog("no_such_catalog");
+ }
+
+ @Test(expected = NoSuchObjectException.class)
+ public void dropNoSuchCatalog() throws MetaException, NoSuchObjectException {
+ objectStore.dropCatalog("no_such_catalog");
+ }
+
+ // TODO test dropping non-empty catalog
+
+ /**
+ * Test database operations
+ */
+ @Test
+ public void testDatabaseOps() throws MetaException, InvalidObjectException,
+ NoSuchObjectException {
+ String catName = "tdo1_cat";
+ createTestCatalog(catName);
+ Database db1 = new Database(DB1, "description", "locationurl", null);
+ Database db2 = new Database(DB2, "description", "locationurl", null);
+ db1.setCatalogName(catName);
+ db2.setCatalogName(catName);
+ objectStore.createDatabase(db1);
+ objectStore.createDatabase(db2);
+
+ List<String> databases = objectStore.getAllDatabases(catName);
+ LOG.info("databases: " + databases);
+ Assert.assertEquals(2, databases.size());
+ Assert.assertEquals(DB1, databases.get(0));
+ Assert.assertEquals(DB2, databases.get(1));
+
+ objectStore.dropDatabase(catName, DB1);
+ databases = objectStore.getAllDatabases(catName);
+ Assert.assertEquals(1, databases.size());
+ Assert.assertEquals(DB2, databases.get(0));
+
+ objectStore.dropDatabase(catName, DB2);
+ }
+
+ /**
+ * Test table operations
+ */
+ @Test
+ public void testTableOps() throws MetaException, InvalidObjectException, NoSuchObjectException,
+ InvalidInputException {
+ Database db1 = new DatabaseBuilder()
+ .setName(DB1)
+ .setDescription("description")
+ .setLocation("locationurl")
+ .build(conf);
+ objectStore.createDatabase(db1);
+ StorageDescriptor sd1 =
+ new StorageDescriptor(ImmutableList.of(new FieldSchema("pk_col", "double", null)),
+ "location", null, null, false, 0, new SerDeInfo("SerDeName", "serializationLib", null),
+ null, null, null);
+ HashMap<String, String> params = new HashMap<>();
+ params.put("EXTERNAL", "false");
+ Table tbl1 =
+ new Table(TABLE1, DB1, "owner", 1, 2, 3, sd1, null, params, null, null, "MANAGED_TABLE");
+ objectStore.createTable(tbl1);
+
+ List<String> tables = objectStore.getAllTables(DEFAULT_CATALOG_NAME, DB1);
+ Assert.assertEquals(1, tables.size());
+ Assert.assertEquals(TABLE1, tables.get(0));
+
+ StorageDescriptor sd2 =
+ new StorageDescriptor(ImmutableList.of(new FieldSchema("fk_col", "double", null)),
+ "location", null, null, false, 0, new SerDeInfo("SerDeName", "serializationLib", null),
+ null, null, null);
+ Table newTbl1 = new Table("new" + TABLE1, DB1, "owner", 1, 2, 3, sd2, null, params, null, null,
+ "MANAGED_TABLE");
+
+ // Change different fields and verify they were altered
+ newTbl1.setOwner("role1");
+ newTbl1.setOwnerType(PrincipalType.ROLE);
+
- objectStore.alterTable(DEFAULT_CATALOG_NAME, DB1, TABLE1, newTbl1);
++ objectStore.alterTable(DEFAULT_CATALOG_NAME, DB1, TABLE1, newTbl1, -1, null);
+ tables = objectStore.getTables(DEFAULT_CATALOG_NAME, DB1, "new*");
+ Assert.assertEquals(1, tables.size());
+ Assert.assertEquals("new" + TABLE1, tables.get(0));
+
+ // Verify fields were altered during the alterTable operation
+ Table alteredTable = objectStore.getTable(DEFAULT_CATALOG_NAME, DB1, "new" + TABLE1);
+ Assert.assertEquals("Owner of table was not altered", newTbl1.getOwner(), alteredTable.getOwner());
+ Assert.assertEquals("Owner type of table was not altered", newTbl1.getOwnerType(), alteredTable.getOwnerType());
+
+ objectStore.createTable(tbl1);
+ tables = objectStore.getAllTables(DEFAULT_CATALOG_NAME, DB1);
+ Assert.assertEquals(2, tables.size());
+
+ List<SQLForeignKey> foreignKeys = objectStore.getForeignKeys(DEFAULT_CATALOG_NAME, DB1, TABLE1, null, null);
+ Assert.assertEquals(0, foreignKeys.size());
+
+ SQLPrimaryKey pk = new SQLPrimaryKey(DB1, TABLE1, "pk_col", 1,
+ "pk_const_1", false, false, false);
+ pk.setCatName(DEFAULT_CATALOG_NAME);
+ objectStore.addPrimaryKeys(ImmutableList.of(pk));
+ SQLForeignKey fk = new SQLForeignKey(DB1, TABLE1, "pk_col",
+ DB1, "new" + TABLE1, "fk_col", 1,
+ 0, 0, "fk_const_1", "pk_const_1", false, false, false);
+ objectStore.addForeignKeys(ImmutableList.of(fk));
+
+ // Retrieve from PK side
+ foreignKeys = objectStore.getForeignKeys(DEFAULT_CATALOG_NAME, null, null, DB1, "new" + TABLE1);
+ Assert.assertEquals(1, foreignKeys.size());
+
+ List<SQLForeignKey> fks = objectStore.getForeignKeys(DEFAULT_CATALOG_NAME, null, null, DB1, "new" + TABLE1);
+ if (fks != null) {
+ for (SQLForeignKey fkcol : fks) {
+ objectStore.dropConstraint(fkcol.getCatName(), fkcol.getFktable_db(), fkcol.getFktable_name(),
+ fkcol.getFk_name());
+ }
+ }
+ // Retrieve from FK side
+ foreignKeys = objectStore.getForeignKeys(DEFAULT_CATALOG_NAME, DB1, TABLE1, null, null);
+ Assert.assertEquals(0, foreignKeys.size());
+ // Retrieve from PK side
+ foreignKeys = objectStore.getForeignKeys(DEFAULT_CATALOG_NAME, null, null, DB1, "new" + TABLE1);
+ Assert.assertEquals(0, foreignKeys.size());
+
+ objectStore.dropTable(DEFAULT_CATALOG_NAME, DB1, TABLE1);
+ tables = objectStore.getAllTables(DEFAULT_CATALOG_NAME, DB1);
+ Assert.assertEquals(1, tables.size());
+
+ objectStore.dropTable(DEFAULT_CATALOG_NAME, DB1, "new" + TABLE1);
+ tables = objectStore.getAllTables(DEFAULT_CATALOG_NAME, DB1);
+ Assert.assertEquals(0, tables.size());
+
+ objectStore.dropDatabase(db1.getCatalogName(), DB1);
+ }
+
+ private StorageDescriptor createFakeSd(String location) {
+ return new StorageDescriptor(null, location, null, null, false, 0,
+ new SerDeInfo("SerDeName", "serializationLib", null), null, null, null);
+ }
+
+
+ /**
+ * Tests partition operations
+ */
+ @Test
+ public void testPartitionOps() throws MetaException, InvalidObjectException,
+ NoSuchObjectException, InvalidInputException {
+ Database db1 = new DatabaseBuilder()
+ .setName(DB1)
+ .setDescription("description")
+ .setLocation("locationurl")
+ .build(conf);
+ objectStore.createDatabase(db1);
+ StorageDescriptor sd = createFakeSd("location");
+ HashMap<String, String> tableParams = new HashMap<>();
+ tableParams.put("EXTERNAL", "false");
+ FieldSchema partitionKey1 = new FieldSchema("Country", ColumnType.STRING_TYPE_NAME, "");
+ FieldSchema partitionKey2 = new FieldSchema("State", ColumnType.STRING_TYPE_NAME, "");
+ Table tbl1 =
+ new Table(TABLE1, DB1, "owner", 1, 2, 3, sd, Arrays.asList(partitionKey1, partitionKey2),
+ tableParams, null, null, "MANAGED_TABLE");
+ objectStore.createTable(tbl1);
+ HashMap<String, String> partitionParams = new HashMap<>();
+ partitionParams.put("PARTITION_LEVEL_PRIVILEGE", "true");
+ List<String> value1 = Arrays.asList("US", "CA");
+ Partition part1 = new Partition(value1, DB1, TABLE1, 111, 111, sd, partitionParams);
+ part1.setCatName(DEFAULT_CATALOG_NAME);
+ objectStore.addPartition(part1);
+ List<String> value2 = Arrays.asList("US", "MA");
+ Partition part2 = new Partition(value2, DB1, TABLE1, 222, 222, sd, partitionParams);
+ part2.setCatName(DEFAULT_CATALOG_NAME);
+ objectStore.addPartition(part2);
+
+ Deadline.startTimer("getPartition");
+ List<Partition> partitions = objectStore.getPartitions(DEFAULT_CATALOG_NAME, DB1, TABLE1, 10);
+ Assert.assertEquals(2, partitions.size());
+ Assert.assertEquals(111, partitions.get(0).getCreateTime());
+ Assert.assertEquals(222, partitions.get(1).getCreateTime());
+
+ int numPartitions = objectStore.getNumPartitionsByFilter(DEFAULT_CATALOG_NAME, DB1, TABLE1, "");
+ Assert.assertEquals(partitions.size(), numPartitions);
+
+ numPartitions = objectStore.getNumPartitionsByFilter(DEFAULT_CATALOG_NAME, DB1, TABLE1, "country = \"US\"");
+ Assert.assertEquals(2, numPartitions);
+
+ objectStore.dropPartition(DEFAULT_CATALOG_NAME, DB1, TABLE1, value1);
+ partitions = objectStore.getPartitions(DEFAULT_CATALOG_NAME, DB1, TABLE1, 10);
+ Assert.assertEquals(1, partitions.size());
+ Assert.assertEquals(222, partitions.get(0).getCreateTime());
+
+ objectStore.dropPartition(DEFAULT_CATALOG_NAME, DB1, TABLE1, value2);
+ objectStore.dropTable(DEFAULT_CATALOG_NAME, DB1, TABLE1);
+ objectStore.dropDatabase(db1.getCatalogName(), DB1);
+ }
+
+ /**
+ * Checks if the JDO cache is able to handle directSQL partition drops in one session.
+ * @throws MetaException
+ * @throws InvalidObjectException
+ * @throws NoSuchObjectException
+ * @throws SQLException
+ */
+ @Test
+ public void testDirectSQLDropPartitionsCacheInSession()
+ throws MetaException, InvalidObjectException, NoSuchObjectException, InvalidInputException {
+ createPartitionedTable(false, false);
+ // query the partitions with JDO
+ Deadline.startTimer("getPartition");
+ List<Partition> partitions = objectStore.getPartitionsInternal(DEFAULT_CATALOG_NAME, DB1, TABLE1,
+ 10, false, true);
+ Assert.assertEquals(3, partitions.size());
+
+ // drop partitions with directSql
+ objectStore.dropPartitionsInternal(DEFAULT_CATALOG_NAME, DB1, TABLE1,
+ Arrays.asList("test_part_col=a0", "test_part_col=a1"), true, false);
+
+ // query the partitions with JDO, checking the cache is not causing any problem
+ partitions = objectStore.getPartitionsInternal(DEFAULT_CATALOG_NAME, DB1, TABLE1,
+ 10, false, true);
+ Assert.assertEquals(1, partitions.size());
+ }
+
+ /**
+ * Checks if the JDO cache is able to handle directSQL partition drops cross sessions.
+ * @throws MetaException
+ * @throws InvalidObjectException
+ * @throws NoSuchObjectException
+ * @throws SQLException
+ */
+ @Test
+ public void testDirectSQLDropPartitionsCacheCrossSession()
+ throws MetaException, InvalidObjectException, NoSuchObjectException, InvalidInputException {
+ ObjectStore objectStore2 = new ObjectStore();
+ objectStore2.setConf(conf);
+
+ createPartitionedTable(false, false);
+ // query the partitions with JDO in the 1st session
+ Deadline.startTimer("getPartition");
+ List<Partition> partitions = objectStore.getPartitionsInternal(DEFAULT_CATALOG_NAME, DB1, TABLE1,
+ 10, false, true);
+ Assert.assertEquals(3, partitions.size());
+
+ // query the partitions with JDO in the 2nd session
+ partitions = objectStore2.getPartitionsInternal(DEFAULT_CATALOG_NAME, DB1, TABLE1, 10,
+ false, true);
+ Assert.assertEquals(3, partitions.size());
+
+ // drop partitions with directSql in the 1st session
+ objectStore.dropPartitionsInternal(DEFAULT_CATALOG_NAME, DB1, TABLE1,
+ Arrays.asList("test_part_col=a0", "test_part_col=a1"), true, false);
+
+ // query the partitions with JDO in the 2nd session, checking the cache is not causing any
+ // problem
+ partitions = objectStore2.getPartitionsInternal(DEFAULT_CATALOG_NAME, DB1, TABLE1,
+ 10, false, true);
+ Assert.assertEquals(1, partitions.size());
+ }
+
+ /**
+ * Checks if the directSQL partition drop removes every connected data from the RDBMS tables.
+ * @throws MetaException
+ * @throws InvalidObjectException
+ * @throws NoSuchObjectException
+ * @throws SQLException
+ */
+ @Test
+ public void testDirectSQLDropParitionsCleanup() throws MetaException, InvalidObjectException,
+ NoSuchObjectException, SQLException, InvalidInputException {
+
+ createPartitionedTable(true, true);
+
+ // Check, that every table in the expected state before the drop
+ checkBackendTableSize("PARTITIONS", 3);
+ checkBackendTableSize("PART_PRIVS", 3);
+ checkBackendTableSize("PART_COL_PRIVS", 3);
+ checkBackendTableSize("PART_COL_STATS", 3);
+ checkBackendTableSize("PARTITION_PARAMS", 3);
+ checkBackendTableSize("PARTITION_KEY_VALS", 3);
+ checkBackendTableSize("SD_PARAMS", 3);
+ checkBackendTableSize("BUCKETING_COLS", 3);
+ checkBackendTableSize("SKEWED_COL_NAMES", 3);
+ checkBackendTableSize("SDS", 4); // Table has an SDS
+ checkBackendTableSize("SORT_COLS", 3);
+ checkBackendTableSize("SERDE_PARAMS", 3);
+ checkBackendTableSize("SERDES", 4); // Table has a serde
+
+ // drop the partitions
+ Deadline.startTimer("dropPartitions");
+ objectStore.dropPartitionsInternal(DEFAULT_CATALOG_NAME, DB1, TABLE1,
+ Arrays.asList("test_part_col=a0", "test_part_col=a1", "test_part_col=a2"), true, false);
+
+ // Check, if every data is dropped connected to the partitions
+ checkBackendTableSize("PARTITIONS", 0);
+ checkBackendTableSize("PART_PRIVS", 0);
+ checkBackendTableSize("PART_COL_PRIVS", 0);
+ checkBackendTableSize("PART_COL_STATS", 0);
+ checkBackendTableSize("PARTITION_PARAMS", 0);
+ checkBackendTableSize("PARTITION_KEY_VALS", 0);
+ checkBackendTableSize("SD_PARAMS", 0);
+ checkBackendTableSize("BUCKETING_COLS", 0);
+ checkBackendTableSize("SKEWED_COL_NAMES", 0);
+ checkBackendTableSize("SDS", 1); // Table has an SDS
+ checkBackendTableSize("SORT_COLS", 0);
+ checkBackendTableSize("SERDE_PARAMS", 0);
+ checkBackendTableSize("SERDES", 1); // Table has a serde
+ }
+
+ /**
+ * Creates DB1 database, TABLE1 table with 3 partitions.
+ * @param withPrivileges Should we create privileges as well
+ * @param withStatistics Should we create statitics as well
+ * @throws MetaException
+ * @throws InvalidObjectException
+ */
+ private void createPartitionedTable(boolean withPrivileges, boolean withStatistics)
+ throws MetaException, InvalidObjectException, NoSuchObjectException, InvalidInputException {
+ Database db1 = new DatabaseBuilder()
+ .setName(DB1)
+ .setDescription("description")
+ .setLocation("locationurl")
+ .build(conf);
+ objectStore.createDatabase(db1);
+ Table tbl1 =
+ new TableBuilder()
+ .setDbName(DB1)
+ .setTableName(TABLE1)
+ .addCol("test_col1", "int")
+ .addCol("test_col2", "int")
+ .addPartCol("test_part_col", "int")
+ .addCol("test_bucket_col", "int", "test bucket col comment")
+ .addCol("test_skewed_col", "int", "test skewed col comment")
+ .addCol("test_sort_col", "int", "test sort col comment")
+ .build(conf);
+ objectStore.createTable(tbl1);
+
+ PrivilegeBag privilegeBag = new PrivilegeBag();
+ // Create partitions for the partitioned table
+ for(int i=0; i < 3; i++) {
+ Partition part = new PartitionBuilder()
+ .inTable(tbl1)
+ .addValue("a" + i)
+ .addSerdeParam("serdeParam", "serdeParamValue")
+ .addStorageDescriptorParam("sdParam", "sdParamValue")
+ .addBucketCol("test_bucket_col")
+ .addSkewedColName("test_skewed_col")
+ .addSortCol("test_sort_col", 1)
+ .build(conf);
+ objectStore.addPartition(part);
+
+ if (withPrivileges) {
+ HiveObjectRef partitionReference = new HiveObjectRefBuilder().buildPartitionReference(part);
+ HiveObjectRef partitionColumnReference = new HiveObjectRefBuilder()
+ .buildPartitionColumnReference(tbl1, "test_part_col", part.getValues());
+ PrivilegeGrantInfo privilegeGrantInfo = new PrivilegeGrantInfoBuilder()
+ .setPrivilege("a")
+ .build();
+ HiveObjectPrivilege partitionPriv = new HiveObjectPrivilegeBuilder()
+ .setHiveObjectRef(partitionReference)
+ .setPrincipleName("a")
+ .setPrincipalType(PrincipalType.USER)
+ .setGrantInfo(privilegeGrantInfo)
+ .build();
+ privilegeBag.addToPrivileges(partitionPriv);
+ HiveObjectPrivilege partitionColPriv = new HiveObjectPrivilegeBuilder()
+ .setHiveObjectRef(partitionColumnReference)
+ .setPrincipleName("a")
+ .setPrincipalType(PrincipalType.USER)
+ .setGrantInfo(privilegeGrantInfo)
+ .build();
+ privilegeBag.addToPrivileges(partitionColPriv);
+ }
+
+ if (withStatistics) {
+ ColumnStatistics stats = new ColumnStatistics();
+ ColumnStatisticsDesc desc = new ColumnStatisticsDesc();
+ desc.setCatName(tbl1.getCatName());
+ desc.setDbName(tbl1.getDbName());
+ desc.setTableName(tbl1.getTableName());
+ desc.setPartName("test_part_col=a" + i);
+ stats.setStatsDesc(desc);
+
+ List<ColumnStatisticsObj> statsObjList = new ArrayList<>(1);
+ stats.setStatsObj(statsObjList);
+
+ ColumnStatisticsData data = new ColumnStatisticsData();
+ BooleanColumnStatsData boolStats = new BooleanColumnStatsData();
+ boolStats.setNumTrues(0);
+ boolStats.setNumFalses(0);
+ boolStats.setNumNulls(0);
+ data.setBooleanStats(boolStats);
+
+ ColumnStatisticsObj partStats = new ColumnStatisticsObj("test_part_col", "int", data);
+ statsObjList.add(partStats);
+
+ objectStore.updatePartitionColumnStatistics(stats, part.getValues());
+ }
+ }
+ if (withPrivileges) {
+ objectStore.grantPrivileges(privilegeBag);
+ }
+ }
+
+ /**
+ * Checks if the HMS backend db row number is as expected. If they are not, an
+ * {@link AssertionError} is thrown.
+ * @param tableName The table in which we count the rows
+ * @param size The expected row number
+ * @throws SQLException If there is a problem connecting to / querying the backend DB
+ */
+ private void checkBackendTableSize(String tableName, int size) throws SQLException {
+ String connectionStr = MetastoreConf.getVar(conf, MetastoreConf.ConfVars.CONNECT_URL_KEY);
+ Connection conn = DriverManager.getConnection(connectionStr);
+ Statement stmt = conn.createStatement();
+
+ ResultSet rs = stmt.executeQuery("SELECT COUNT(1) FROM " + tableName);
+ rs.next();
+ Assert.assertEquals(tableName + " table should contain " + size + " rows", size,
+ rs.getLong(1));
+ }
+
+ /**
+ * Test master keys operation
+ */
+ @Test
+ public void testMasterKeyOps() throws MetaException, NoSuchObjectException {
+ int id1 = objectStore.addMasterKey(KEY1);
+ int id2 = objectStore.addMasterKey(KEY2);
+
+ String[] keys = objectStore.getMasterKeys();
+ Assert.assertEquals(2, keys.length);
+ Assert.assertEquals(KEY1, keys[0]);
+ Assert.assertEquals(KEY2, keys[1]);
+
+ objectStore.updateMasterKey(id1, "new" + KEY1);
+ objectStore.updateMasterKey(id2, "new" + KEY2);
+ keys = objectStore.getMasterKeys();
+ Assert.assertEquals(2, keys.length);
+ Assert.assertEquals("new" + KEY1, keys[0]);
+ Assert.assertEquals("new" + KEY2, keys[1]);
+
+ objectStore.removeMasterKey(id1);
+ keys = objectStore.getMasterKeys();
+ Assert.assertEquals(1, keys.length);
+ Assert.assertEquals("new" + KEY2, keys[0]);
+
+ objectStore.removeMasterKey(id2);
+ }
+
+ /**
+ * Test role operation
+ */
+ @Test
+ public void testRoleOps() throws InvalidObjectException, MetaException, NoSuchObjectException {
+ objectStore.addRole(ROLE1, OWNER);
+ objectStore.addRole(ROLE2, OWNER);
+ List<String> roles = objectStore.listRoleNames();
+ Assert.assertEquals(2, roles.size());
+ Assert.assertEquals(ROLE2, roles.get(1));
+ Role role1 = objectStore.getRole(ROLE1);
+ Assert.assertEquals(OWNER, role1.getOwnerName());
+ objectStore.grantRole(role1, USER1, PrincipalType.USER, OWNER, PrincipalType.ROLE, true);
+ objectStore.revokeRole(role1, USER1, PrincipalType.USER, false);
+ objectStore.removeRole(ROLE1);
+ }
+
+ @Test
+ public void testDirectSqlErrorMetrics() throws Exception {
+ Configuration conf = MetastoreConf.newMetastoreConf();
+ MetastoreConf.setBoolVar(conf, MetastoreConf.ConfVars.METRICS_ENABLED, true);
+ Metrics.initialize(conf);
+ MetastoreConf.setVar(conf, MetastoreConf.ConfVars.HIVE_CODAHALE_METRICS_REPORTER_CLASSES,
+ "org.apache.hadoop.hive.common.metrics.metrics2.JsonFileMetricsReporter, " +
+ "org.apache.hadoop.hive.common.metrics.metrics2.JmxMetricsReporter"
+ );
+
+ // recall setup so that we get an object store with the metrics initalized
+ setUp();
+ Counter directSqlErrors =
+ Metrics.getRegistry().getCounters().get(MetricsConstants.DIRECTSQL_ERRORS);
+
+ objectStore.new GetDbHelper(DEFAULT_CATALOG_NAME, "foo", true, true) {
+ @Override
+ protected Database getSqlResult(ObjectStore.GetHelper<Database> ctx) throws MetaException {
+ return null;
+ }
+
+ @Override
+ protected Database getJdoResult(ObjectStore.GetHelper<Database> ctx) throws MetaException,
+ NoSuchObjectException {
+ return null;
+ }
+ }.run(false);
+
+ Assert.assertEquals(0, directSqlErrors.getCount());
+
+ objectStore.new GetDbHelper(DEFAULT_CATALOG_NAME, "foo", true, true) {
+ @Override
+ protected Database getSqlResult(ObjectStore.GetHelper<Database> ctx) throws MetaException {
+ throw new RuntimeException();
+ }
+
+ @Override
+ protected Database getJdoResult(ObjectStore.GetHelper<Database> ctx) throws MetaException,
+ NoSuchObjectException {
+ return null;
+ }
+ }.run(false);
+
+ Assert.assertEquals(1, directSqlErrors.getCount());
+ }
+
+ private static void dropAllStoreObjects(RawStore store)
+ throws MetaException, InvalidObjectException, InvalidInputException {
+ try {
+ Deadline.registerIfNot(100000);
+ List<Function> functions = store.getAllFunctions(DEFAULT_CATALOG_NAME);
+ for (Function func : functions) {
+ store.dropFunction(DEFAULT_CATALOG_NAME, func.getDbName(), func.getFunctionName());
+ }
+ for (String catName : store.getCatalogs()) {
+ List<String> dbs = store.getAllDatabases(catName);
+ for (String db : dbs) {
+ List<String> tbls = store.getAllTables(DEFAULT_CATALOG_NAME, db);
+ for (String tbl : tbls) {
+ Deadline.startTimer("getPartition");
+ List<Partition> parts = store.getPartitions(DEFAULT_CATALOG_NAME, db, tbl, 100);
+ for (Partition part : parts) {
+ store.dropPartition(DEFAULT_CATALOG_NAME, db, tbl, part.getValues());
+ }
+ // Find any constraints and drop them
+ Set<String> constraints = new HashSet<>();
+ List<SQLPrimaryKey> pk = store.getPrimaryKeys(DEFAULT_CATALOG_NAME, db, tbl);
+ if (pk != null) {
+ for (SQLPrimaryKey pkcol : pk) {
+ constraints.add(pkcol.getPk_name());
+ }
+ }
+ List<SQLForeignKey> fks = store.getForeignKeys(DEFAULT_CATALOG_NAME, null, null, db, tbl);
+ if (fks != null) {
+ for (SQLForeignKey fkcol : fks) {
+ constraints.add(fkcol.getFk_name());
+ }
+ }
+ for (String constraint : constraints) {
+ store.dropConstraint(DEFAULT_CATALOG_NAME, db, tbl, constraint);
+ }
+ store.dropTable(DEFAULT_CATALOG_NAME, db, tbl);
+ }
+ store.dropDatabase(catName, db);
+ }
+ store.dropCatalog(catName);
+ }
+ List<String> roles = store.listRoleNames();
+ for (String role : roles) {
+ store.removeRole(role);
+ }
+ } catch (NoSuchObjectException e) {
+ }
+ }
+
+ @Test
+ public void testQueryCloseOnError() throws Exception {
+ ObjectStore spy = Mockito.spy(objectStore);
+ spy.getAllDatabases(DEFAULT_CATALOG_NAME);
+ spy.getAllFunctions(DEFAULT_CATALOG_NAME);
+ spy.getAllTables(DEFAULT_CATALOG_NAME, DB1);
+ spy.getPartitionCount();
+ Mockito.verify(spy, Mockito.times(3))
+ .rollbackAndCleanup(Mockito.anyBoolean(), Mockito.<Query>anyObject());
+ }
+
+ @Test
+ public void testRetryingExecutorSleep() throws Exception {
+ RetryingExecutor re = new ObjectStore.RetryingExecutor(MetastoreConf.newMetastoreConf(), null);
+ Assert.assertTrue("invalid sleep value", re.getSleepInterval() >= 0);
+ }
+
+ @Ignore // See comment in ObjectStore.getDataSourceProps
+ @Test
+ public void testNonConfDatanucleusValueSet() {
+ String key = "datanucleus.no.such.key";
+ String value = "test_value";
+ String key1 = "blabla.no.such.key";
+ String value1 = "another_value";
+ Assume.assumeTrue(System.getProperty(key) == null);
+ Configuration localConf = MetastoreConf.newMetastoreConf();
+ MetaStoreTestUtils.setConfForStandloneMode(localConf);
+ localConf.set(key, value);
+ localConf.set(key1, value1);
+ objectStore = new ObjectStore();
+ objectStore.setConf(localConf);
+ Assert.assertEquals(value, objectStore.getProp().getProperty(key));
+ Assert.assertNull(objectStore.getProp().getProperty(key1));
+ }
+
+ /**
+ * Test notification operations
+ */
+ // TODO MS-SPLIT uncomment once we move EventMessage over
+ @Test
+ public void testNotificationOps() throws InterruptedException, MetaException {
+ final int NO_EVENT_ID = 0;
+ final int FIRST_EVENT_ID = 1;
+ final int SECOND_EVENT_ID = 2;
+
+ NotificationEvent event =
+ new NotificationEvent(0, 0, EventMessage.EventType.CREATE_DATABASE.toString(), "");
+ NotificationEventResponse eventResponse;
+ CurrentNotificationEventId eventId;
+
+ // Verify that there is no notifications available yet
+ eventId = objectStore.getCurrentNotificationEventId();
+ Assert.assertEquals(NO_EVENT_ID, eventId.getEventId());
+
+ // Verify that addNotificationEvent() updates the NotificationEvent with the new event ID
+ objectStore.addNotificationEvent(event);
+ Assert.assertEquals(FIRST_EVENT_ID, event.getEventId());
+ objectStore.addNotificationEvent(event);
+ Assert.assertEquals(SECOND_EVENT_ID, event.getEventId());
+
+ // Verify that objectStore fetches the latest notification event ID
+ eventId = objectStore.getCurrentNotificationEventId();
+ Assert.assertEquals(SECOND_EVENT_ID, eventId.getEventId());
+
+ // Verify that getNextNotification() returns all events
+ eventResponse = objectStore.getNextNotification(new NotificationEventRequest());
+ Assert.assertEquals(2, eventResponse.getEventsSize());
+ Assert.assertEquals(FIRST_EVENT_ID, eventResponse.getEvents().get(0).getEventId());
+ Assert.assertEquals(SECOND_EVENT_ID, eventResponse.getEvents().get(1).getEventId());
+
+ // Verify that getNextNotification(last) returns events after a specified event
+ eventResponse = objectStore.getNextNotification(new NotificationEventRequest(FIRST_EVENT_ID));
+ Assert.assertEquals(1, eventResponse.getEventsSize());
+ Assert.assertEquals(SECOND_EVENT_ID, eventResponse.getEvents().get(0).getEventId());
+
+ // Verify that getNextNotification(last) returns zero events if there are no more notifications available
+ eventResponse = objectStore.getNextNotification(new NotificationEventRequest(SECOND_EVENT_ID));
+ Assert.assertEquals(0, eventResponse.getEventsSize());
+
+ // Verify that cleanNotificationEvents() cleans up all old notifications
+ Thread.sleep(1);
+ objectStore.cleanNotificationEvents(1);
+ eventResponse = objectStore.getNextNotification(new NotificationEventRequest());
+ Assert.assertEquals(0, eventResponse.getEventsSize());
+ }
+
+ @Ignore(
+ "This test is here to allow testing with other databases like mysql / postgres etc\n"
+ + " with user changes to the code. This cannot be run on apache derby because of\n"
+ + " https://db.apache.org/derby/docs/10.10/devguide/cdevconcepts842385.html"
+ )
+ @Test
+ public void testConcurrentAddNotifications() throws ExecutionException, InterruptedException, MetaException {
+
+ final int NUM_THREADS = 10;
+ CyclicBarrier cyclicBarrier = new CyclicBarrier(NUM_THREADS,
+ () -> LoggerFactory.getLogger("test")
+ .debug(NUM_THREADS + " threads going to add notification"));
+
+ Configuration conf = MetastoreConf.newMetastoreConf();
+ MetaStoreTestUtils.setConfForStandloneMode(conf);
+ /*
+ Below are the properties that need to be set based on what database this test is going to be run
+ */
+
+ // conf.setVar(HiveConf.ConfVars.METASTORE_CONNECTION_DRIVER, "com.mysql.jdbc.Driver");
+ // conf.setVar(HiveConf.ConfVars.METASTORECONNECTURLKEY,
+ // "jdbc:mysql://localhost:3306/metastore_db");
+ // conf.setVar(HiveConf.ConfVars.METASTORE_CONNECTION_USER_NAME, "");
+ // conf.setVar(HiveConf.ConfVars.METASTOREPWD, "");
+
+ /*
+ we have to add this one manually as for tests the db is initialized via the metastoreDiretSQL
+ and we don't run the schema creation sql that includes the an insert for notification_sequence
+ which can be locked. the entry in notification_sequence happens via notification_event insertion.
+ */
+ objectStore.getPersistenceManager().newQuery(MNotificationLog.class, "eventType==''").execute();
+ objectStore.getPersistenceManager().newQuery(MNotificationNextId.class, "nextEventId==-1").execute();
+
+ objectStore.addNotificationEvent(
+ new NotificationEvent(0, 0,
+ EventMessage.EventType.CREATE_DATABASE.toString(),
+ "CREATE DATABASE DB initial"));
+
+ ExecutorService executorService = Executors.newFixedThreadPool(NUM_THREADS);
+ for (int i = 0; i < NUM_THREADS; i++) {
+ final int n = i;
+
+ executorService.execute(
+ () -> {
+ ObjectStore store = new ObjectStore();
+ store.setConf(conf);
+
+ String eventType = EventMessage.EventType.CREATE_DATABASE.toString();
+ NotificationEvent dbEvent =
+ new NotificationEvent(0, 0, eventType,
+ "CREATE DATABASE DB" + n);
+ System.out.println("ADDING NOTIFICATION");
+
+ try {
+ cyclicBarrier.await();
+ store.addNotificationEvent(dbEvent);
+ } catch (InterruptedException | BrokenBarrierException | MetaException e) {
+ throw new RuntimeException(e);
+ }
+ System.out.println("FINISH NOTIFICATION");
+ });
+ }
+ executorService.shutdown();
+ Assert.assertTrue(executorService.awaitTermination(15, TimeUnit.SECONDS));
+
+ // we have to setup this again as the underlying PMF keeps getting reinitialized with original
+ // reference closed
+ ObjectStore store = new ObjectStore();
+ store.setConf(conf);
+
+ NotificationEventResponse eventResponse = store.getNextNotification(
+ new NotificationEventRequest());
+ Assert.assertEquals(NUM_THREADS + 1, eventResponse.getEventsSize());
+ long previousId = 0;
+ for (NotificationEvent event : eventResponse.getEvents()) {
+ Assert.assertTrue("previous:" + previousId + " current:" + event.getEventId(),
+ previousId < event.getEventId());
+ Assert.assertTrue(previousId + 1 == event.getEventId());
+ previousId = event.getEventId();
+ }
+ }
+
+ private void createTestCatalog(String catName) throws MetaException {
+ Catalog cat = new CatalogBuilder()
+ .setName(catName)
+ .setLocation("/tmp")
+ .build();
+ objectStore.createCatalog(cat);
+ }
+ }
+