You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by se...@apache.org on 2018/07/25 18:27:14 UTC
[05/50] [abbrv] hive git commit: HIVE-19416 : merge master into
branch (Sergey Shelukhin) 0719
http://git-wip-us.apache.org/repos/asf/hive/blob/651e7950/standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/TestHiveMetaStore.java
----------------------------------------------------------------------
diff --cc standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/TestHiveMetaStore.java
index 0000000,cb32236..75ab80b
mode 000000,100644..100644
--- a/standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/TestHiveMetaStore.java
+++ b/standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/TestHiveMetaStore.java
@@@ -1,0 -1,3103 +1,3102 @@@
+ /*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+ package org.apache.hadoop.hive.metastore;
+
+ import java.lang.reflect.Field;
+ import java.io.IOException;
+ import java.sql.Connection;
+ import java.sql.DriverManager;
+ import java.sql.PreparedStatement;
+ import java.sql.SQLException;
+ import java.util.ArrayList;
+ import java.util.Arrays;
+ import java.util.Collections;
+ import java.util.HashMap;
+ import java.util.HashSet;
+ import java.util.LinkedHashMap;
+ import java.util.List;
+ import java.util.Map;
+ import java.util.Set;
+ import java.util.concurrent.Callable;
+ import java.util.concurrent.ExecutorService;
+ import java.util.concurrent.Executors;
+ import java.util.concurrent.Future;
+ import java.util.concurrent.TimeUnit;
+
+ import com.google.common.collect.Sets;
+ import org.apache.hadoop.hive.metastore.api.CreationMetadata;
+ import org.apache.hadoop.hive.metastore.client.builder.DatabaseBuilder;
+ import org.apache.hadoop.hive.metastore.client.builder.TableBuilder;
+ import org.apache.hadoop.hive.metastore.conf.MetastoreConf;
+ import org.apache.hadoop.hive.metastore.conf.MetastoreConf.ConfVars;
+ import org.apache.hadoop.hive.metastore.utils.FileUtils;
+ import org.apache.hadoop.hive.metastore.utils.SecurityUtils;
+ import org.datanucleus.api.jdo.JDOPersistenceManager;
+ import org.datanucleus.api.jdo.JDOPersistenceManagerFactory;
+ import org.junit.Assert;
+ import org.junit.Before;
+ import org.slf4j.Logger;
+ import org.slf4j.LoggerFactory;
+ import org.apache.hadoop.conf.Configuration;
+ import org.apache.hadoop.fs.FileSystem;
+ import org.apache.hadoop.fs.Path;
+ import org.apache.hadoop.fs.permission.FsPermission;
+ import org.apache.hadoop.hive.metastore.api.AggrStats;
+ import org.apache.hadoop.hive.metastore.api.ColumnStatistics;
+ import org.apache.hadoop.hive.metastore.api.ColumnStatisticsData;
+ import org.apache.hadoop.hive.metastore.api.ColumnStatisticsDesc;
+ import org.apache.hadoop.hive.metastore.api.ColumnStatisticsObj;
+ import org.apache.hadoop.hive.metastore.api.ConfigValSecurityException;
+ import org.apache.hadoop.hive.metastore.api.Database;
+ import org.apache.hadoop.hive.metastore.api.DoubleColumnStatsData;
+ import org.apache.hadoop.hive.metastore.api.FieldSchema;
+ import org.apache.hadoop.hive.metastore.api.Function;
+ import org.apache.hadoop.hive.metastore.api.FunctionType;
+ import org.apache.hadoop.hive.metastore.api.GetAllFunctionsResponse;
+ import org.apache.hadoop.hive.metastore.api.InvalidObjectException;
+ import org.apache.hadoop.hive.metastore.api.InvalidOperationException;
+ import org.apache.hadoop.hive.metastore.api.MetaException;
+ import org.apache.hadoop.hive.metastore.api.NoSuchObjectException;
+ import org.apache.hadoop.hive.metastore.api.Partition;
+ import org.apache.hadoop.hive.metastore.api.PrincipalType;
+ import org.apache.hadoop.hive.metastore.api.ResourceType;
+ import org.apache.hadoop.hive.metastore.api.ResourceUri;
+ import org.apache.hadoop.hive.metastore.api.SerDeInfo;
+ import org.apache.hadoop.hive.metastore.api.StorageDescriptor;
+ import org.apache.hadoop.hive.metastore.api.StringColumnStatsData;
+ import org.apache.hadoop.hive.metastore.api.Table;
+ import org.apache.hadoop.hive.metastore.api.Type;
+ import org.apache.hadoop.hive.metastore.api.UnknownDBException;
+ import org.apache.hadoop.hive.metastore.utils.MetaStoreUtils;
+ import org.apache.hadoop.util.StringUtils;
+ import org.apache.thrift.TException;
+ import org.junit.Test;
+
+ import com.google.common.collect.Lists;
+
+ import static org.junit.Assert.assertEquals;
+ import static org.junit.Assert.assertFalse;
+ import static org.junit.Assert.assertNotNull;
+ import static org.junit.Assert.assertNull;
+ import static org.junit.Assert.assertTrue;
+ import static org.junit.Assert.fail;
+
+ public abstract class TestHiveMetaStore {
+ private static final Logger LOG = LoggerFactory.getLogger(TestHiveMetaStore.class);
+ protected static HiveMetaStoreClient client;
+ protected static Configuration conf = MetastoreConf.newMetastoreConf();
+ protected static Warehouse warehouse;
+ protected static boolean isThriftClient = false;
+
+ private static final String TEST_DB1_NAME = "testdb1";
+ private static final String TEST_DB2_NAME = "testdb2";
+
+ private static final int DEFAULT_LIMIT_PARTITION_REQUEST = 100;
+
+ protected abstract HiveMetaStoreClient createClient() throws Exception;
+
+ @Before
+ public void setUp() throws Exception {
+ warehouse = new Warehouse(conf);
+
+ // set some values to use for getting conf. vars
+ MetastoreConf.setBoolVar(conf, ConfVars.METRICS_ENABLED, true);
+ conf.set("hive.key1", "value1");
+ conf.set("hive.key2", "http://www.example.com");
+ conf.set("hive.key3", "");
+ conf.set("hive.key4", "0");
+ conf.set("datanucleus.autoCreateTables", "false");
+
+ MetaStoreTestUtils.setConfForStandloneMode(conf);
+ MetastoreConf.setLongVar(conf, ConfVars.BATCH_RETRIEVE_MAX, 2);
+ MetastoreConf.setLongVar(conf, ConfVars.LIMIT_PARTITION_REQUEST, DEFAULT_LIMIT_PARTITION_REQUEST);
+ MetastoreConf.setVar(conf, ConfVars.STORAGE_SCHEMA_READER_IMPL, "no.such.class");
+ }
+
+ @Test
+ public void testNameMethods() {
+ Map<String, String> spec = new LinkedHashMap<>();
+ spec.put("ds", "2008-07-01 14:13:12");
+ spec.put("hr", "14");
+ List<String> vals = new ArrayList<>();
+ vals.addAll(spec.values());
+ String partName = "ds=2008-07-01 14%3A13%3A12/hr=14";
+
+ try {
+ List<String> testVals = client.partitionNameToVals(partName);
+ assertTrue("Values from name are incorrect", vals.equals(testVals));
+
+ Map<String, String> testSpec = client.partitionNameToSpec(partName);
+ assertTrue("Spec from name is incorrect", spec.equals(testSpec));
+
+ List<String> emptyVals = client.partitionNameToVals("");
+ assertTrue("Values should be empty", emptyVals.size() == 0);
+
+ Map<String, String> emptySpec = client.partitionNameToSpec("");
+ assertTrue("Spec should be empty", emptySpec.size() == 0);
+ } catch (Exception e) {
+ fail();
+ }
+ }
+
+ /**
+ * tests create table and partition and tries to drop the table without
+ * droppping the partition
+ *
+ */
+ @Test
+ public void testPartition() throws Exception {
+ partitionTester(client, conf);
+ }
+
+ private static void partitionTester(HiveMetaStoreClient client, Configuration conf)
+ throws Exception {
+ try {
+ String dbName = "compdb";
+ String tblName = "comptbl";
+ String typeName = "Person";
+ List<String> vals = makeVals("2008-07-01 14:13:12", "14");
+ List<String> vals2 = makeVals("2008-07-01 14:13:12", "15");
+ List<String> vals3 = makeVals("2008-07-02 14:13:12", "15");
+ List<String> vals4 = makeVals("2008-07-03 14:13:12", "151");
+
+ client.dropTable(dbName, tblName);
+ silentDropDatabase(dbName);
+ new DatabaseBuilder()
+ .setName(dbName)
+ .create(client, conf);
+ Database db = client.getDatabase(dbName);
+ Path dbPath = new Path(db.getLocationUri());
+ FileSystem fs = FileSystem.get(dbPath.toUri(), conf);
+
+ client.dropType(typeName);
+ Type typ1 = new Type();
+ typ1.setName(typeName);
+ typ1.setFields(new ArrayList<>(2));
+ typ1.getFields().add(
+ new FieldSchema("name", ColumnType.STRING_TYPE_NAME, ""));
+ typ1.getFields().add(
+ new FieldSchema("income", ColumnType.INT_TYPE_NAME, ""));
+ client.createType(typ1);
+
+ List<String> skewedColValue = Collections.singletonList("1");
+ Table tbl = new TableBuilder()
+ .setDbName(dbName)
+ .setTableName(tblName)
+ .setCols(typ1.getFields())
+ .setNumBuckets(1)
+ .addBucketCol("name")
+ .addTableParam("test_param_1", "Use this for comments etc")
+ .addSerdeParam(ColumnType.SERIALIZATION_FORMAT, "1")
+ .addSkewedColName("name")
+ .setSkewedColValues(Collections.singletonList(skewedColValue))
+ .setSkewedColValueLocationMaps(Collections.singletonMap(skewedColValue, "location1"))
+ .addPartCol("ds", ColumnType.STRING_TYPE_NAME)
+ .addPartCol("hr", ColumnType.STRING_TYPE_NAME)
+ .create(client, conf);
+
+ if (isThriftClient) {
+ // the createTable() above does not update the location in the 'tbl'
+ // object when the client is a thrift client and the code below relies
+ // on the location being present in the 'tbl' object - so get the table
+ // from the metastore
+ tbl = client.getTable(dbName, tblName);
+ }
+
+ Partition part = makePartitionObject(dbName, tblName, vals, tbl, "/part1");
+ Partition part2 = makePartitionObject(dbName, tblName, vals2, tbl, "/part2");
+ Partition part3 = makePartitionObject(dbName, tblName, vals3, tbl, "/part3");
+ Partition part4 = makePartitionObject(dbName, tblName, vals4, tbl, "/part4");
+
+ // check if the partition exists (it shouldn't)
+ boolean exceptionThrown = false;
+ try {
+ Partition p = client.getPartition(dbName, tblName, vals);
+ } catch(Exception e) {
+ assertEquals("partition should not have existed",
+ NoSuchObjectException.class, e.getClass());
+ exceptionThrown = true;
+ }
+ assertTrue("getPartition() should have thrown NoSuchObjectException", exceptionThrown);
+ Partition retp = client.add_partition(part);
+ assertNotNull("Unable to create partition " + part, retp);
+ Partition retp2 = client.add_partition(part2);
+ assertNotNull("Unable to create partition " + part2, retp2);
+ Partition retp3 = client.add_partition(part3);
+ assertNotNull("Unable to create partition " + part3, retp3);
+ Partition retp4 = client.add_partition(part4);
+ assertNotNull("Unable to create partition " + part4, retp4);
+
+ Partition part_get = client.getPartition(dbName, tblName, part.getValues());
- if(isThriftClient) {
- // since we are using thrift, 'part' will not have the create time and
- // last DDL time set since it does not get updated in the add_partition()
- // call - likewise part2 and part3 - set it correctly so that equals check
- // doesn't fail
- adjust(client, part, dbName, tblName);
- adjust(client, part2, dbName, tblName);
- adjust(client, part3, dbName, tblName);
- }
++ // since we are using thrift, 'part' will not have the create time and
++ // last DDL time set since it does not get updated in the add_partition()
++ // call - likewise part2 and part3 - set it correctly so that equals check
++ // doesn't fail
++ adjust(client, part, dbName, tblName, isThriftClient);
++ adjust(client, part2, dbName, tblName, isThriftClient);
++ adjust(client, part3, dbName, tblName, isThriftClient);
+ assertTrue("Partitions are not same", part.equals(part_get));
+
+ // check null cols schemas for a partition
+ List<String> vals6 = makeVals("2016-02-22 00:00:00", "16");
+ Partition part6 = makePartitionObject(dbName, tblName, vals6, tbl, "/part5");
+ part6.getSd().setCols(null);
+ LOG.info("Creating partition will null field schema");
+ client.add_partition(part6);
+ LOG.info("Listing all partitions for table " + dbName + "." + tblName);
+ final List<Partition> partitions = client.listPartitions(dbName, tblName, (short) -1);
+ boolean foundPart = false;
+ for (Partition p : partitions) {
+ if (p.getValues().equals(vals6)) {
+ assertNull(p.getSd().getCols());
+ LOG.info("Found partition " + p + " having null field schema");
+ foundPart = true;
+ }
+ }
+ assertTrue(foundPart);
+
+ String partName = "ds=" + FileUtils.escapePathName("2008-07-01 14:13:12") + "/hr=14";
+ String part2Name = "ds=" + FileUtils.escapePathName("2008-07-01 14:13:12") + "/hr=15";
+ String part3Name = "ds=" + FileUtils.escapePathName("2008-07-02 14:13:12") + "/hr=15";
+ String part4Name = "ds=" + FileUtils.escapePathName("2008-07-03 14:13:12") + "/hr=151";
+
+ part_get = client.getPartition(dbName, tblName, partName);
+ assertTrue("Partitions are not the same", part.equals(part_get));
+
+ // Test partition listing with a partial spec - ds is specified but hr is not
+ List<String> partialVals = new ArrayList<>();
+ partialVals.add(vals.get(0));
+ Set<Partition> parts = new HashSet<>();
+ parts.add(part);
+ parts.add(part2);
+
+ List<Partition> partial = client.listPartitions(dbName, tblName, partialVals,
+ (short) -1);
+ assertTrue("Should have returned 2 partitions", partial.size() == 2);
+ assertTrue("Not all parts returned", partial.containsAll(parts));
+
+ Set<String> partNames = new HashSet<>();
+ partNames.add(partName);
+ partNames.add(part2Name);
+ List<String> partialNames = client.listPartitionNames(dbName, tblName, partialVals,
+ (short) -1);
+ assertTrue("Should have returned 2 partition names", partialNames.size() == 2);
+ assertTrue("Not all part names returned", partialNames.containsAll(partNames));
+
+ partNames.add(part3Name);
+ partNames.add(part4Name);
+ partialVals.clear();
+ partialVals.add("");
+ partialNames = client.listPartitionNames(dbName, tblName, partialVals, (short) -1);
+ assertTrue("Should have returned 5 partition names", partialNames.size() == 5);
+ assertTrue("Not all part names returned", partialNames.containsAll(partNames));
+
+ // Test partition listing with a partial spec - hr is specified but ds is not
+ parts.clear();
+ parts.add(part2);
+ parts.add(part3);
+
+ partialVals.clear();
+ partialVals.add("");
+ partialVals.add(vals2.get(1));
+
+ partial = client.listPartitions(dbName, tblName, partialVals, (short) -1);
+ assertEquals("Should have returned 2 partitions", 2, partial.size());
+ assertTrue("Not all parts returned", partial.containsAll(parts));
+
+ partNames.clear();
+ partNames.add(part2Name);
+ partNames.add(part3Name);
+ partialNames = client.listPartitionNames(dbName, tblName, partialVals,
+ (short) -1);
+ assertEquals("Should have returned 2 partition names", 2, partialNames.size());
+ assertTrue("Not all part names returned", partialNames.containsAll(partNames));
+
+ // Verify escaped partition names don't return partitions
+ exceptionThrown = false;
+ try {
+ String badPartName = "ds=2008-07-01 14%3A13%3A12/hrs=14";
+ client.getPartition(dbName, tblName, badPartName);
+ } catch(NoSuchObjectException e) {
+ exceptionThrown = true;
+ }
+ assertTrue("Bad partition spec should have thrown an exception", exceptionThrown);
+
+ Path partPath = new Path(part.getSd().getLocation());
+
+
+ assertTrue(fs.exists(partPath));
+ client.dropPartition(dbName, tblName, part.getValues(), true);
+ assertFalse(fs.exists(partPath));
+
+ // Test append_partition_by_name
+ client.appendPartition(dbName, tblName, partName);
+ Partition part5 = client.getPartition(dbName, tblName, part.getValues());
+ assertTrue("Append partition by name failed", part5.getValues().equals(vals));
+ Path part5Path = new Path(part5.getSd().getLocation());
+ assertTrue(fs.exists(part5Path));
+
+ // Test drop_partition_by_name
+ assertTrue("Drop partition by name failed",
+ client.dropPartition(dbName, tblName, partName, true));
+ assertFalse(fs.exists(part5Path));
+
+ // add the partition again so that drop table with a partition can be
+ // tested
+ retp = client.add_partition(part);
+ assertNotNull("Unable to create partition " + part, retp);
+
+ // test add_partitions
+
+ List<String> mvals1 = makeVals("2008-07-04 14:13:12", "14641");
+ List<String> mvals2 = makeVals("2008-07-04 14:13:12", "14642");
+ List<String> mvals3 = makeVals("2008-07-04 14:13:12", "14643");
+ List<String> mvals4 = makeVals("2008-07-04 14:13:12", "14643"); // equal to 3
+ List<String> mvals5 = makeVals("2008-07-04 14:13:12", "14645");
+
+ Exception savedException;
+
+ // add_partitions(empty list) : ok, normal operation
+ client.add_partitions(new ArrayList<>());
+
+ // add_partitions(1,2,3) : ok, normal operation
+ Partition mpart1 = makePartitionObject(dbName, tblName, mvals1, tbl, "/mpart1");
+ Partition mpart2 = makePartitionObject(dbName, tblName, mvals2, tbl, "/mpart2");
+ Partition mpart3 = makePartitionObject(dbName, tblName, mvals3, tbl, "/mpart3");
+ client.add_partitions(Arrays.asList(mpart1,mpart2,mpart3));
+
- if(isThriftClient) {
- // do DDL time munging if thrift mode
- adjust(client, mpart1, dbName, tblName);
- adjust(client, mpart2, dbName, tblName);
- adjust(client, mpart3, dbName, tblName);
- }
++ // do DDL time munging if thrift mode
++ adjust(client, mpart1, dbName, tblName, isThriftClient);
++ adjust(client, mpart2, dbName, tblName, isThriftClient);
++ adjust(client, mpart3, dbName, tblName, isThriftClient);
+ verifyPartitionsPublished(client, dbName, tblName,
+ Arrays.asList(mvals1.get(0)),
+ Arrays.asList(mpart1,mpart2,mpart3));
+
+ Partition mpart4 = makePartitionObject(dbName, tblName, mvals4, tbl, "/mpart4");
+ Partition mpart5 = makePartitionObject(dbName, tblName, mvals5, tbl, "/mpart5");
+
+ // create dir for /mpart5
+ Path mp5Path = new Path(mpart5.getSd().getLocation());
+ warehouse.mkdirs(mp5Path);
+ assertTrue(fs.exists(mp5Path));
+
+ // add_partitions(5,4) : err = duplicate keyvals on mpart4
+ savedException = null;
+ try {
+ client.add_partitions(Arrays.asList(mpart5,mpart4));
+ } catch (Exception e) {
+ savedException = e;
+ } finally {
+ assertNotNull(savedException);
+ }
+
+ // check that /mpart4 does not exist, but /mpart5 still does.
+ assertTrue(fs.exists(mp5Path));
+ assertFalse(fs.exists(new Path(mpart4.getSd().getLocation())));
+
+ // add_partitions(5) : ok
+ client.add_partitions(Arrays.asList(mpart5));
+
- if(isThriftClient) {
- // do DDL time munging if thrift mode
- adjust(client, mpart5, dbName, tblName);
- }
++ // do DDL time munging if thrift mode
++ adjust(client, mpart5, dbName, tblName, isThriftClient);
+
+ verifyPartitionsPublished(client, dbName, tblName,
+ Arrays.asList(mvals1.get(0)),
+ Arrays.asList(mpart1,mpart2,mpart3,mpart5));
+
+ //// end add_partitions tests
+
+ client.dropTable(dbName, tblName);
+
+ client.dropType(typeName);
+
+ // recreate table as external, drop partition and it should
+ // still exist
+ tbl.setParameters(new HashMap<>());
+ tbl.getParameters().put("EXTERNAL", "TRUE");
+ client.createTable(tbl);
+ retp = client.add_partition(part);
+ assertTrue(fs.exists(partPath));
+ client.dropPartition(dbName, tblName, part.getValues(), true);
+ assertTrue(fs.exists(partPath));
+
+ for (String tableName : client.getTables(dbName, "*")) {
+ client.dropTable(dbName, tableName);
+ }
+
+ client.dropDatabase(dbName);
+
+ } catch (Exception e) {
+ System.err.println(StringUtils.stringifyException(e));
+ System.err.println("testPartition() failed.");
+ throw e;
+ }
+ }
+
+ private static void verifyPartitionsPublished(HiveMetaStoreClient client,
+ String dbName, String tblName, List<String> partialSpec,
+ List<Partition> expectedPartitions) throws TException {
+ // Test partition listing with a partial spec
+
+ List<Partition> mpartial = client.listPartitions(dbName, tblName, partialSpec,
+ (short) -1);
+ assertEquals("Should have returned "+expectedPartitions.size()+
+ " partitions, returned " + mpartial.size(),
+ expectedPartitions.size(), mpartial.size());
+ assertTrue("Not all parts returned", mpartial.containsAll(expectedPartitions));
+ }
+
+ private static List<String> makeVals(String ds, String id) {
+ List <String> vals4 = new ArrayList<>(2);
+ vals4 = new ArrayList<>(2);
+ vals4.add(ds);
+ vals4.add(id);
+ return vals4;
+ }
+
+ private static Partition makePartitionObject(String dbName, String tblName,
+ List<String> ptnVals, Table tbl, String ptnLocationSuffix) throws MetaException {
+ Partition part4 = new Partition();
+ part4.setDbName(dbName);
+ part4.setTableName(tblName);
+ part4.setValues(ptnVals);
+ part4.setParameters(new HashMap<>());
+ part4.setSd(tbl.getSd().deepCopy());
+ part4.getSd().setSerdeInfo(tbl.getSd().getSerdeInfo().deepCopy());
+ part4.getSd().setLocation(tbl.getSd().getLocation() + ptnLocationSuffix);
+ MetaStoreUtils.updatePartitionStatsFast(part4, tbl, warehouse, false, false, null, true);
+ return part4;
+ }
+
+ @Test
+ public void testListPartitions() throws Throwable {
+ // create a table with multiple partitions
+ String dbName = "compdb";
+ String tblName = "comptbl";
+ String typeName = "Person";
+
+ cleanUp(dbName, tblName, typeName);
+
+ List<List<String>> values = new ArrayList<>();
+ values.add(makeVals("2008-07-01 14:13:12", "14"));
+ values.add(makeVals("2008-07-01 14:13:12", "15"));
+ values.add(makeVals("2008-07-02 14:13:12", "15"));
+ values.add(makeVals("2008-07-03 14:13:12", "151"));
+
+ createMultiPartitionTableSchema(dbName, tblName, typeName, values);
+
+ List<Partition> partitions = client.listPartitions(dbName, tblName, (short)-1);
+ assertNotNull("should have returned partitions", partitions);
+ assertEquals(" should have returned " + values.size() +
+ " partitions", values.size(), partitions.size());
+
+ partitions = client.listPartitions(dbName, tblName, (short)(values.size()/2));
+
+ assertNotNull("should have returned partitions", partitions);
+ assertEquals(" should have returned " + values.size() / 2 +
+ " partitions",values.size() / 2, partitions.size());
+
+
+ partitions = client.listPartitions(dbName, tblName, (short) (values.size() * 2));
+
+ assertNotNull("should have returned partitions", partitions);
+ assertEquals(" should have returned " + values.size() +
+ " partitions",values.size(), partitions.size());
+
+ cleanUp(dbName, tblName, typeName);
+
+ }
+
+ @Test
+ public void testListPartitionsWihtLimitEnabled() throws Throwable {
+ // create a table with multiple partitions
+ String dbName = "compdb";
+ String tblName = "comptbl";
+ String typeName = "Person";
+
+ cleanUp(dbName, tblName, typeName);
+
+ // Create too many partitions, just enough to validate over limit requests
+ List<List<String>> values = new ArrayList<>();
+ for (int i=0; i<DEFAULT_LIMIT_PARTITION_REQUEST + 1; i++) {
+ values.add(makeVals("2008-07-01 14:13:12", Integer.toString(i)));
+ }
+
+ createMultiPartitionTableSchema(dbName, tblName, typeName, values);
+
+ List<Partition> partitions;
+ short maxParts;
+
+ // Requesting more partitions than allowed should throw an exception
+ try {
+ maxParts = -1;
+ partitions = client.listPartitions(dbName, tblName, maxParts);
+ fail("should have thrown MetaException about partition limit");
+ } catch (MetaException e) {
+ assertTrue(true);
+ }
+
+ // Requesting more partitions than allowed should throw an exception
+ try {
+ maxParts = DEFAULT_LIMIT_PARTITION_REQUEST + 1;
+ partitions = client.listPartitions(dbName, tblName, maxParts);
+ fail("should have thrown MetaException about partition limit");
+ } catch (MetaException e) {
+ assertTrue(true);
+ }
+
+ // Requesting less partitions than allowed should work
+ maxParts = DEFAULT_LIMIT_PARTITION_REQUEST / 2;
+ partitions = client.listPartitions(dbName, tblName, maxParts);
+ assertNotNull("should have returned partitions", partitions);
+ assertEquals(" should have returned 50 partitions", maxParts, partitions.size());
+ }
+
+ @Test
+ public void testAlterTableCascade() throws Throwable {
+ // create a table with multiple partitions
+ String dbName = "compdb";
+ String tblName = "comptbl";
+ String typeName = "Person";
+
+ cleanUp(dbName, tblName, typeName);
+
+ List<List<String>> values = new ArrayList<>();
+ values.add(makeVals("2008-07-01 14:13:12", "14"));
+ values.add(makeVals("2008-07-01 14:13:12", "15"));
+ values.add(makeVals("2008-07-02 14:13:12", "15"));
+ values.add(makeVals("2008-07-03 14:13:12", "151"));
+
+ createMultiPartitionTableSchema(dbName, tblName, typeName, values);
+ Table tbl = client.getTable(dbName, tblName);
+ List<FieldSchema> cols = tbl.getSd().getCols();
+ cols.add(new FieldSchema("new_col", ColumnType.STRING_TYPE_NAME, ""));
+ tbl.getSd().setCols(cols);
+ //add new column with cascade option
+ client.alter_table(dbName, tblName, tbl, true);
+ //
+ Table tbl2 = client.getTable(dbName, tblName);
+ assertEquals("Unexpected number of cols", 3, tbl2.getSd().getCols().size());
+ assertEquals("Unexpected column name", "new_col", tbl2.getSd().getCols().get(2).getName());
+ //get a partition
+ List<String> pvalues = new ArrayList<>(2);
+ pvalues.add("2008-07-01 14:13:12");
+ pvalues.add("14");
+ Partition partition = client.getPartition(dbName, tblName, pvalues);
+ assertEquals("Unexpected number of cols", 3, partition.getSd().getCols().size());
+ assertEquals("Unexpected column name", "new_col", partition.getSd().getCols().get(2).getName());
+
+ //add another column
+ cols = tbl.getSd().getCols();
+ cols.add(new FieldSchema("new_col2", ColumnType.STRING_TYPE_NAME, ""));
+ tbl.getSd().setCols(cols);
+ //add new column with no cascade option
+ client.alter_table(dbName, tblName, tbl, false);
+ tbl2 = client.getTable(dbName, tblName);
+ assertEquals("Unexpected number of cols", 4, tbl2.getSd().getCols().size());
+ assertEquals("Unexpected column name", "new_col2", tbl2.getSd().getCols().get(3).getName());
+ //get partition, this partition should not have the newly added column since cascade option
+ //was false
+ partition = client.getPartition(dbName, tblName, pvalues);
+ assertEquals("Unexpected number of cols", 3, partition.getSd().getCols().size());
+ }
+
+
+ @Test
+ public void testListPartitionNames() throws Throwable {
+ // create a table with multiple partitions
+ String dbName = "compdb";
+ String tblName = "comptbl";
+ String typeName = "Person";
+
+ cleanUp(dbName, tblName, typeName);
+
+ List<List<String>> values = new ArrayList<>();
+ values.add(makeVals("2008-07-01 14:13:12", "14"));
+ values.add(makeVals("2008-07-01 14:13:12", "15"));
+ values.add(makeVals("2008-07-02 14:13:12", "15"));
+ values.add(makeVals("2008-07-03 14:13:12", "151"));
+
+
+
+ createMultiPartitionTableSchema(dbName, tblName, typeName, values);
+
+ List<String> partitions = client.listPartitionNames(dbName, tblName, (short)-1);
+ assertNotNull("should have returned partitions", partitions);
+ assertEquals(" should have returned " + values.size() +
+ " partitions", values.size(), partitions.size());
+
+ partitions = client.listPartitionNames(dbName, tblName, (short)(values.size()/2));
+
+ assertNotNull("should have returned partitions", partitions);
+ assertEquals(" should have returned " + values.size() / 2 +
+ " partitions",values.size() / 2, partitions.size());
+
+
+ partitions = client.listPartitionNames(dbName, tblName, (short) (values.size() * 2));
+
+ assertNotNull("should have returned partitions", partitions);
+ assertEquals(" should have returned " + values.size() +
+ " partitions",values.size(), partitions.size());
+
+ cleanUp(dbName, tblName, typeName);
+
+ }
+
+
+ @Test
+ public void testDropTable() throws Throwable {
+ // create a table with multiple partitions
+ String dbName = "compdb";
+ String tblName = "comptbl";
+ String typeName = "Person";
+
+ cleanUp(dbName, tblName, typeName);
+
+ List<List<String>> values = new ArrayList<>();
+ values.add(makeVals("2008-07-01 14:13:12", "14"));
+ values.add(makeVals("2008-07-01 14:13:12", "15"));
+ values.add(makeVals("2008-07-02 14:13:12", "15"));
+ values.add(makeVals("2008-07-03 14:13:12", "151"));
+
+ createMultiPartitionTableSchema(dbName, tblName, typeName, values);
+
+ client.dropTable(dbName, tblName);
+ client.dropType(typeName);
+
+ boolean exceptionThrown = false;
+ try {
+ client.getTable(dbName, tblName);
+ } catch(Exception e) {
+ assertEquals("table should not have existed",
+ NoSuchObjectException.class, e.getClass());
+ exceptionThrown = true;
+ }
+ assertTrue("Table " + tblName + " should have been dropped ", exceptionThrown);
+
+ }
+
+ @Test
+ public void testAlterViewParititon() throws Throwable {
+ String dbName = "compdb";
+ String tblName = "comptbl";
+ String viewName = "compView";
+
+ client.dropTable(dbName, tblName);
+ silentDropDatabase(dbName);
+ new DatabaseBuilder()
+ .setName(dbName)
+ .setDescription("Alter Partition Test database")
+ .create(client, conf);
+
+ Table tbl = new TableBuilder()
+ .setDbName(dbName)
+ .setTableName(tblName)
+ .addCol("name", ColumnType.STRING_TYPE_NAME)
+ .addCol("income", ColumnType.INT_TYPE_NAME)
+ .create(client, conf);
+
+ if (isThriftClient) {
+ // the createTable() above does not update the location in the 'tbl'
+ // object when the client is a thrift client and the code below relies
+ // on the location being present in the 'tbl' object - so get the table
+ // from the metastore
+ tbl = client.getTable(dbName, tblName);
+ }
+
+ ArrayList<FieldSchema> viewCols = new ArrayList<>(1);
+ viewCols.add(new FieldSchema("income", ColumnType.INT_TYPE_NAME, ""));
+
+ ArrayList<FieldSchema> viewPartitionCols = new ArrayList<>(1);
+ viewPartitionCols.add(new FieldSchema("name", ColumnType.STRING_TYPE_NAME, ""));
+
+ Table view = new Table();
+ view.setDbName(dbName);
+ view.setTableName(viewName);
+ view.setTableType(TableType.VIRTUAL_VIEW.name());
+ view.setPartitionKeys(viewPartitionCols);
+ view.setViewOriginalText("SELECT income, name FROM " + tblName);
+ view.setViewExpandedText("SELECT `" + tblName + "`.`income`, `" + tblName +
+ "`.`name` FROM `" + dbName + "`.`" + tblName + "`");
+ view.setRewriteEnabled(false);
+ StorageDescriptor viewSd = new StorageDescriptor();
+ view.setSd(viewSd);
+ viewSd.setCols(viewCols);
+ viewSd.setCompressed(false);
+ viewSd.setParameters(new HashMap<>());
+ viewSd.setSerdeInfo(new SerDeInfo());
+ viewSd.getSerdeInfo().setParameters(new HashMap<>());
+
+ client.createTable(view);
+
+ if (isThriftClient) {
+ // the createTable() above does not update the location in the 'tbl'
+ // object when the client is a thrift client and the code below relies
+ // on the location being present in the 'tbl' object - so get the table
+ // from the metastore
+ view = client.getTable(dbName, viewName);
+ }
+
+ List<String> vals = new ArrayList<>(1);
+ vals.add("abc");
+
+ Partition part = new Partition();
+ part.setDbName(dbName);
+ part.setTableName(viewName);
+ part.setValues(vals);
+ part.setParameters(new HashMap<>());
+
+ client.add_partition(part);
+
+ Partition part2 = client.getPartition(dbName, viewName, part.getValues());
+
+ part2.getParameters().put("a", "b");
+
+ client.alter_partition(dbName, viewName, part2, null);
+
+ Partition part3 = client.getPartition(dbName, viewName, part.getValues());
+ assertEquals("couldn't view alter partition", part3.getParameters().get(
+ "a"), "b");
+
+ client.dropTable(dbName, viewName);
+
+ client.dropTable(dbName, tblName);
+
+ client.dropDatabase(dbName);
+ }
+
+ @Test
+ public void testAlterPartition() throws Throwable {
+
+ try {
+ String dbName = "compdb";
+ String tblName = "comptbl";
+ List<String> vals = new ArrayList<>(2);
+ vals.add("2008-07-01");
+ vals.add("14");
+
+ client.dropTable(dbName, tblName);
+ silentDropDatabase(dbName);
+ new DatabaseBuilder()
+ .setName(dbName)
+ .setDescription("Alter Partition Test database")
+ .create(client, conf);
+
+ Table tbl = new TableBuilder()
+ .setDbName(dbName)
+ .setTableName(tblName)
+ .addCol("name", ColumnType.STRING_TYPE_NAME)
+ .addCol("income", ColumnType.INT_TYPE_NAME)
+ .addTableParam("test_param_1", "Use this for comments etc")
+ .addBucketCol("name")
+ .addSerdeParam(ColumnType.SERIALIZATION_FORMAT, "1")
+ .addPartCol("ds", ColumnType.STRING_TYPE_NAME)
+ .addPartCol("hr", ColumnType.INT_TYPE_NAME)
+ .create(client, conf);
+
+ if (isThriftClient) {
+ // the createTable() above does not update the location in the 'tbl'
+ // object when the client is a thrift client and the code below relies
+ // on the location being present in the 'tbl' object - so get the table
+ // from the metastore
+ tbl = client.getTable(dbName, tblName);
+ }
+
+ Partition part = new Partition();
+ part.setDbName(dbName);
+ part.setTableName(tblName);
+ part.setValues(vals);
+ part.setParameters(new HashMap<>());
+ part.setSd(tbl.getSd());
+ part.getSd().setSerdeInfo(tbl.getSd().getSerdeInfo());
+ part.getSd().setLocation(tbl.getSd().getLocation() + "/part1");
+
+ client.add_partition(part);
+
+ Partition part2 = client.getPartition(dbName, tblName, part.getValues());
+
+ part2.getParameters().put("retention", "10");
+ part2.getSd().setNumBuckets(12);
+ part2.getSd().getSerdeInfo().getParameters().put("abc", "1");
+ client.alter_partition(dbName, tblName, part2, null);
+
+ Partition part3 = client.getPartition(dbName, tblName, part.getValues());
+ assertEquals("couldn't alter partition", part3.getParameters().get(
+ "retention"), "10");
+ assertEquals("couldn't alter partition", part3.getSd().getSerdeInfo()
+ .getParameters().get("abc"), "1");
+ assertEquals("couldn't alter partition", part3.getSd().getNumBuckets(),
+ 12);
+
+ client.dropTable(dbName, tblName);
+
+ client.dropDatabase(dbName);
+ } catch (Exception e) {
+ System.err.println(StringUtils.stringifyException(e));
+ System.err.println("testPartition() failed.");
+ throw e;
+ }
+ }
+
+ @Test
+ public void testRenamePartition() throws Throwable {
+
+ try {
+ String dbName = "compdb1";
+ String tblName = "comptbl1";
+ List<String> vals = new ArrayList<>(2);
+ vals.add("2011-07-11");
+ vals.add("8");
+ String part_path = "/ds=2011-07-11/hr=8";
+ List<String> tmp_vals = new ArrayList<>(2);
+ tmp_vals.add("tmp_2011-07-11");
+ tmp_vals.add("-8");
+ String part2_path = "/ds=tmp_2011-07-11/hr=-8";
+
+ client.dropTable(dbName, tblName);
+ silentDropDatabase(dbName);
+ new DatabaseBuilder()
+ .setName(dbName)
+ .setDescription("Rename Partition Test database")
+ .create(client, conf);
+
+ Table tbl = new TableBuilder()
+ .setDbName(dbName)
+ .setTableName(tblName)
+ .addCol("name", ColumnType.STRING_TYPE_NAME)
+ .addCol("income", ColumnType.INT_TYPE_NAME)
+ .addPartCol("ds", ColumnType.STRING_TYPE_NAME)
+ .addPartCol("hr", ColumnType.INT_TYPE_NAME)
+ .create(client, conf);
+
+ if (isThriftClient) {
+ // the createTable() above does not update the location in the 'tbl'
+ // object when the client is a thrift client and the code below relies
+ // on the location being present in the 'tbl' object - so get the table
+ // from the metastore
+ tbl = client.getTable(dbName, tblName);
+ }
+
+ Partition part = new Partition();
+ part.setDbName(dbName);
+ part.setTableName(tblName);
+ part.setValues(vals);
+ part.setParameters(new HashMap<>());
+ part.setSd(tbl.getSd().deepCopy());
+ part.getSd().setSerdeInfo(tbl.getSd().getSerdeInfo());
+ part.getSd().setLocation(tbl.getSd().getLocation() + "/part1");
+ part.getParameters().put("retention", "10");
+ part.getSd().setNumBuckets(12);
+ part.getSd().getSerdeInfo().getParameters().put("abc", "1");
+
+ client.add_partition(part);
+
+ part.setValues(tmp_vals);
+ client.renamePartition(dbName, tblName, vals, part);
+
+ boolean exceptionThrown = false;
+ try {
+ Partition p = client.getPartition(dbName, tblName, vals);
+ } catch(Exception e) {
+ assertEquals("partition should not have existed",
+ NoSuchObjectException.class, e.getClass());
+ exceptionThrown = true;
+ }
+ assertTrue("Expected NoSuchObjectException", exceptionThrown);
+
+ Partition part3 = client.getPartition(dbName, tblName, tmp_vals);
+ assertEquals("couldn't rename partition", part3.getParameters().get(
+ "retention"), "10");
+ assertEquals("couldn't rename partition", part3.getSd().getSerdeInfo()
+ .getParameters().get("abc"), "1");
+ assertEquals("couldn't rename partition", part3.getSd().getNumBuckets(),
+ 12);
+ assertEquals("new partition sd matches", part3.getSd().getLocation(),
+ tbl.getSd().getLocation() + part2_path);
+
+ part.setValues(vals);
+ client.renamePartition(dbName, tblName, tmp_vals, part);
+
+ exceptionThrown = false;
+ try {
+ Partition p = client.getPartition(dbName, tblName, tmp_vals);
+ } catch(Exception e) {
+ assertEquals("partition should not have existed",
+ NoSuchObjectException.class, e.getClass());
+ exceptionThrown = true;
+ }
+ assertTrue("Expected NoSuchObjectException", exceptionThrown);
+
+ part3 = client.getPartition(dbName, tblName, vals);
+ assertEquals("couldn't rename partition", part3.getParameters().get(
+ "retention"), "10");
+ assertEquals("couldn't rename partition", part3.getSd().getSerdeInfo()
+ .getParameters().get("abc"), "1");
+ assertEquals("couldn't rename partition", part3.getSd().getNumBuckets(),
+ 12);
+ assertEquals("new partition sd matches", part3.getSd().getLocation(),
+ tbl.getSd().getLocation() + part_path);
+
+ client.dropTable(dbName, tblName);
+
+ client.dropDatabase(dbName);
+ } catch (Exception e) {
+ System.err.println(StringUtils.stringifyException(e));
+ System.err.println("testRenamePartition() failed.");
+ throw e;
+ }
+ }
+
+ @Test
+ public void testDatabase() throws Throwable {
+ try {
+ // clear up any existing databases
+ silentDropDatabase(TEST_DB1_NAME);
+ silentDropDatabase(TEST_DB2_NAME);
+
+ Database db = new DatabaseBuilder()
+ .setName(TEST_DB1_NAME)
+ .setOwnerName(SecurityUtils.getUser())
+ .build(conf);
+ Assert.assertEquals(SecurityUtils.getUser(), db.getOwnerName());
+ client.createDatabase(db);
+
+ db = client.getDatabase(TEST_DB1_NAME);
+
+ assertEquals("name of returned db is different from that of inserted db",
+ TEST_DB1_NAME, db.getName());
+ assertEquals("location of the returned db is different from that of inserted db",
+ warehouse.getDatabasePath(db).toString(), db.getLocationUri());
+ assertEquals(db.getOwnerName(), SecurityUtils.getUser());
+ assertEquals(db.getOwnerType(), PrincipalType.USER);
+ assertEquals(Warehouse.DEFAULT_CATALOG_NAME, db.getCatalogName());
+ Database db2 = new DatabaseBuilder()
+ .setName(TEST_DB2_NAME)
+ .create(client, conf);
+
+ db2 = client.getDatabase(TEST_DB2_NAME);
+
+ assertEquals("name of returned db is different from that of inserted db",
+ TEST_DB2_NAME, db2.getName());
+ assertEquals("location of the returned db is different from that of inserted db",
+ warehouse.getDatabasePath(db2).toString(), db2.getLocationUri());
+
+ List<String> dbs = client.getDatabases(".*");
+
+ assertTrue("first database is not " + TEST_DB1_NAME, dbs.contains(TEST_DB1_NAME));
+ assertTrue("second database is not " + TEST_DB2_NAME, dbs.contains(TEST_DB2_NAME));
+
+ client.dropDatabase(TEST_DB1_NAME);
+ client.dropDatabase(TEST_DB2_NAME);
+ silentDropDatabase(TEST_DB1_NAME);
+ silentDropDatabase(TEST_DB2_NAME);
+ } catch (Throwable e) {
+ System.err.println(StringUtils.stringifyException(e));
+ System.err.println("testDatabase() failed.");
+ throw e;
+ }
+ }
+
+ @Test
+ public void testDatabaseLocationWithPermissionProblems() throws Exception {
+
+ // Note: The following test will fail if you are running this test as root. Setting
+ // permission to '0' on the database folder will not preclude root from being able
+ // to create the necessary files.
+
+ if (System.getProperty("user.name").equals("root")) {
+ System.err.println("Skipping test because you are running as root!");
+ return;
+ }
+
+ silentDropDatabase(TEST_DB1_NAME);
+
+ String dbLocation =
+ MetastoreConf.getVar(conf, ConfVars.WAREHOUSE) + "/test/_testDB_create_";
+ FileSystem fs = FileSystem.get(new Path(dbLocation).toUri(), conf);
+ fs.mkdirs(
+ new Path(MetastoreConf.getVar(conf, ConfVars.WAREHOUSE) + "/test"),
+ new FsPermission((short) 0));
+ Database db = new DatabaseBuilder()
+ .setName(TEST_DB1_NAME)
+ .setLocation(dbLocation)
+ .build(conf);
+
+
+ boolean createFailed = false;
+ try {
+ client.createDatabase(db);
+ } catch (MetaException cantCreateDB) {
+ createFailed = true;
+ } finally {
+ // Cleanup
+ if (!createFailed) {
+ try {
+ client.dropDatabase(TEST_DB1_NAME);
+ } catch(Exception e) {
+ System.err.println("Failed to remove database in cleanup: " + e.getMessage());
+ }
+ }
+
+ fs.setPermission(new Path(MetastoreConf.getVar(conf, ConfVars.WAREHOUSE) + "/test"),
+ new FsPermission((short) 755));
+ fs.delete(new Path(MetastoreConf.getVar(conf, ConfVars.WAREHOUSE) + "/test"), true);
+ }
+
+ assertTrue("Database creation succeeded even with permission problem", createFailed);
+ }
+
+ @Test
+ public void testDatabaseLocation() throws Throwable {
+ try {
+ // clear up any existing databases
+ silentDropDatabase(TEST_DB1_NAME);
+
+ String dbLocation =
+ MetastoreConf.getVar(conf, ConfVars.WAREHOUSE) + "/_testDB_create_";
+ new DatabaseBuilder()
+ .setName(TEST_DB1_NAME)
+ .setLocation(dbLocation)
+ .create(client, conf);
+
+ Database db = client.getDatabase(TEST_DB1_NAME);
+
+ assertEquals("name of returned db is different from that of inserted db",
+ TEST_DB1_NAME, db.getName());
+ assertEquals("location of the returned db is different from that of inserted db",
+ warehouse.getDnsPath(new Path(dbLocation)).toString(), db.getLocationUri());
+
+ client.dropDatabase(TEST_DB1_NAME);
+ silentDropDatabase(TEST_DB1_NAME);
+
+ boolean objectNotExist = false;
+ try {
+ client.getDatabase(TEST_DB1_NAME);
+ } catch (NoSuchObjectException e) {
+ objectNotExist = true;
+ }
+ assertTrue("Database " + TEST_DB1_NAME + " exists ", objectNotExist);
+
+ dbLocation =
+ MetastoreConf.getVar(conf, ConfVars.WAREHOUSE) + "/_testDB_file_";
+ FileSystem fs = FileSystem.get(new Path(dbLocation).toUri(), conf);
+ fs.createNewFile(new Path(dbLocation));
+ fs.deleteOnExit(new Path(dbLocation));
+ db = new DatabaseBuilder()
+ .setName(TEST_DB1_NAME)
+ .setLocation(dbLocation)
+ .build(conf);
+
+ boolean createFailed = false;
+ try {
+ client.createDatabase(db);
+ } catch (MetaException cantCreateDB) {
+ System.err.println(cantCreateDB.getMessage());
+ createFailed = true;
+ }
+ assertTrue("Database creation succeeded even location exists and is a file", createFailed);
+
+ objectNotExist = false;
+ try {
+ client.getDatabase(TEST_DB1_NAME);
+ } catch (NoSuchObjectException e) {
+ objectNotExist = true;
+ }
+ assertTrue("Database " + TEST_DB1_NAME + " exists when location is specified and is a file",
+ objectNotExist);
+
+ } catch (Throwable e) {
+ System.err.println(StringUtils.stringifyException(e));
+ System.err.println("testDatabaseLocation() failed.");
+ throw e;
+ }
+ }
+
+
+ @Test
+ public void testSimpleTypeApi() throws Exception {
+ try {
+ client.dropType(ColumnType.INT_TYPE_NAME);
+
+ Type typ1 = new Type();
+ typ1.setName(ColumnType.INT_TYPE_NAME);
+ boolean ret = client.createType(typ1);
+ assertTrue("Unable to create type", ret);
+
+ Type typ1_2 = client.getType(ColumnType.INT_TYPE_NAME);
+ assertNotNull(typ1_2);
+ assertEquals(typ1.getName(), typ1_2.getName());
+
+ ret = client.dropType(ColumnType.INT_TYPE_NAME);
+ assertTrue("unable to drop type integer", ret);
+
+ boolean exceptionThrown = false;
+ try {
+ client.getType(ColumnType.INT_TYPE_NAME);
+ } catch (NoSuchObjectException e) {
+ exceptionThrown = true;
+ }
+ assertTrue("Expected NoSuchObjectException", exceptionThrown);
+ } catch (Exception e) {
+ System.err.println(StringUtils.stringifyException(e));
+ System.err.println("testSimpleTypeApi() failed.");
+ throw e;
+ }
+ }
+
+ // TODO:pc need to enhance this with complex fields and getType_all function
+ @Test
+ public void testComplexTypeApi() throws Exception {
+ try {
+ client.dropType("Person");
+
+ Type typ1 = new Type();
+ typ1.setName("Person");
+ typ1.setFields(new ArrayList<>(2));
+ typ1.getFields().add(
+ new FieldSchema("name", ColumnType.STRING_TYPE_NAME, ""));
+ typ1.getFields().add(
+ new FieldSchema("income", ColumnType.INT_TYPE_NAME, ""));
+ boolean ret = client.createType(typ1);
+ assertTrue("Unable to create type", ret);
+
+ Type typ1_2 = client.getType("Person");
+ assertNotNull("type Person not found", typ1_2);
+ assertEquals(typ1.getName(), typ1_2.getName());
+ assertEquals(typ1.getFields().size(), typ1_2.getFields().size());
+ assertEquals(typ1.getFields().get(0), typ1_2.getFields().get(0));
+ assertEquals(typ1.getFields().get(1), typ1_2.getFields().get(1));
+
+ client.dropType("Family");
+
+ Type fam = new Type();
+ fam.setName("Family");
+ fam.setFields(new ArrayList<>(2));
+ fam.getFields().add(
+ new FieldSchema("name", ColumnType.STRING_TYPE_NAME, ""));
+ fam.getFields().add(
+ new FieldSchema("members",
+ ColumnType.getListType(typ1.getName()), ""));
+
+ ret = client.createType(fam);
+ assertTrue("Unable to create type " + fam.getName(), ret);
+
+ Type fam2 = client.getType("Family");
+ assertNotNull("type Person not found", fam2);
+ assertEquals(fam.getName(), fam2.getName());
+ assertEquals(fam.getFields().size(), fam2.getFields().size());
+ assertEquals(fam.getFields().get(0), fam2.getFields().get(0));
+ assertEquals(fam.getFields().get(1), fam2.getFields().get(1));
+
+ ret = client.dropType("Family");
+ assertTrue("unable to drop type Family", ret);
+
+ ret = client.dropType("Person");
+ assertTrue("unable to drop type Person", ret);
+
+ boolean exceptionThrown = false;
+ try {
+ client.getType("Person");
+ } catch (NoSuchObjectException e) {
+ exceptionThrown = true;
+ }
+ assertTrue("Expected NoSuchObjectException", exceptionThrown);
+ } catch (Exception e) {
+ System.err.println(StringUtils.stringifyException(e));
+ System.err.println("testComplexTypeApi() failed.");
+ throw e;
+ }
+ }
+
+ @Test
+ public void testSimpleTable() throws Exception {
+ try {
+ String dbName = "simpdb";
+ String tblName = "simptbl";
+ String tblName2 = "simptbl2";
+ String typeName = "Person";
+
+ client.dropTable(dbName, tblName);
+ silentDropDatabase(dbName);
+
+ new DatabaseBuilder()
+ .setName(dbName)
+ .create(client, conf);
+
+ client.dropType(typeName);
+ Type typ1 = new Type();
+ typ1.setName(typeName);
+ typ1.setFields(new ArrayList<>(2));
+ typ1.getFields().add(
+ new FieldSchema("name", ColumnType.STRING_TYPE_NAME, ""));
+ typ1.getFields().add(
+ new FieldSchema("income", ColumnType.INT_TYPE_NAME, ""));
+ client.createType(typ1);
+
+ Table tbl = new TableBuilder()
+ .setDbName(dbName)
+ .setTableName(tblName)
+ .setCols(typ1.getFields())
+ .setNumBuckets(1)
+ .addBucketCol("name")
+ .addStorageDescriptorParam("test_param_1", "Use this for comments etc")
+ .create(client, conf);
+
+ if (isThriftClient) {
+ // the createTable() above does not update the location in the 'tbl'
+ // object when the client is a thrift client and the code below relies
+ // on the location being present in the 'tbl' object - so get the table
+ // from the metastore
+ tbl = client.getTable(dbName, tblName);
+ }
+
+ Table tbl2 = client.getTable(dbName, tblName);
+ assertNotNull(tbl2);
+ assertEquals(tbl2.getDbName(), dbName);
+ assertEquals(tbl2.getTableName(), tblName);
+ assertEquals(tbl2.getSd().getCols().size(), typ1.getFields().size());
+ assertEquals(tbl2.getSd().isCompressed(), false);
+ assertEquals(tbl2.getSd().getNumBuckets(), 1);
+ assertEquals(tbl2.getSd().getLocation(), tbl.getSd().getLocation());
+ assertNotNull(tbl2.getSd().getSerdeInfo());
+ tbl.getSd().getSerdeInfo().setParameters(new HashMap<>());
+ tbl.getSd().getSerdeInfo().getParameters().put(ColumnType.SERIALIZATION_FORMAT, "1");
+
+ tbl2.setTableName(tblName2);
+ tbl2.setParameters(new HashMap<>());
+ tbl2.getParameters().put("EXTERNAL", "TRUE");
+ tbl2.getSd().setLocation(tbl.getSd().getLocation() + "-2");
+
+ List<FieldSchema> fieldSchemas = client.getFields(dbName, tblName);
+ assertNotNull(fieldSchemas);
+ assertEquals(fieldSchemas.size(), tbl.getSd().getCols().size());
+ for (FieldSchema fs : tbl.getSd().getCols()) {
+ assertTrue(fieldSchemas.contains(fs));
+ }
+
+ List<FieldSchema> fieldSchemasFull = client.getSchema(dbName, tblName);
+ assertNotNull(fieldSchemasFull);
+ assertEquals(fieldSchemasFull.size(), tbl.getSd().getCols().size()
+ + tbl.getPartitionKeys().size());
+ for (FieldSchema fs : tbl.getSd().getCols()) {
+ assertTrue(fieldSchemasFull.contains(fs));
+ }
+ for (FieldSchema fs : tbl.getPartitionKeys()) {
+ assertTrue(fieldSchemasFull.contains(fs));
+ }
+
+ client.createTable(tbl2);
+ if (isThriftClient) {
+ tbl2 = client.getTable(tbl2.getDbName(), tbl2.getTableName());
+ }
+
+ Table tbl3 = client.getTable(dbName, tblName2);
+ assertNotNull(tbl3);
+ assertEquals(tbl3.getDbName(), dbName);
+ assertEquals(tbl3.getTableName(), tblName2);
+ assertEquals(tbl3.getSd().getCols().size(), typ1.getFields().size());
+ assertEquals(tbl3.getSd().isCompressed(), false);
+ assertEquals(tbl3.getSd().getNumBuckets(), 1);
+ assertEquals(tbl3.getSd().getLocation(), tbl2.getSd().getLocation());
+ assertEquals(tbl3.getParameters(), tbl2.getParameters());
+
+ fieldSchemas = client.getFields(dbName, tblName2);
+ assertNotNull(fieldSchemas);
+ assertEquals(fieldSchemas.size(), tbl2.getSd().getCols().size());
+ for (FieldSchema fs : tbl2.getSd().getCols()) {
+ assertTrue(fieldSchemas.contains(fs));
+ }
+
+ fieldSchemasFull = client.getSchema(dbName, tblName2);
+ assertNotNull(fieldSchemasFull);
+ assertEquals(fieldSchemasFull.size(), tbl2.getSd().getCols().size()
+ + tbl2.getPartitionKeys().size());
+ for (FieldSchema fs : tbl2.getSd().getCols()) {
+ assertTrue(fieldSchemasFull.contains(fs));
+ }
+ for (FieldSchema fs : tbl2.getPartitionKeys()) {
+ assertTrue(fieldSchemasFull.contains(fs));
+ }
+
+ assertEquals("Use this for comments etc", tbl2.getSd().getParameters()
+ .get("test_param_1"));
+ assertEquals("name", tbl2.getSd().getBucketCols().get(0));
+ assertTrue("Partition key list is not empty",
+ (tbl2.getPartitionKeys() == null)
+ || (tbl2.getPartitionKeys().size() == 0));
+
+ //test get_table_objects_by_name functionality
+ ArrayList<String> tableNames = new ArrayList<>();
+ tableNames.add(tblName2);
+ tableNames.add(tblName);
+ tableNames.add(tblName2);
+ List<Table> foundTables = client.getTableObjectsByName(dbName, tableNames);
+
+ assertEquals(2, foundTables.size());
+ for (Table t: foundTables) {
+ if (t.getTableName().equals(tblName2)) {
+ assertEquals(t.getSd().getLocation(), tbl2.getSd().getLocation());
+ } else {
+ assertEquals(t.getTableName(), tblName);
+ assertEquals(t.getSd().getLocation(), tbl.getSd().getLocation());
+ }
+ assertEquals(t.getSd().getCols().size(), typ1.getFields().size());
+ assertEquals(t.getSd().isCompressed(), false);
+ assertEquals(foundTables.get(0).getSd().getNumBuckets(), 1);
+ assertNotNull(t.getSd().getSerdeInfo());
+ assertEquals(t.getDbName(), dbName);
+ }
+
+ tableNames.add(1, "table_that_doesnt_exist");
+ foundTables = client.getTableObjectsByName(dbName, tableNames);
+ assertEquals(foundTables.size(), 2);
+
+ InvalidOperationException ioe = null;
+ try {
+ foundTables = client.getTableObjectsByName(dbName, null);
+ } catch (InvalidOperationException e) {
+ ioe = e;
+ }
+ assertNotNull(ioe);
+ assertTrue("Table not found", ioe.getMessage().contains("null tables"));
+
+ UnknownDBException udbe = null;
+ try {
+ foundTables = client.getTableObjectsByName("db_that_doesnt_exist", tableNames);
+ } catch (UnknownDBException e) {
+ udbe = e;
+ }
+ assertNotNull(udbe);
+ assertTrue("DB not found",
+ udbe.getMessage().contains("not find database hive.db_that_doesnt_exist"));
+
+ udbe = null;
+ try {
+ foundTables = client.getTableObjectsByName("", tableNames);
+ } catch (UnknownDBException e) {
+ udbe = e;
+ }
+ assertNotNull(udbe);
+ assertTrue("DB not found", udbe.getMessage().contains("is null or empty"));
+
+ FileSystem fs = FileSystem.get((new Path(tbl.getSd().getLocation())).toUri(), conf);
+ client.dropTable(dbName, tblName);
+ assertFalse(fs.exists(new Path(tbl.getSd().getLocation())));
+
+ client.dropTable(dbName, tblName2);
+ assertTrue(fs.exists(new Path(tbl2.getSd().getLocation())));
+
+ client.dropType(typeName);
+ client.dropDatabase(dbName);
+ } catch (Exception e) {
+ System.err.println(StringUtils.stringifyException(e));
+ System.err.println("testSimpleTable() failed.");
+ throw e;
+ }
+ }
+
+ // Tests that in the absence of stats for partitions, and/or absence of columns
+ // to get stats for, the metastore does not break. See HIVE-12083 for motivation.
+ @Test
+ public void testStatsFastTrivial() throws Throwable {
+ String dbName = "tstatsfast";
+ String tblName = "t1";
+ String tblOwner = "statstester";
+ String typeName = "Person";
+ int lastAccessed = 12083;
+
+ cleanUp(dbName,tblName,typeName);
+
+ List<List<String>> values = new ArrayList<>();
+ values.add(makeVals("2008-07-01 14:13:12", "14"));
+ values.add(makeVals("2008-07-01 14:13:12", "15"));
+ values.add(makeVals("2008-07-02 14:13:12", "15"));
+ values.add(makeVals("2008-07-03 14:13:12", "151"));
+
+ createMultiPartitionTableSchema(dbName, tblName, typeName, values);
+
+ List<String> emptyColNames = new ArrayList<>();
+ List<String> emptyPartNames = new ArrayList<>();
+
+ List<String> colNames = new ArrayList<>();
+ colNames.add("name");
+ colNames.add("income");
+ List<String> partNames = client.listPartitionNames(dbName,tblName,(short)-1);
+
+ assertEquals(0,emptyColNames.size());
+ assertEquals(0,emptyPartNames.size());
+ assertEquals(2,colNames.size());
+ assertEquals(4,partNames.size());
+
+ // Test for both colNames and partNames being empty:
+ AggrStats aggrStatsEmpty = client.getAggrColStatsFor(dbName,tblName,emptyColNames,emptyPartNames);
+ assertNotNull(aggrStatsEmpty); // short-circuited on client-side, verifying that it's an empty object, not null
+ assertEquals(0,aggrStatsEmpty.getPartsFound());
+ assertNotNull(aggrStatsEmpty.getColStats());
+ assert(aggrStatsEmpty.getColStats().isEmpty());
+
+ // Test for only colNames being empty
+ AggrStats aggrStatsOnlyParts = client.getAggrColStatsFor(dbName,tblName,emptyColNames,partNames);
+ assertNotNull(aggrStatsOnlyParts); // short-circuited on client-side, verifying that it's an empty object, not null
+ assertEquals(0,aggrStatsOnlyParts.getPartsFound());
+ assertNotNull(aggrStatsOnlyParts.getColStats());
+ assert(aggrStatsOnlyParts.getColStats().isEmpty());
+
+ // Test for only partNames being empty
+ AggrStats aggrStatsOnlyCols = client.getAggrColStatsFor(dbName,tblName,colNames,emptyPartNames);
+ assertNotNull(aggrStatsOnlyCols); // short-circuited on client-side, verifying that it's an empty object, not null
+ assertEquals(0,aggrStatsOnlyCols.getPartsFound());
+ assertNotNull(aggrStatsOnlyCols.getColStats());
+ assert(aggrStatsOnlyCols.getColStats().isEmpty());
+
+ // Test for valid values for both.
+ AggrStats aggrStatsFull = client.getAggrColStatsFor(dbName,tblName,colNames,partNames);
+ assertNotNull(aggrStatsFull);
+ assertEquals(0,aggrStatsFull.getPartsFound()); // would still be empty, because no stats are actually populated.
+ assertNotNull(aggrStatsFull.getColStats());
+ assert(aggrStatsFull.getColStats().isEmpty());
+
+ }
+
+ @Test
+ public void testColumnStatistics() throws Throwable {
+
+ String dbName = "columnstatstestdb";
+ String tblName = "tbl";
+ String typeName = "Person";
+ String tblOwner = "testowner";
+ int lastAccessed = 6796;
+
+ try {
+ cleanUp(dbName, tblName, typeName);
+ new DatabaseBuilder()
+ .setName(dbName)
+ .create(client, conf);
+ createTableForTestFilter(dbName,tblName, tblOwner, lastAccessed, true);
+
+ // Create a ColumnStatistics Obj
+ String[] colName = new String[]{"income", "name"};
+ double lowValue = 50000.21;
+ double highValue = 1200000.4525;
+ long numNulls = 3;
+ long numDVs = 22;
+ double avgColLen = 50.30;
+ long maxColLen = 102;
+ String[] colType = new String[] {"double", "string"};
+ boolean isTblLevel = true;
+ String partName = null;
+ List<ColumnStatisticsObj> statsObjs = new ArrayList<>();
+
+ ColumnStatisticsDesc statsDesc = new ColumnStatisticsDesc();
+ statsDesc.setDbName(dbName);
+ statsDesc.setTableName(tblName);
+ statsDesc.setIsTblLevel(isTblLevel);
+ statsDesc.setPartName(partName);
+
+ ColumnStatisticsObj statsObj = new ColumnStatisticsObj();
+ statsObj.setColName(colName[0]);
+ statsObj.setColType(colType[0]);
+
+ ColumnStatisticsData statsData = new ColumnStatisticsData();
+ DoubleColumnStatsData numericStats = new DoubleColumnStatsData();
+ statsData.setDoubleStats(numericStats);
+
+ statsData.getDoubleStats().setHighValue(highValue);
+ statsData.getDoubleStats().setLowValue(lowValue);
+ statsData.getDoubleStats().setNumDVs(numDVs);
+ statsData.getDoubleStats().setNumNulls(numNulls);
+
+ statsObj.setStatsData(statsData);
+ statsObjs.add(statsObj);
+
+ statsObj = new ColumnStatisticsObj();
+ statsObj.setColName(colName[1]);
+ statsObj.setColType(colType[1]);
+
+ statsData = new ColumnStatisticsData();
+ StringColumnStatsData stringStats = new StringColumnStatsData();
+ statsData.setStringStats(stringStats);
+ statsData.getStringStats().setAvgColLen(avgColLen);
+ statsData.getStringStats().setMaxColLen(maxColLen);
+ statsData.getStringStats().setNumDVs(numDVs);
+ statsData.getStringStats().setNumNulls(numNulls);
+
+ statsObj.setStatsData(statsData);
+ statsObjs.add(statsObj);
+
+ ColumnStatistics colStats = new ColumnStatistics();
+ colStats.setStatsDesc(statsDesc);
+ colStats.setStatsObj(statsObjs);
+
+ // write stats objs persistently
+ client.updateTableColumnStatistics(colStats);
+
+ // retrieve the stats obj that was just written
+ ColumnStatisticsObj colStats2 = client.getTableColumnStatistics(
+ dbName, tblName, Lists.newArrayList(colName[0])).get(0);
+
+ // compare stats obj to ensure what we get is what we wrote
+ assertNotNull(colStats2);
+ assertEquals(colStats2.getColName(), colName[0]);
+ assertEquals(colStats2.getStatsData().getDoubleStats().getLowValue(), lowValue, 0.01);
+ assertEquals(colStats2.getStatsData().getDoubleStats().getHighValue(), highValue, 0.01);
+ assertEquals(colStats2.getStatsData().getDoubleStats().getNumNulls(), numNulls);
+ assertEquals(colStats2.getStatsData().getDoubleStats().getNumDVs(), numDVs);
+
+ // test delete column stats; if no col name is passed all column stats associated with the
+ // table is deleted
+ boolean status = client.deleteTableColumnStatistics(dbName, tblName, null);
+ assertTrue(status);
+ // try to query stats for a column for which stats doesn't exist
+ assertTrue(client.getTableColumnStatistics(
+ dbName, tblName, Lists.newArrayList(colName[1])).isEmpty());
+
+ colStats.setStatsDesc(statsDesc);
+ colStats.setStatsObj(statsObjs);
+
+ // update table level column stats
+ client.updateTableColumnStatistics(colStats);
+
+ // query column stats for column whose stats were updated in the previous call
+ colStats2 = client.getTableColumnStatistics(
+ dbName, tblName, Lists.newArrayList(colName[0])).get(0);
+
+ // partition level column statistics test
+ // create a table with multiple partitions
+ cleanUp(dbName, tblName, typeName);
+
+ List<List<String>> values = new ArrayList<>();
+ values.add(makeVals("2008-07-01 14:13:12", "14"));
+ values.add(makeVals("2008-07-01 14:13:12", "15"));
+ values.add(makeVals("2008-07-02 14:13:12", "15"));
+ values.add(makeVals("2008-07-03 14:13:12", "151"));
+
+ createMultiPartitionTableSchema(dbName, tblName, typeName, values);
+
+ List<String> partitions = client.listPartitionNames(dbName, tblName, (short)-1);
+
+ partName = partitions.get(0);
+ isTblLevel = false;
+
+ // create a new columnstatistics desc to represent partition level column stats
+ statsDesc = new ColumnStatisticsDesc();
+ statsDesc.setDbName(dbName);
+ statsDesc.setTableName(tblName);
+ statsDesc.setPartName(partName);
+ statsDesc.setIsTblLevel(isTblLevel);
+
+ colStats = new ColumnStatistics();
+ colStats.setStatsDesc(statsDesc);
+ colStats.setStatsObj(statsObjs);
+
+ client.updatePartitionColumnStatistics(colStats);
+
+ colStats2 = client.getPartitionColumnStatistics(dbName, tblName,
+ Lists.newArrayList(partName), Lists.newArrayList(colName[1])).get(partName).get(0);
+
+ // compare stats obj to ensure what we get is what we wrote
+ assertNotNull(colStats2);
+ assertEquals(colStats.getStatsDesc().getPartName(), partName);
+ assertEquals(colStats2.getColName(), colName[1]);
+ assertEquals(colStats2.getStatsData().getStringStats().getMaxColLen(), maxColLen);
+ assertEquals(colStats2.getStatsData().getStringStats().getAvgColLen(), avgColLen, 0.01);
+ assertEquals(colStats2.getStatsData().getStringStats().getNumNulls(), numNulls);
+ assertEquals(colStats2.getStatsData().getStringStats().getNumDVs(), numDVs);
+
+ // test stats deletion at partition level
+ client.deletePartitionColumnStatistics(dbName, tblName, partName, colName[1]);
+
+ colStats2 = client.getPartitionColumnStatistics(dbName, tblName,
+ Lists.newArrayList(partName), Lists.newArrayList(colName[0])).get(partName).get(0);
+
+ // test get stats on a column for which stats doesn't exist
+ assertTrue(client.getPartitionColumnStatistics(dbName, tblName,
+ Lists.newArrayList(partName), Lists.newArrayList(colName[1])).isEmpty());
+ } catch (Exception e) {
+ System.err.println(StringUtils.stringifyException(e));
+ System.err.println("testColumnStatistics() failed.");
+ throw e;
+ } finally {
+ cleanUp(dbName, tblName, typeName);
+ }
+ }
+
+ @Test(expected = MetaException.class)
+ public void testGetSchemaWithNoClassDefFoundError() throws TException {
+ String dbName = "testDb";
+ String tblName = "testTable";
+
+ client.dropTable(dbName, tblName);
+ silentDropDatabase(dbName);
+
+ new DatabaseBuilder()
+ .setName(dbName)
+ .create(client, conf);
+
+ Table tbl = new TableBuilder()
+ .setDbName(dbName)
+ .setTableName(tblName)
+ .addCol("name", ColumnType.STRING_TYPE_NAME, "")
+ .setSerdeLib("no.such.class")
+ .create(client, conf);
+
+ client.getSchema(dbName, tblName);
+ }
+
+ @Test
+ public void testAlterTable() throws Exception {
+ String dbName = "alterdb";
+ String invTblName = "alter-tbl";
+ String tblName = "altertbl";
+
+ try {
+ client.dropTable(dbName, tblName);
+ silentDropDatabase(dbName);
+
+ new DatabaseBuilder()
+ .setName(dbName)
+ .create(client, conf);
+
+ ArrayList<FieldSchema> invCols = new ArrayList<>(2);
+ invCols.add(new FieldSchema("n-ame", ColumnType.STRING_TYPE_NAME, ""));
+ invCols.add(new FieldSchema("in.come", ColumnType.INT_TYPE_NAME, ""));
+
+ Table tbl = new TableBuilder()
+ .setDbName(dbName)
+ .setTableName(invTblName)
+ .setCols(invCols)
+ .build(conf);
+
+ boolean failed = false;
+ try {
+ client.createTable(tbl);
+ } catch (InvalidObjectException ex) {
+ failed = true;
+ }
+ if (!failed) {
+ assertTrue("Able to create table with invalid name: " + invTblName,
+ false);
+ }
+
+ // create an invalid table which has wrong column type
+ ArrayList<FieldSchema> invColsInvType = new ArrayList<>(2);
+ invColsInvType.add(new FieldSchema("name", ColumnType.STRING_TYPE_NAME, ""));
+ invColsInvType.add(new FieldSchema("income", "xyz", ""));
+ tbl.setTableName(tblName);
+ tbl.getSd().setCols(invColsInvType);
+ boolean failChecker = false;
+ try {
+ client.createTable(tbl);
+ } catch (InvalidObjectException ex) {
+ failChecker = true;
+ }
+ if (!failChecker) {
+ assertTrue("Able to create table with invalid column type: " + invTblName,
+ false);
+ }
+
+ ArrayList<FieldSchema> cols = new ArrayList<>(2);
+ cols.add(new FieldSchema("name", ColumnType.STRING_TYPE_NAME, ""));
+ cols.add(new FieldSchema("income", ColumnType.INT_TYPE_NAME, ""));
+
+ // create a valid table
+ tbl.setTableName(tblName);
+ tbl.getSd().setCols(cols);
+ client.createTable(tbl);
+
+ if (isThriftClient) {
+ tbl = client.getTable(tbl.getDbName(), tbl.getTableName());
+ }
+
+ // now try to invalid alter table
+ Table tbl2 = client.getTable(dbName, tblName);
+ failed = false;
+ try {
+ tbl2.setTableName(invTblName);
+ tbl2.getSd().setCols(invCols);
+ client.alter_table(dbName, tblName, tbl2);
+ } catch (InvalidOperationException ex) {
+ failed = true;
+ }
+ if (!failed) {
+ assertTrue("Able to rename table with invalid name: " + invTblName,
+ false);
+ }
+
+ //try an invalid alter table with partition key name
+ Table tbl_pk = client.getTable(tbl.getDbName(), tbl.getTableName());
+ List<FieldSchema> partitionKeys = tbl_pk.getPartitionKeys();
+ for (FieldSchema fs : partitionKeys) {
+ fs.setName("invalid_to_change_name");
+ fs.setComment("can_change_comment");
+ }
+ tbl_pk.setPartitionKeys(partitionKeys);
+ try {
+ client.alter_table(dbName, tblName, tbl_pk);
+ } catch (InvalidOperationException ex) {
+ failed = true;
+ }
+ assertTrue("Should not have succeeded in altering partition key name", failed);
+
+ //try a valid alter table partition key comment
+ failed = false;
+ tbl_pk = client.getTable(tbl.getDbName(), tbl.getTableName());
+ partitionKeys = tbl_pk.getPartitionKeys();
+ for (FieldSchema fs : partitionKeys) {
+ fs.setComment("can_change_comment");
+ }
+ tbl_pk.setPartitionKeys(partitionKeys);
+ try {
+ client.alter_table(dbName, tblName, tbl_pk);
+ } catch (InvalidOperationException ex) {
+ failed = true;
+ }
+ assertFalse("Should not have failed alter table partition comment", failed);
+ Table newT = client.getTable(tbl.getDbName(), tbl.getTableName());
+ assertEquals(partitionKeys, newT.getPartitionKeys());
+
+ // try a valid alter table
+ tbl2.setTableName(tblName + "_renamed");
+ tbl2.getSd().setCols(cols);
+ tbl2.getSd().setNumBuckets(32);
+ client.alter_table(dbName, tblName, tbl2);
+ Table tbl3 = client.getTable(dbName, tbl2.getTableName());
+ assertEquals("Alter table didn't succeed. Num buckets is different ",
+ tbl2.getSd().getNumBuckets(), tbl3.getSd().getNumBuckets());
+ // check that data has moved
+ FileSystem fs = FileSystem.get((new Path(tbl.getSd().getLocation())).toUri(), conf);
+ assertFalse("old table location still exists", fs.exists(new Path(tbl
+ .getSd().getLocation())));
+ assertTrue("data did not move to new location", fs.exists(new Path(tbl3
+ .getSd().getLocation())));
+
+ if (!isThriftClient) {
+ assertEquals("alter table didn't move data correct location", tbl3
+ .getSd().getLocation(), tbl2.getSd().getLocation());
+ }
+
+ // alter table with invalid column type
+ tbl_pk.getSd().setCols(invColsInvType);
+ failed = false;
+ try {
+ client.alter_table(dbName, tbl2.getTableName(), tbl_pk);
+ } catch (InvalidOperationException ex) {
+ failed = true;
+ }
+ assertTrue("Should not have succeeded in altering column", failed);
+
+ } catch (Exception e) {
+ System.err.println(StringUtils.stringifyException(e));
+ System.err.println("testSimpleTable() failed.");
+ throw e;
+ } finally {
+ silentDropDatabase(dbName);
+ }
+ }
+
+ @Test
+ public void testComplexTable() throws Exception {
+
+ String dbName = "compdb";
+ String tblName = "comptbl";
+ String typeName = "Person";
+
+ try {
+ client.dropTable(dbName, tblName);
+ silentDropDatabase(dbName);
+ new DatabaseBuilder()
+ .setName(dbName)
+ .create(client, conf);
+
+ client.dropType(typeName);
+ Type typ1 = new Type();
+ typ1.setName(typeName);
+ typ1.setFields(new ArrayList<>(2));
+ typ1.getFields().add(
+ new FieldSchema("name", ColumnType.STRING_TYPE_NAME, ""));
+ typ1.getFields().add(
+ new FieldSchema("income", ColumnType.INT_TYPE_NAME, ""));
+ client.createType(typ1);
+
+ Table tbl = new TableBuilder()
+ .setDbName(dbName)
+ .setTableName(tblName)
+ .setCols(typ1.getFields())
+ .addPartCol("ds", ColumnType.DATE_TYPE_NAME)
+ .addPartCol("hr", ColumnType.INT_TYPE_NAME)
+ .setNumBuckets(1)
+ .addBucketCol("name")
+ .addStorageDescriptorParam("test_param_1","Use this for comments etc")
+ .create(client, conf);
+
+ Table tbl2 = client.getTable(dbName, tblName);
+ assertEquals(tbl2.getDbName(), dbName);
+ assertEquals(tbl2.getTableName(), tblName);
+ assertEquals(tbl2.getSd().getCols().size(), typ1.getFields().size());
+ assertFalse(tbl2.getSd().isCompressed());
+ assertFalse(tbl2.getSd().isStoredAsSubDirectories());
+ assertEquals(tbl2.getSd().getNumBuckets(), 1);
+
+ assertEquals("Use this for comments etc", tbl2.getSd().getParameters()
+ .get("test_param_1"));
+ assertEquals("name", tbl2.getSd().getBucketCols().get(0));
+
+ assertNotNull(tbl2.getPartitionKeys());
+ assertEquals(2, tbl2.getPartitionKeys().size());
+ assertEquals(ColumnType.DATE_TYPE_NAME, tbl2.getPartitionKeys().get(0)
+ .getType());
+ assertEquals(ColumnType.INT_TYPE_NAME, tbl2.getPartitionKeys().get(1)
+ .getType());
+ assertEquals("ds", tbl2.getPartitionKeys().get(0).getName());
+ assertEquals("hr", tbl2.getPartitionKeys().get(1).getName());
+
+ List<FieldSchema> fieldSchemas = client.getFields(dbName, tblName);
+ assertNotNull(fieldSchemas);
+ assertEquals(fieldSchemas.size(), tbl.getSd().getCols().size());
+ for (FieldSchema fs : tbl.getSd().getCols()) {
+ assertTrue(fieldSchemas.contains(fs));
+ }
+
+ List<FieldSchema> fieldSchemasFull = client.getSchema(dbName, tblName);
+ assertNotNull(fieldSchemasFull);
+ assertEquals(fieldSchemasFull.size(), tbl.getSd().getCols().size()
+ + tbl.getPartitionKeys().size());
+ for (FieldSchema fs : tbl.getSd().getCols()) {
+ assertTrue(fieldSchemasFull.contains(fs));
+ }
+ for (FieldSchema fs : tbl.getPartitionKeys()) {
+ assertTrue(fieldSchemasFull.contains(fs));
+ }
+ } catch (Exception e) {
+ System.err.println(StringUtils.stringifyException(e));
+ System.err.println("testComplexTable() failed.");
+ throw e;
+ } finally {
+ client.dropTable(dbName, tblName);
+ boolean ret = client.dropType(typeName);
+ assertTrue("Unable to drop type " + typeName, ret);
+ client.dropDatabase(dbName);
+ }
+ }
+
+ @Test
+ public void testTableDatabase() throws Exception {
+ String dbName = "testDb";
+ String tblName_1 = "testTbl_1";
+ String tblName_2 = "testTbl_2";
+
+ try {
+ silentDropDatabase(dbName);
+
+ String dbLocation =
+ MetastoreConf.getVar(conf, ConfVars.WAREHOUSE) + "_testDB_table_create_";
+ new DatabaseBuilder()
+ .setName(dbName)
+ .setLocation(dbLocation)
+ .create(client, conf);
+ Database db = client.getDatabase(dbName);
+
+ Table tbl = new TableBuilder()
+ .setDbName(dbName)
+ .setTableName(tblName_1)
+ .addCol("name", ColumnType.STRING_TYPE_NAME)
+ .addCol("income", ColumnType.INT_TYPE_NAME)
+ .create(client, conf);
+
+ tbl = client.getTable(dbName, tblName_1);
+
+ Path path = new Path(tbl.getSd().getLocation());
+ System.err.println("Table's location " + path + ", Database's location " + db.getLocationUri());
+ assertEquals("Table location is not a subset of the database location",
+ path.getParent().toString(), db.getLocationUri());
+
+ } catch (Exception e) {
+ System.err.println(StringUtils.stringifyException(e));
+ System.err.println("testTableDatabase() failed.");
+ throw e;
+ } finally {
+ silentDropDatabase(dbName);
+ }
+ }
+
+
+ @Test
+ public void testGetConfigValue() {
+
+ String val = "value";
+
+ if (!isThriftClient) {
+ try {
+ assertEquals(client.getConfigValue("hive.key1", val), "value1");
+ assertEquals(client.getConfigValue("hive.key2", val), "http://www.example.com");
+ assertEquals(client.getConfigValue("hive.key3", val), "");
+ assertEquals(client.getConfigValue("hive.key4", val), "0");
+ assertEquals(client.getConfigValue("hive.key5", val), val);
+ assertEquals(client.getConfigValue(null, val), val);
+ } catch (TException e) {
+ e.printStackTrace();
+ fail();
+ }
+ }
+
+ boolean threwException = false;
+ try {
+ // Attempting to get the password should throw an exception
+ client.getConfigValue("javax.jdo.option.ConnectionPassword", "password");
+ } catch (ConfigValSecurityException e) {
+ threwException = true;
+ } catch (TException e) {
+ e.printStackTrace();
+ fail();
+ }
+ assert (threwException);
+ }
+
+ private static void adjust(HiveMetaStoreClient client, Partition part,
- String dbName, String tblName) throws TException {
++ String dbName, String tblName, boolean isThriftClient) throws TException {
+ Partition part_get = client.getPartition(dbName, tblName, part.getValues());
- part.setCreateTime(part_get.getCreateTime());
- part.putToParameters(org.apache.hadoop.hive.metastore.api.hive_metastoreConstants.DDL_TIME, Long.toString(part_get.getCreateTime()));
++ if (isThriftClient) {
++ part.setCreateTime(part_get.getCreateTime());
++ part.putToParameters(org.apache.hadoop.hive.metastore.api.hive_metastoreConstants.DDL_TIME, Long.toString(part_get.getCreateTime()));
++ }
++ part.setWriteId(part_get.getWriteId());
+ }
+
++
++
+ private static void silentDropDatabase(String dbName) throws TException {
+ try {
+ for (String tableName : client.getTables(dbName, "*")) {
+ client.dropTable(dbName, tableName);
+ }
+ client.dropDatabase(dbName);
+ } catch (NoSuchObjectException|InvalidOperationException e) {
+ // NOP
+ }
+ }
+
+ /**
+ * Tests for list partition by filter functionality.
+ */
+
+ @Test
+ public void testPartitionFilter() throws Exception {
+ String dbName = "filterdb";
+ String tblName = "filtertbl";
+
+ silentDropDatabase(dbName);
+
+ new DatabaseBuilder()
+ .setName(dbName)
+ .create(client, conf);
+
+ Table tbl = new TableBuilder()
+ .setDbName(dbName)
+ .setTableName(tblName)
+ .addCol("c1", ColumnType.STRING_TYPE_NAME)
+ .addCol("c2", ColumnType.INT_TYPE_NAME)
+ .addPartCol("p1", ColumnType.STRING_TYPE_NAME)
+ .addPartCol("p2", ColumnType.STRING_TYPE_NAME)
+ .addPartCol("p3", ColumnType.INT_TYPE_NAME)
+ .create(client, conf);
+
+ tbl = client.getTable(dbName, tblName);
+
+ add_partition(client, tbl, Lists.newArrayList("p11", "p21", "31"), "part1");
+ add_partition(client, tbl, Lists.newArrayList("p11", "p22", "32"), "part2");
+ add_partition(client, tbl, Lists.newArrayList("p12", "p21", "31"), "part3");
+ add_partition(client, tbl, Lists.newArrayList("p12", "p23", "32"), "part4");
+ add_partition(client, tbl, Lists.newArrayList("p13", "p24", "31"), "part5");
+ add_partition(client, tbl, Lists.newArrayList("p13", "p25", "-33"), "part6");
+
+ // Test equals operator for strings and integers.
+ checkFilter(client, dbName, tblName, "p1 = \"p11\"", 2);
+ checkFilter(client, dbName, tblName, "p1 = \"p12\"", 2);
+ checkFilter(client, dbName, tblName, "p2 = \"p21\"", 2);
+ checkFilter(client, dbName, tblName, "p2 = \"p23\"", 1);
+ checkFilter(client, dbName, tblName, "p3 = 31", 3);
+ checkFilter(client, dbName, tblName, "p3 = 33", 0);
+ checkFilter(client, dbName, tblName, "p3 = -33", 1);
+ checkFilter(client, dbName, tblName, "p1 = \"p11\" and p2=\"p22\"", 1);
+ checkFilter(client, dbName, tblName, "p1 = \"p11\" or p2=\"p23\"", 3);
+ checkFilter(client, dbName, tblName, "p1 = \"p11\" or p1=\"p12\"", 4);
+ checkFilter(client, dbName, tblName, "p1 = \"p11\" or p1=\"p12\"", 4);
+ checkFilter(client, dbName, tblName, "p1 = \"p11\" or p1=\"p12\"", 4);
+ checkFilter(client, dbName, tblName, "p1 = \"p11\" and p3 = 31", 1);
+ checkFilter(client, dbName, tblName, "p3 = -33 or p1 = \"p12\"", 3);
+
+ // Test not-equals operator for strings and integers.
+ checkFilter(client, dbName, tblName, "p1 != \"p11\"", 4);
+ checkFilter(client, dbName, tblName, "p2 != \"p23\"", 5);
+ checkFilter(client, dbName, tblName, "p2 != \"p33\"", 6);
+ checkFilter(client, dbName, tblName, "p3 != 32", 4);
+ checkFilter(client, dbName, tblName, "p3 != 8589934592", 6);
+ checkFilter(client, dbName, tblName, "p1 != \"p11\" and p1 != \"p12\"", 2);
+ checkFilter(client, dbName, tblName, "p1 != \"p11\" and p2 != \"p22\"", 4);
+ checkFilter(client, dbName, tblName, "p1 != \"p11\" or p2 != \"p22\"", 5);
+ checkFilter(client, dbName, tblName, "p1 != \"p12\" and p2 != \"p25\"", 3);
+ checkFilter(client, dbName, tblName, "p1 != \"p12\" or p2 != \"p25\"", 6);
+ checkFilter(client, dbName, tblName, "p3 != -33 or p1 != \"p13\"", 5);
+ checkFilter(client, dbName, tblName, "p1 != \"p11\" and p3 = 31", 2);
+ checkFilter(client, dbName, tblName, "p3 != 31 and p1 = \"p12\"", 1);
+
+ // Test reverse order.
+ checkFilter(client, dbName, tblName, "31 != p3 and p1 = \"p12\"", 1);
+ checkFilter(client, dbName, tblName, "\"p23\" = p2", 1);
+
+ // Test and/or more...
+ checkFilter(client, dbName, tblName,
+ "p1 = \"p11\" or (p1=\"p12\" and p2=\"p21\")", 3);
+ checkFilter(client, dbName, tblName,
+ "p1 = \"p11\" or (p1=\"p12\" and p2=\"p21\") Or " +
+ "(p1=\"p13\" aNd p2=\"p24\")", 4);
+ //test for and or precedence
+ checkFilter(client, dbName, tblName,
+ "p1=\"p12\" and (p2=\"p27\" Or p2=\"p21\")", 1);
+ checkFilter(client, dbName, tblName,
+ "p1=\"p12\" and p2=\"p27\" Or p2=\"p21\"", 2);
+
+ // Test gt/lt/lte/gte/like for strings.
+ checkFilter(client, dbName, tblName, "p1 > \"p12\"", 2);
+ checkFilter(client, dbName, tblName, "p1 >= \"p12\"", 4);
+ checkFilter(client, dbName, tblName, "p1 < \"p12\"", 2);
+ checkFilter(client, dbName, tblName, "p1 <= \"p12\"", 4);
+ checkFilter(client, dbName, tblName, "p1 like \"p1.*\"", 6);
+ checkFilter(client, dbName, tblName, "p2 like \"p.*3\"", 1);
+
+ // Test gt/lt/lte/gte for numbers.
+ checkFilter(client, dbName, tblName, "p3 < 0", 1);
+ checkFilter(client, dbName, tblName, "p3 >= -33", 6);
+ checkFilter(client, dbName, tblName, "p3 > -33", 5);
+ checkFilter(client, dbName, tblName, "p3 > 31 and p3 < 32", 0);
+ checkFilter(client, dbName, tblName, "p3 > 31 or p3 < 31", 3);
+ checkFilter(client, dbName, tblName, "p3 > 30 or p3 < 30", 6);
+ checkFilter(client, dbName, tblName, "p3 >= 31 or p3 < -32", 6);
+ checkFilter(client, dbName, tblName, "p3 >= 32", 2);
+ checkFilter(client, dbName, tblName, "p3 > 32", 0);
+
+ // Test between
+ checkFilter(client, dbName, tblName, "p1 between \"p11\" and \"p12\"", 4);
+ checkFilter(client, dbName, tblName, "p1 not between \"p11\" and \"p12\"", 2);
+ checkFilter(client, dbName, tblName, "p3 not between 0 and 2", 6);
+ checkFilter(client, dbName, tblName, "p3 between 31 and 32", 5);
+ checkFilter(client, dbName, tblName, "p3 between 32 and 31", 0);
+ checkFilter(client, dbName, tblName, "p3 between -32 and 34 and p3 not between 31 and 32", 0);
+ checkFilter(client, dbName, tblName, "p3 between 1 and 3 or p3 not between 1 and 3", 6);
+ checkFilter(client, dbName, tblName,
+ "p3 between 31 and 32 and p1 between \"p12\" and \"p14\"", 3);
+
+ //Test for setting the maximum partition count
+ List<Partition> partitions = client.listPartitionsByFilter(dbName,
+ tblName, "p1 >= \"p12\"", (short) 2);
+ assertEquals("User specified row limit for partitions",
+ 2, partitions.size());
+
+ //Negative tests
+ Exception me = null;
+ try {
+ client.listPartitionsByFilter(dbName,
+ tblName, "p3 >= \"p12\"", (short) -1);
+ } catch(MetaException e) {
+ me = e;
+ }
+ assertNotNull(me);
+ assertTrue("Filter on int partition key", me.getMessage().contains(
+ "Filtering is supported only on partition keys of type string"));
+
+ me = null;
+ try {
+ client.listPartitionsByFilter(dbName,
+ tblName, "c1 >= \"p12\"", (short) -1);
+ } catch(MetaException e) {
+ me = e;
+ }
+ assertNotNull(me);
+ assertTrue("Filter on invalid key", me.getMessage().contains(
+ "<c1> is not a partitioning key for the table"));
+
+ me = null;
+ try {
+ client.listPartitionsByFilter(dbName,
+ tblName, "c1 >= ", (short) -1);
+ } catch(MetaException e) {
+ me = e;
+ }
+ assertNotNull(me);
+ assertTrue("Invalid filter string", me.getMessage().contain
<TRUNCATED>