You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@bigtop.apache.org by rv...@apache.org on 2017/03/23 17:27:39 UTC
[08/50] [abbrv] bigtop git commit: ODPI-184 Pulled my head out. I had
written the tests so that they were running on the client rather than
connecting to the server. Fixed that so they will now talk to the metastore
server.
ODPI-184 Pulled my head out. I had written the tests so that they were running on the client rather than connecting to the server. Fixed that so they will now talk to the metastore server.
(cherry picked from commit 76dc0e3854a36bc41bd9f3298298e0db9393de54)
Project: http://git-wip-us.apache.org/repos/asf/bigtop/repo
Commit: http://git-wip-us.apache.org/repos/asf/bigtop/commit/46cd194b
Tree: http://git-wip-us.apache.org/repos/asf/bigtop/tree/46cd194b
Diff: http://git-wip-us.apache.org/repos/asf/bigtop/diff/46cd194b
Branch: refs/heads/master
Commit: 46cd194bac9fd6fe9f8259023c80efa87413a9c4
Parents: 166e9f7
Author: Alan Gates <ga...@hortonworks.com>
Authored: Tue Nov 1 16:38:14 2016 -0700
Committer: Roman Shaposhnik <rv...@apache.org>
Committed: Thu Mar 23 10:27:11 2017 -0700
----------------------------------------------------------------------
.../org/odpi/specs/runtime/hive/TestThrift.java | 129 ++++++++-----------
1 file changed, 57 insertions(+), 72 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/bigtop/blob/46cd194b/bigtop-tests/spec-tests/runtime/src/test/java/org/odpi/specs/runtime/hive/TestThrift.java
----------------------------------------------------------------------
diff --git a/bigtop-tests/spec-tests/runtime/src/test/java/org/odpi/specs/runtime/hive/TestThrift.java b/bigtop-tests/spec-tests/runtime/src/test/java/org/odpi/specs/runtime/hive/TestThrift.java
index 1aede96..5eaab95 100644
--- a/bigtop-tests/spec-tests/runtime/src/test/java/org/odpi/specs/runtime/hive/TestThrift.java
+++ b/bigtop-tests/spec-tests/runtime/src/test/java/org/odpi/specs/runtime/hive/TestThrift.java
@@ -17,29 +17,27 @@
*/
package org.odpi.specs.runtime.hive;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.hive.conf.HiveConf;
-import org.apache.hadoop.hive.metastore.HiveMetaStore;
+import org.apache.hadoop.hive.metastore.HiveMetaStoreClient;
+import org.apache.hadoop.hive.metastore.IMetaStoreClient;
import org.apache.hadoop.hive.metastore.TableType;
-import org.apache.hadoop.hive.metastore.api.AddPartitionsRequest;
import org.apache.hadoop.hive.metastore.api.Database;
-import org.apache.hadoop.hive.metastore.api.DropPartitionsRequest;
-import org.apache.hadoop.hive.metastore.api.EnvironmentContext;
import org.apache.hadoop.hive.metastore.api.FieldSchema;
import org.apache.hadoop.hive.metastore.api.MetaException;
import org.apache.hadoop.hive.metastore.api.Partition;
-import org.apache.hadoop.hive.metastore.api.RequestPartsSpec;
import org.apache.hadoop.hive.metastore.api.SerDeInfo;
import org.apache.hadoop.hive.metastore.api.StorageDescriptor;
import org.apache.hadoop.hive.metastore.api.Table;
-import org.apache.hadoop.hive.metastore.api.ThriftHiveMetastore;
import org.apache.thrift.TException;
import org.junit.Assert;
import org.junit.Assume;
import org.junit.Before;
import org.junit.BeforeClass;
-import org.junit.Ignore;
import org.junit.Test;
+import java.util.ArrayList;
import java.util.Collections;
import java.util.HashMap;
import java.util.List;
@@ -47,7 +45,9 @@ import java.util.Random;
public class TestThrift {
- private static ThriftHiveMetastore.Iface client = null;
+ private static final Log LOG = LogFactory.getLog(JdbcConnector.class.getName());
+
+ private static IMetaStoreClient client = null;
private static HiveConf conf;
private Random rand;
@@ -58,7 +58,8 @@ public class TestThrift {
String url = JdbcConnector.getProperty(JdbcConnector.METASTORE_URL, "Thrift metastore URL");
conf = new HiveConf();
conf.setVar(HiveConf.ConfVars.METASTOREURIS, url);
- client = new HiveMetaStore.HMSHandler("ODPi test", conf, true);
+ LOG.info("Set to test against metastore at " + url);
+ client = new HiveMetaStoreClient(conf);
}
}
@@ -72,22 +73,20 @@ public class TestThrift {
public void db() throws TException {
final String dbName = "odpi_thrift_db_" + rand.nextInt(Integer.MAX_VALUE);
- String location = JdbcConnector.getProperty(JdbcConnector.LOCATION, " HDFS location we can " +
- "write to");
- Database db = new Database(dbName, "a db", location, new HashMap<String, String>());
- client.create_database(db);
- db = client.get_database(dbName);
+ Database db = new Database(dbName, "a db", null, new HashMap<String, String>());
+ client.createDatabase(db);
+ db = client.getDatabase(dbName);
Assert.assertNotNull(db);
db = new Database(db);
db.getParameters().put("a", "b");
- client.alter_database(dbName, db);
- List<String> alldbs = client.get_databases("odpi_*");
+ client.alterDatabase(dbName, db);
+ List<String> alldbs = client.getDatabases("odpi_*");
Assert.assertNotNull(alldbs);
Assert.assertTrue(alldbs.size() > 0);
- alldbs = client.get_all_databases();
+ alldbs = client.getAllDatabases();
Assert.assertNotNull(alldbs);
Assert.assertTrue(alldbs.size() > 0);
- client.drop_database(dbName, true, true);
+ client.dropDatabase(dbName, true, true);
}
// Not testing types calls, as they aren't used AFAIK
@@ -95,138 +94,129 @@ public class TestThrift {
@Test
public void nonPartitionedTable() throws TException {
final String tableName = "odpi_thrift_table_" + rand.nextInt(Integer.MAX_VALUE);
- String location = JdbcConnector.getProperty(JdbcConnector.LOCATION, " HDFS location we can " +
- "write to");
// I don't test every operation related to tables, but only those that are frequently used.
SerDeInfo serde = new SerDeInfo("default_serde",
conf.getVar(HiveConf.ConfVars.HIVEDEFAULTSERDE), new HashMap<String, String>());
FieldSchema fs = new FieldSchema("a", "int", "no comment");
- StorageDescriptor sd = new StorageDescriptor(Collections.singletonList(fs), location,
+ StorageDescriptor sd = new StorageDescriptor(Collections.singletonList(fs), null,
conf.getVar(HiveConf.ConfVars.HIVEDEFAULTFILEFORMAT),
conf.getVar(HiveConf.ConfVars.HIVEDEFAULTFILEFORMAT), false, 0, serde, null, null,
new HashMap<String, String>());
Table table = new Table(tableName, "default", "me", 0, 0, 0, sd, null,
new HashMap<String, String>(), null, null, TableType.MANAGED_TABLE.toString());
- EnvironmentContext envContext = new EnvironmentContext(new HashMap<String, String>());
- client.create_table_with_environment_context(table, envContext);
+ client.createTable(table);
- table = client.get_table("default", tableName);
+ table = client.getTable("default", tableName);
Assert.assertNotNull(table);
List<Table> tables =
- client.get_table_objects_by_name("default", Collections.singletonList(tableName));
+ client.getTableObjectsByName("default", Collections.singletonList(tableName));
Assert.assertNotNull(tables);
Assert.assertEquals(1, tables.size());
- List<String> tableNames = client.get_tables("default", "odpi_*");
+ List<String> tableNames = client.getTables("default", "odpi_*");
Assert.assertNotNull(tableNames);
Assert.assertTrue(tableNames.size() >= 1);
- tableNames = client.get_all_tables("default");
+ tableNames = client.getAllTables("default");
Assert.assertNotNull(tableNames);
Assert.assertTrue(tableNames.size() >= 1);
- List<FieldSchema> cols = client.get_fields("default", tableName);
+ List<FieldSchema> cols = client.getFields("default", tableName);
Assert.assertNotNull(cols);
Assert.assertEquals(1, cols.size());
- cols = client.get_schema_with_environment_context("default", tableName, envContext);
+ cols = client.getSchema("default", tableName);
Assert.assertNotNull(cols);
Assert.assertEquals(1, cols.size());
table = new Table(table);
table.getParameters().put("a", "b");
- client.alter_table_with_cascade("default", tableName, table, false);
+ client.alter_table("default", tableName, table, false);
table.getParameters().put("c", "d");
- client.alter_table_with_environment_context("default", tableName, table, envContext);
+ client.alter_table("default", tableName, table);
- client.drop_table_with_environment_context("default", tableName, true, envContext);
+ client.dropTable("default", tableName, true, false);
}
@Test
public void partitionedTable() throws TException {
final String tableName = "odpi_thrift_partitioned_table_" + rand.nextInt(Integer.MAX_VALUE);
- String location = JdbcConnector.getProperty(JdbcConnector.LOCATION, " HDFS location we can " +
- "write to");
// I don't test every operation related to tables, but only those that are frequently used.
SerDeInfo serde = new SerDeInfo("default_serde",
conf.getVar(HiveConf.ConfVars.HIVEDEFAULTSERDE), new HashMap<String, String>());
FieldSchema fs = new FieldSchema("a", "int", "no comment");
- StorageDescriptor sd = new StorageDescriptor(Collections.singletonList(fs), location,
+ StorageDescriptor sd = new StorageDescriptor(Collections.singletonList(fs), null,
conf.getVar(HiveConf.ConfVars.HIVEDEFAULTFILEFORMAT),
conf.getVar(HiveConf.ConfVars.HIVEDEFAULTFILEFORMAT), false, 0, serde, null, null,
new HashMap<String, String>());
FieldSchema pk = new FieldSchema("pk", "string", "");
Table table = new Table(tableName, "default", "me", 0, 0, 0, sd, Collections.singletonList(pk),
new HashMap<String, String>(), null, null, TableType.MANAGED_TABLE.toString());
- EnvironmentContext envContext = new EnvironmentContext(new HashMap<String, String>());
- client.create_table_with_environment_context(table, envContext);
+ client.createTable(table);
- sd = new StorageDescriptor(Collections.singletonList(fs), location + "/x",
+ sd = new StorageDescriptor(Collections.singletonList(fs), null,
conf.getVar(HiveConf.ConfVars.HIVEDEFAULTSERDE),
conf.getVar(HiveConf.ConfVars.HIVEDEFAULTSERDE), false, 0, serde, null, null,
new HashMap<String, String>());
Partition partition = new Partition(Collections.singletonList("x"), "default", tableName, 0,
0, sd, new HashMap<String, String>());
- client.add_partition_with_environment_context(partition, envContext);
+ client.add_partition(partition);
- sd = new StorageDescriptor(Collections.singletonList(fs), location + "/y",
+ List<Partition> partitions = new ArrayList<>(2);
+ sd = new StorageDescriptor(Collections.singletonList(fs), null,
conf.getVar(HiveConf.ConfVars.HIVEDEFAULTSERDE),
conf.getVar(HiveConf.ConfVars.HIVEDEFAULTSERDE), false, 0, serde, null, null,
new HashMap<String, String>());
- partition = new Partition(Collections.singletonList("y"), "default", tableName, 0,
- 0, sd, new HashMap<String, String>());
- client.add_partitions(Collections.singletonList(partition));
-
- sd = new StorageDescriptor(Collections.singletonList(fs), location + "/z",
+ partitions.add(new Partition(Collections.singletonList("y"), "default", tableName, 0,
+ 0, sd, new HashMap<String, String>()));
+ sd = new StorageDescriptor(Collections.singletonList(fs), null,
conf.getVar(HiveConf.ConfVars.HIVEDEFAULTSERDE),
conf.getVar(HiveConf.ConfVars.HIVEDEFAULTSERDE), false, 0, serde, null, null,
new HashMap<String, String>());
- partition = new Partition(Collections.singletonList("z"), "default", tableName, 0,
- 0, sd, new HashMap<String, String>());
- AddPartitionsRequest rqst = new AddPartitionsRequest("default", tableName,
- Collections.singletonList(partition), true);
- client.add_partitions_req(rqst);
+ partitions.add(new Partition(Collections.singletonList("z"), "default", tableName, 0,
+ 0, sd, new HashMap<String, String>()));
+ client.add_partitions(partitions);
- List<Partition> parts = client.get_partitions("default", tableName, (short)-1);
+ List<Partition> parts = client.listPartitions("default", tableName, (short)-1);
Assert.assertNotNull(parts);
Assert.assertEquals(3, parts.size());
- parts = client.get_partitions_with_auth("default", tableName, (short)-1, "me",
+ parts = client.listPartitions("default", tableName, Collections.singletonList("x"),
+ (short)-1);
+ Assert.assertNotNull(parts);
+ Assert.assertEquals(1, parts.size());
+
+ parts = client.listPartitionsWithAuthInfo("default", tableName, (short)-1, "me",
Collections.<String>emptyList());
Assert.assertNotNull(parts);
Assert.assertEquals(3, parts.size());
- parts = client.get_partitions_ps("default", tableName, Collections.singletonList("x"),
- (short)-1);
- Assert.assertNotNull(parts);
- Assert.assertEquals(1, parts.size());
+ List<String> partNames = client.listPartitionNames("default", tableName, (short)-1);
+ Assert.assertNotNull(partNames);
+ Assert.assertEquals(3, partNames.size());
- parts = client.get_partitions_by_filter("default", tableName, "pk = \"x\"", (short)-1);
+ parts = client.listPartitionsByFilter("default", tableName, "pk = \"x\"", (short)-1);
Assert.assertNotNull(parts);
Assert.assertEquals(1, parts.size());
- parts = client.get_partitions_by_names("default", tableName, Collections.singletonList("pk=x"));
+ parts = client.getPartitionsByNames("default", tableName, Collections.singletonList("pk=x"));
Assert.assertNotNull(parts);
Assert.assertEquals(1, parts.size());
- partition = client.get_partition("default", tableName, Collections.singletonList("x"));
+ partition = client.getPartition("default", tableName, Collections.singletonList("x"));
Assert.assertNotNull(partition);
- partition = client.get_partition_by_name("default", tableName, "pk=x");
+ partition = client.getPartition("default", tableName, "pk=x");
Assert.assertNotNull(partition);
- partition = client.get_partition_with_auth("default", tableName, Collections.singletonList("x"),
+ partition = client.getPartitionWithAuthInfo("default", tableName, Collections.singletonList("x"),
"me", Collections.<String>emptyList());
Assert.assertNotNull(partition);
- List<String> partitionNames = client.get_partition_names("default", tableName, (short)-1);
- Assert.assertNotNull(partitionNames);
- Assert.assertEquals(3, partitionNames.size());
-
partition = new Partition(partition);
partition.getParameters().put("a", "b");
client.alter_partition("default", tableName, partition);
@@ -240,13 +230,8 @@ public class TestThrift {
// Not testing partition marking events, not used by anyone but Hive replication AFAIK
- client.drop_partition_by_name_with_environment_context("default", tableName, "pk=x", true,
- envContext);
- client.drop_partition_with_environment_context("default", tableName,
- Collections.singletonList("y"), true, envContext);
- DropPartitionsRequest dropRequest = new DropPartitionsRequest("default", tableName,
- RequestPartsSpec.names(Collections.singletonList("pk=z")));
- client.drop_partitions_req(dropRequest);
+ client.dropPartition("default", tableName, "pk=x", true);
+ client.dropPartition("default", tableName, Collections.singletonList("y"), true);
}
// Not testing index calls, as no one uses indices