You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by xu...@apache.org on 2015/07/16 06:40:55 UTC
[08/50] [abbrv] hive git commit: HIVE-10895: ObjectStore does not
close Query objects in some calls,
causing a potential leak in some metastore db resources (Aihua Xu reviewed by
Chaoyu Tang, Sergey Shelukhin, Vaibhav Gumashta)
http://git-wip-us.apache.org/repos/asf/hive/blob/08595ffa/metastore/src/java/org/apache/hadoop/hive/metastore/tools/HiveMetaTool.java
----------------------------------------------------------------------
diff --git a/metastore/src/java/org/apache/hadoop/hive/metastore/tools/HiveMetaTool.java b/metastore/src/java/org/apache/hadoop/hive/metastore/tools/HiveMetaTool.java
index d0ff329..411ac21 100644
--- a/metastore/src/java/org/apache/hadoop/hive/metastore/tools/HiveMetaTool.java
+++ b/metastore/src/java/org/apache/hadoop/hive/metastore/tools/HiveMetaTool.java
@@ -149,16 +149,21 @@ public class HiveMetaTool {
initObjectStore(hiveConf);
System.out.println("Executing query: " + query);
- Collection<?> result = objStore.executeJDOQLSelect(query);
- if (result != null) {
- Iterator<?> iter = result.iterator();
- while (iter.hasNext()) {
- Object o = iter.next();
- System.out.println(o.toString());
+ ObjectStore.QueryWrapper queryWrapper = new ObjectStore.QueryWrapper();
+ try {
+ Collection<?> result = objStore.executeJDOQLSelect(query, queryWrapper);
+ if (result != null) {
+ Iterator<?> iter = result.iterator();
+ while (iter.hasNext()) {
+ Object o = iter.next();
+ System.out.println(o.toString());
+ }
+ } else {
+ System.err.println("Encountered error during executeJDOQLSelect -" +
+ "commit of JDO transaction failed.");
}
- } else {
- System.err.println("Encountered error during executeJDOQLSelect -" +
- "commit of JDO transaction failed.");
+ } finally {
+ queryWrapper.close();
}
}
http://git-wip-us.apache.org/repos/asf/hive/blob/08595ffa/metastore/src/test/org/apache/hadoop/hive/metastore/TestObjectStore.java
----------------------------------------------------------------------
diff --git a/metastore/src/test/org/apache/hadoop/hive/metastore/TestObjectStore.java b/metastore/src/test/org/apache/hadoop/hive/metastore/TestObjectStore.java
new file mode 100644
index 0000000..a4f9f6c
--- /dev/null
+++ b/metastore/src/test/org/apache/hadoop/hive/metastore/TestObjectStore.java
@@ -0,0 +1,230 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hive.metastore;
+
+import java.util.Arrays;
+import java.util.HashMap;
+import java.util.List;
+
+import org.apache.hadoop.hive.conf.HiveConf;
+import org.apache.hadoop.hive.metastore.api.Database;
+import org.apache.hadoop.hive.metastore.api.FieldSchema;
+import org.apache.hadoop.hive.metastore.api.InvalidInputException;
+import org.apache.hadoop.hive.metastore.api.InvalidObjectException;
+import org.apache.hadoop.hive.metastore.api.MetaException;
+import org.apache.hadoop.hive.metastore.api.NoSuchObjectException;
+import org.apache.hadoop.hive.metastore.api.Partition;
+import org.apache.hadoop.hive.metastore.api.PrincipalType;
+import org.apache.hadoop.hive.metastore.api.Role;
+import org.apache.hadoop.hive.metastore.api.SerDeInfo;
+import org.apache.hadoop.hive.metastore.api.StorageDescriptor;
+import org.apache.hadoop.hive.metastore.api.Table;
+import org.apache.hadoop.hive.serde2.typeinfo.PrimitiveTypeInfo;
+import org.junit.After;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Test;
+
+public class TestObjectStore {
+ private ObjectStore objectStore = null;
+
+ private static final String DB1 = "testobjectstoredb1";
+ private static final String DB2 = "testobjectstoredb2";
+ private static final String TABLE1 = "testobjectstoretable1";
+ private static final String KEY1 = "testobjectstorekey1";
+ private static final String KEY2 = "testobjectstorekey2";
+ private static final String OWNER = "testobjectstoreowner";
+ private static final String USER1 = "testobjectstoreuser1";
+ private static final String ROLE1 = "testobjectstorerole1";
+ private static final String ROLE2 = "testobjectstorerole2";
+
+ public static class MockPartitionExpressionProxy implements PartitionExpressionProxy {
+ @Override
+ public String convertExprToFilter(byte[] expr) throws MetaException {
+ return null;
+ }
+
+ @Override
+ public boolean filterPartitionsByExpr(List<String> partColumnNames,
+ List<PrimitiveTypeInfo> partColumnTypeInfos, byte[] expr,
+ String defaultPartitionName, List<String> partitionNames)
+ throws MetaException {
+ return false;
+ }
+ }
+
+ @Before
+ public void setUp() {
+ HiveConf conf = new HiveConf();
+ conf.setVar(HiveConf.ConfVars.METASTORE_EXPRESSION_PROXY_CLASS, MockPartitionExpressionProxy.class.getName());
+
+ objectStore = new ObjectStore();
+ objectStore.setConf(conf);
+
+ Deadline.registerIfNot(100000);
+ try {
+ objectStore.dropDatabase(DB1);
+ } catch (Exception e) {
+ }
+ try {
+ objectStore.dropDatabase(DB2);
+ } catch (Exception e) {
+ }
+ }
+
+ @After
+ public void tearDown() {
+ }
+
+ /**
+ * Test database operations
+ */
+ @Test
+ public void testDatabaseOps() throws MetaException, InvalidObjectException, NoSuchObjectException {
+ Database db1 = new Database(DB1, "description", "locationurl", null);
+ Database db2 = new Database(DB2, "description", "locationurl", null);
+ objectStore.createDatabase(db1);
+ objectStore.createDatabase(db2);
+
+ List<String> databases = objectStore.getAllDatabases();
+ Assert.assertEquals(2, databases.size());
+ Assert.assertEquals(DB1, databases.get(0));
+ Assert.assertEquals(DB2, databases.get(1));
+
+ objectStore.dropDatabase(DB1);
+ databases = objectStore.getAllDatabases();
+ Assert.assertEquals(1, databases.size());
+ Assert.assertEquals(DB2, databases.get(0));
+
+ objectStore.dropDatabase(DB2);
+ }
+
+ /**
+ * Test table operations
+ */
+ @Test
+ public void testTableOps() throws MetaException, InvalidObjectException, NoSuchObjectException, InvalidInputException {
+ Database db1 = new Database(DB1, "description", "locationurl", null);
+ objectStore.createDatabase(db1);
+ StorageDescriptor sd = new StorageDescriptor(null, "location", null, null, false, 0, new SerDeInfo("SerDeName", "serializationLib", null), null, null, null);
+ HashMap<String,String> params = new HashMap<String,String>();
+ params.put("EXTERNAL", "false");
+ Table tbl1 = new Table(TABLE1, DB1, "owner", 1, 2, 3, sd, null, params, "viewOriginalText", "viewExpandedText", "MANAGED_TABLE");
+ objectStore.createTable(tbl1);
+
+ List<String> tables = objectStore.getAllTables(DB1);
+ Assert.assertEquals(1, tables.size());
+ Assert.assertEquals(TABLE1, tables.get(0));
+
+ Table newTbl1 = new Table("new" + TABLE1, DB1, "owner", 1, 2, 3, sd, null, params, "viewOriginalText", "viewExpandedText", "MANAGED_TABLE");
+ objectStore.alterTable(DB1, TABLE1, newTbl1);
+ tables = objectStore.getTables(DB1, "new*");
+ Assert.assertEquals(1, tables.size());
+ Assert.assertEquals("new" + TABLE1, tables.get(0));
+
+ objectStore.dropTable(DB1, "new" + TABLE1);
+ tables = objectStore.getAllTables(DB1);
+ Assert.assertEquals(0, tables.size());
+
+ objectStore.dropDatabase(DB1);
+ }
+
+ /**
+ * Tests partition operations
+ */
+ @Test
+ public void testPartitionOps() throws MetaException, InvalidObjectException, NoSuchObjectException, InvalidInputException {
+ Database db1 = new Database(DB1, "description", "locationurl", null);
+ objectStore.createDatabase(db1);
+ StorageDescriptor sd = new StorageDescriptor(null, "location", null, null, false, 0, new SerDeInfo("SerDeName", "serializationLib", null), null, null, null);
+ HashMap<String,String> tableParams = new HashMap<String,String>();
+ tableParams.put("EXTERNAL", "false");
+ FieldSchema partitionKey1 = new FieldSchema("Country", "String", "");
+ FieldSchema partitionKey2 = new FieldSchema("State", "String", "");
+ Table tbl1 = new Table(TABLE1, DB1, "owner", 1, 2, 3, sd, Arrays.asList(partitionKey1, partitionKey2), tableParams, "viewOriginalText", "viewExpandedText", "MANAGED_TABLE");
+ objectStore.createTable(tbl1);
+ HashMap<String, String> partitionParams = new HashMap<String, String>();
+ partitionParams.put("PARTITION_LEVEL_PRIVILEGE", "true");
+ List<String> value1 = Arrays.asList("US", "CA");
+ Partition part1 = new Partition(value1, DB1, TABLE1, 111, 111, sd, partitionParams);
+ objectStore.addPartition(part1);
+ List<String> value2 = Arrays.asList("US", "MA");
+ Partition part2 = new Partition(value2, DB1, TABLE1, 222, 222, sd, partitionParams);
+ objectStore.addPartition(part2);
+
+ Deadline.startTimer("getPartition");
+ List<Partition> partitions = objectStore.getPartitions(DB1, TABLE1, 10);
+ Assert.assertEquals(2, partitions.size());
+ Assert.assertEquals(111, partitions.get(0).getCreateTime());
+ Assert.assertEquals(222, partitions.get(1).getCreateTime());
+
+ objectStore.dropPartition(DB1, TABLE1, value1);
+ partitions = objectStore.getPartitions(DB1, TABLE1, 10);
+ Assert.assertEquals(1, partitions.size());
+ Assert.assertEquals(222, partitions.get(0).getCreateTime());
+
+ objectStore.dropPartition(DB1, TABLE1, value2);
+ objectStore.dropTable(DB1, TABLE1);
+ objectStore.dropDatabase(DB1);
+ }
+
+ /**
+ * Test master keys operation
+ */
+ @Test
+ public void testMasterKeyOps() throws MetaException, NoSuchObjectException {
+ int id1 = objectStore.addMasterKey(KEY1);
+ int id2 = objectStore.addMasterKey(KEY2);
+
+ String[] keys = objectStore.getMasterKeys();
+ Assert.assertEquals(2, keys.length);
+ Assert.assertEquals(KEY1, keys[0]);
+ Assert.assertEquals(KEY2, keys[1]);
+
+ objectStore.updateMasterKey(id1, "new" + KEY1);
+ objectStore.updateMasterKey(id2, "new" + KEY2);
+ keys = objectStore.getMasterKeys();
+ Assert.assertEquals(2, keys.length);
+ Assert.assertEquals("new" + KEY1, keys[0]);
+ Assert.assertEquals("new" + KEY2, keys[1]);
+
+ objectStore.removeMasterKey(id1);
+ keys = objectStore.getMasterKeys();
+ Assert.assertEquals(1, keys.length);
+ Assert.assertEquals("new" + KEY2, keys[0]);
+
+ objectStore.removeMasterKey(id2);
+ }
+
+ /**
+ * Test role operation
+ */
+ @Test
+ public void testRoleOps() throws InvalidObjectException, MetaException, NoSuchObjectException {
+ objectStore.addRole(ROLE1, OWNER);
+ objectStore.addRole(ROLE2, OWNER);
+ List<String> roles = objectStore.listRoleNames();
+ Assert.assertEquals(2, roles.size());
+ Assert.assertEquals(ROLE2, roles.get(1));
+ Role role1 = objectStore.getRole(ROLE1);
+ Assert.assertEquals(OWNER, role1.getOwnerName());
+ objectStore.grantRole(role1, USER1, PrincipalType.USER, OWNER, PrincipalType.ROLE, true);
+ objectStore.revokeRole(role1, USER1, PrincipalType.USER, false);
+ objectStore.removeRole(ROLE1);
+ }
+}