[07/50] [abbrv] hive git commit: HIVE-10895: ObjectStore does not close Query objects in some calls, causing a potential leak in some metastore db resources (Aihua Xu reviewed by Chaoyu Tang, Sergey Shelukhin, Vaibhav Gumashta)

[xu...@apache.org]

http://git-wip-us.apache.org/repos/asf/hive/blob/08595ffa/metastore/src/java/org/apache/hadoop/hive/metastore/tools/HiveMetaTool.java
----------------------------------------------------------------------
diff --git a/metastore/src/java/org/apache/hadoop/hive/metastore/tools/HiveMetaTool.java b/metastore/src/java/org/apache/hadoop/hive/metastore/tools/HiveMetaTool.java
index d0ff329..411ac21 100644
--- a/metastore/src/java/org/apache/hadoop/hive/metastore/tools/HiveMetaTool.java
+++ b/metastore/src/java/org/apache/hadoop/hive/metastore/tools/HiveMetaTool.java
@@ -149,16 +149,21 @@ public class HiveMetaTool {
     initObjectStore(hiveConf);
 
     System.out.println("Executing query: " + query);
-    Collection<?> result = objStore.executeJDOQLSelect(query);
-    if (result != null) {
-      Iterator<?> iter = result.iterator();
-      while (iter.hasNext()) {
-        Object o = iter.next();
-        System.out.println(o.toString());
+    ObjectStore.QueryWrapper queryWrapper = new ObjectStore.QueryWrapper();
+    try {
+      Collection<?> result = objStore.executeJDOQLSelect(query, queryWrapper);
+      if (result != null) {
+        Iterator<?> iter = result.iterator();
+        while (iter.hasNext()) {
+          Object o = iter.next();
+          System.out.println(o.toString());
+        }
+      } else {
+        System.err.println("Encountered error during executeJDOQLSelect -" +
+          "commit of JDO transaction failed.");
       }
-    } else {
-      System.err.println("Encountered error during executeJDOQLSelect -" +
-        "commit of JDO transaction failed.");
+    } finally {
+      queryWrapper.close();
     }
   }
 

http://git-wip-us.apache.org/repos/asf/hive/blob/08595ffa/metastore/src/test/org/apache/hadoop/hive/metastore/TestObjectStore.java
----------------------------------------------------------------------
diff --git a/metastore/src/test/org/apache/hadoop/hive/metastore/TestObjectStore.java b/metastore/src/test/org/apache/hadoop/hive/metastore/TestObjectStore.java
new file mode 100644
index 0000000..a4f9f6c
--- /dev/null
+++ b/metastore/src/test/org/apache/hadoop/hive/metastore/TestObjectStore.java
@@ -0,0 +1,230 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hive.metastore;
+
+import java.util.Arrays;
+import java.util.HashMap;
+import java.util.List;
+
+import org.apache.hadoop.hive.conf.HiveConf;
+import org.apache.hadoop.hive.metastore.api.Database;
+import org.apache.hadoop.hive.metastore.api.FieldSchema;
+import org.apache.hadoop.hive.metastore.api.InvalidInputException;
+import org.apache.hadoop.hive.metastore.api.InvalidObjectException;
+import org.apache.hadoop.hive.metastore.api.MetaException;
+import org.apache.hadoop.hive.metastore.api.NoSuchObjectException;
+import org.apache.hadoop.hive.metastore.api.Partition;
+import org.apache.hadoop.hive.metastore.api.PrincipalType;
+import org.apache.hadoop.hive.metastore.api.Role;
+import org.apache.hadoop.hive.metastore.api.SerDeInfo;
+import org.apache.hadoop.hive.metastore.api.StorageDescriptor;
+import org.apache.hadoop.hive.metastore.api.Table;
+import org.apache.hadoop.hive.serde2.typeinfo.PrimitiveTypeInfo;
+import org.junit.After;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Test;
+
+public class TestObjectStore {
+  private ObjectStore objectStore = null;
+
+  private static final String DB1 = "testobjectstoredb1";
+  private static final String DB2 = "testobjectstoredb2";
+  private static final String TABLE1 = "testobjectstoretable1";
+  private static final String KEY1 = "testobjectstorekey1";
+  private static final String KEY2 = "testobjectstorekey2";
+  private static final String OWNER = "testobjectstoreowner";
+  private static final String USER1 = "testobjectstoreuser1";
+  private static final String ROLE1 = "testobjectstorerole1";
+  private static final String ROLE2 = "testobjectstorerole2";
+
+  public static class MockPartitionExpressionProxy implements PartitionExpressionProxy {
+    @Override
+    public String convertExprToFilter(byte[] expr) throws MetaException {
+      return null;
+    }
+
+    @Override
+    public boolean filterPartitionsByExpr(List<String> partColumnNames,
+        List<PrimitiveTypeInfo> partColumnTypeInfos, byte[] expr,
+        String defaultPartitionName, List<String> partitionNames)
+        throws MetaException {
+      return false;
+    }
+  }
+
+  @Before
+  public void setUp() {
+    HiveConf conf = new HiveConf();
+    conf.setVar(HiveConf.ConfVars.METASTORE_EXPRESSION_PROXY_CLASS, MockPartitionExpressionProxy.class.getName());
+
+    objectStore = new ObjectStore();
+    objectStore.setConf(conf);
+
+    Deadline.registerIfNot(100000);
+    try {
+      objectStore.dropDatabase(DB1);
+    } catch (Exception e) {
+    }
+    try {
+      objectStore.dropDatabase(DB2);
+    } catch (Exception e) {
+    }
+  }
+
+  @After
+  public void tearDown() {
+  }
+
+  /**
+   * Test database operations
+   */
+  @Test
+  public void testDatabaseOps() throws MetaException, InvalidObjectException, NoSuchObjectException {
+    Database db1 = new Database(DB1, "description", "locationurl", null);
+    Database db2 = new Database(DB2, "description", "locationurl", null);
+    objectStore.createDatabase(db1);
+    objectStore.createDatabase(db2);
+
+    List<String> databases = objectStore.getAllDatabases();
+    Assert.assertEquals(2, databases.size());
+    Assert.assertEquals(DB1, databases.get(0));
+    Assert.assertEquals(DB2, databases.get(1));
+
+    objectStore.dropDatabase(DB1);
+    databases = objectStore.getAllDatabases();
+    Assert.assertEquals(1, databases.size());
+    Assert.assertEquals(DB2, databases.get(0));
+
+    objectStore.dropDatabase(DB2);
+  }
+
+  /**
+   * Test table operations
+   */
+  @Test
+  public void testTableOps() throws MetaException, InvalidObjectException, NoSuchObjectException, InvalidInputException {
+    Database db1 = new Database(DB1, "description", "locationurl", null);
+    objectStore.createDatabase(db1);
+    StorageDescriptor sd = new StorageDescriptor(null, "location", null, null, false, 0, new SerDeInfo("SerDeName", "serializationLib", null), null, null, null);
+    HashMap<String,String> params = new HashMap<String,String>();
+    params.put("EXTERNAL", "false");
+    Table tbl1 = new Table(TABLE1, DB1, "owner", 1, 2, 3, sd, null, params, "viewOriginalText", "viewExpandedText", "MANAGED_TABLE");
+    objectStore.createTable(tbl1);
+
+    List<String> tables = objectStore.getAllTables(DB1);
+    Assert.assertEquals(1, tables.size());
+    Assert.assertEquals(TABLE1, tables.get(0));
+
+    Table newTbl1 = new Table("new" + TABLE1, DB1, "owner", 1, 2, 3, sd, null, params, "viewOriginalText", "viewExpandedText", "MANAGED_TABLE");
+    objectStore.alterTable(DB1, TABLE1, newTbl1);
+    tables = objectStore.getTables(DB1, "new*");
+    Assert.assertEquals(1, tables.size());
+    Assert.assertEquals("new" + TABLE1, tables.get(0));
+
+    objectStore.dropTable(DB1, "new" + TABLE1);
+    tables = objectStore.getAllTables(DB1);
+    Assert.assertEquals(0, tables.size());
+
+    objectStore.dropDatabase(DB1);
+  }
+
+  /**
+   * Tests partition operations
+   */
+  @Test
+  public void testPartitionOps() throws MetaException, InvalidObjectException, NoSuchObjectException, InvalidInputException {
+    Database db1 = new Database(DB1, "description", "locationurl", null);
+    objectStore.createDatabase(db1);
+    StorageDescriptor sd = new StorageDescriptor(null, "location", null, null, false, 0, new SerDeInfo("SerDeName", "serializationLib", null), null, null, null);
+    HashMap<String,String> tableParams = new HashMap<String,String>();
+    tableParams.put("EXTERNAL", "false");
+    FieldSchema partitionKey1 = new FieldSchema("Country", "String", "");
+    FieldSchema partitionKey2 = new FieldSchema("State", "String", "");
+    Table tbl1 = new Table(TABLE1, DB1, "owner", 1, 2, 3, sd, Arrays.asList(partitionKey1, partitionKey2), tableParams, "viewOriginalText", "viewExpandedText", "MANAGED_TABLE");
+    objectStore.createTable(tbl1);
+    HashMap<String, String> partitionParams = new HashMap<String, String>();
+    partitionParams.put("PARTITION_LEVEL_PRIVILEGE", "true");
+    List<String> value1 = Arrays.asList("US", "CA");
+    Partition part1 = new Partition(value1, DB1, TABLE1, 111, 111, sd, partitionParams);
+    objectStore.addPartition(part1);
+    List<String> value2 = Arrays.asList("US", "MA");
+    Partition part2 = new Partition(value2, DB1, TABLE1, 222, 222, sd, partitionParams);
+    objectStore.addPartition(part2);
+
+    Deadline.startTimer("getPartition");
+    List<Partition> partitions = objectStore.getPartitions(DB1, TABLE1, 10);
+    Assert.assertEquals(2, partitions.size());
+    Assert.assertEquals(111, partitions.get(0).getCreateTime());
+    Assert.assertEquals(222, partitions.get(1).getCreateTime());
+
+    objectStore.dropPartition(DB1, TABLE1, value1);
+    partitions = objectStore.getPartitions(DB1, TABLE1, 10);
+    Assert.assertEquals(1, partitions.size());
+    Assert.assertEquals(222, partitions.get(0).getCreateTime());
+
+    objectStore.dropPartition(DB1, TABLE1, value2);
+    objectStore.dropTable(DB1, TABLE1);
+    objectStore.dropDatabase(DB1);
+  }
+
+  /**
+   * Test master keys operation
+   */
+  @Test
+  public void testMasterKeyOps() throws MetaException, NoSuchObjectException {
+    int id1 = objectStore.addMasterKey(KEY1);
+    int id2 = objectStore.addMasterKey(KEY2);
+
+    String[] keys = objectStore.getMasterKeys();
+    Assert.assertEquals(2, keys.length);
+    Assert.assertEquals(KEY1, keys[0]);
+    Assert.assertEquals(KEY2, keys[1]);
+
+    objectStore.updateMasterKey(id1, "new" + KEY1);
+    objectStore.updateMasterKey(id2, "new" + KEY2);
+    keys = objectStore.getMasterKeys();
+    Assert.assertEquals(2, keys.length);
+    Assert.assertEquals("new" + KEY1, keys[0]);
+    Assert.assertEquals("new" + KEY2, keys[1]);
+
+    objectStore.removeMasterKey(id1);
+    keys = objectStore.getMasterKeys();
+    Assert.assertEquals(1, keys.length);
+    Assert.assertEquals("new" + KEY2, keys[0]);
+
+    objectStore.removeMasterKey(id2);
+  }
+
+  /**
+   * Test role operation
+   */
+  @Test
+  public void testRoleOps() throws InvalidObjectException, MetaException, NoSuchObjectException {
+    objectStore.addRole(ROLE1, OWNER);
+    objectStore.addRole(ROLE2, OWNER);
+    List<String> roles = objectStore.listRoleNames();
+    Assert.assertEquals(2, roles.size());
+    Assert.assertEquals(ROLE2, roles.get(1));
+    Role role1 = objectStore.getRole(ROLE1);
+    Assert.assertEquals(OWNER, role1.getOwnerName());
+    objectStore.grantRole(role1, USER1, PrincipalType.USER, OWNER, PrincipalType.ROLE, true);
+    objectStore.revokeRole(role1, USER1, PrincipalType.USER, false);
+    objectStore.removeRole(ROLE1);
+  }
+}