You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by ga...@apache.org on 2017/09/14 22:10:16 UTC
[1/7] hive git commit: HIVE-17488 Move first set of classes to
standalone metastore. This closes #244. (Alan Gates,
reviewed by Owen O'Malley)
Repository: hive
Updated Branches:
refs/heads/master 1c84e0c04 -> b0b6db730
http://git-wip-us.apache.org/repos/asf/hive/blob/b0b6db73/standalone-metastore/src/test/java/org/apache/hadoop/hive/metastore/TestLockRequestBuilder.java
----------------------------------------------------------------------
diff --git a/standalone-metastore/src/test/java/org/apache/hadoop/hive/metastore/TestLockRequestBuilder.java b/standalone-metastore/src/test/java/org/apache/hadoop/hive/metastore/TestLockRequestBuilder.java
new file mode 100644
index 0000000..a47c8a6
--- /dev/null
+++ b/standalone-metastore/src/test/java/org/apache/hadoop/hive/metastore/TestLockRequestBuilder.java
@@ -0,0 +1,584 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hive.metastore;
+
+import org.apache.hadoop.hive.metastore.api.LockComponent;
+import org.apache.hadoop.hive.metastore.api.LockLevel;
+import org.apache.hadoop.hive.metastore.api.LockRequest;
+import org.apache.hadoop.hive.metastore.api.LockType;
+import org.junit.Assert;
+import org.junit.Test;
+
+import java.net.InetAddress;
+import java.util.List;
+
+/**
+ * Tests for LockRequestBuilder.
+ */
+public class TestLockRequestBuilder {
+
+ // Test failure if user not set
+ @Test
+ public void noUser() {
+ LockRequestBuilder bldr = new LockRequestBuilder();
+ LockComponent comp = new LockComponent(LockType.EXCLUSIVE, LockLevel.DB, "mydb");
+ bldr.addLockComponent(comp);
+ comp = new LockComponent(LockType.EXCLUSIVE, LockLevel.DB, "mydb");
+ comp.setTablename("mytable");
+ bldr.addLockComponent(comp);
+ boolean caughtException = false;
+ try {
+ LockRequest req = bldr.build();
+ } catch (RuntimeException e) {
+ Assert.assertEquals("Cannot build a lock without giving a user", e.getMessage());
+ caughtException = true;
+ }
+ Assert.assertTrue(caughtException);
+ }
+
+ // Test that database and table don't coalesce.
+ @Test
+ public void testDbTable() throws Exception {
+ LockRequestBuilder bldr = new LockRequestBuilder();
+ LockComponent comp = new LockComponent(LockType.EXCLUSIVE, LockLevel.DB, "mydb");
+ bldr.addLockComponent(comp).setUser("fred");
+ comp = new LockComponent(LockType.EXCLUSIVE, LockLevel.DB, "mydb");
+ comp.setTablename("mytable");
+ bldr.addLockComponent(comp);
+ LockRequest req = bldr.build();
+ List<LockComponent> locks = req.getComponent();
+ Assert.assertEquals(2, locks.size());
+ Assert.assertEquals("fred", req.getUser());
+ Assert.assertEquals(InetAddress.getLocalHost().getHostName(), req.getHostname());
+ }
+
+ // Test that database and table don't coalesce.
+ @Test
+ public void testTablePartition() {
+ LockRequestBuilder bldr = new LockRequestBuilder();
+ LockComponent comp = new LockComponent(LockType.EXCLUSIVE, LockLevel.DB, "mydb");
+ comp.setTablename("mytable");
+ bldr.addLockComponent(comp).setUser(null);
+ comp = new LockComponent(LockType.EXCLUSIVE, LockLevel.DB, "mydb");
+ comp.setTablename("mytable");
+ comp.setPartitionname("mypartition");
+ bldr.addLockComponent(comp);
+ LockRequest req = bldr.build();
+ List<LockComponent> locks = req.getComponent();
+ Assert.assertEquals(2, locks.size());
+ Assert.assertEquals("unknown", req.getUser());
+ }
+
+ // Test that 2 separate databases don't coalesce.
+ @Test
+ public void testTwoSeparateDbs() {
+ LockRequestBuilder bldr = new LockRequestBuilder();
+ LockComponent comp = new LockComponent(LockType.EXCLUSIVE, LockLevel.DB, "mydb");
+ bldr.addLockComponent(comp).setUser("fred");
+ comp = new LockComponent(LockType.EXCLUSIVE, LockLevel.DB, "yourdb");
+ bldr.addLockComponent(comp);
+ LockRequest req = bldr.build();
+ List<LockComponent> locks = req.getComponent();
+ Assert.assertEquals(2, locks.size());
+ }
+
+ // Test that 2 exclusive db locks coalesce to one
+ @Test
+ public void testExExDb() {
+ LockRequestBuilder bldr = new LockRequestBuilder();
+ LockComponent comp = new LockComponent(LockType.EXCLUSIVE, LockLevel.DB, "mydb");
+ bldr.addLockComponent(comp);
+ comp = new LockComponent(LockType.EXCLUSIVE, LockLevel.DB, "mydb");
+ bldr.addLockComponent(comp).setUser("fred");
+ LockRequest req = bldr.build();
+ List<LockComponent> locks = req.getComponent();
+ Assert.assertEquals(1, locks.size());
+ }
+
+ // Test that existing exclusive db with new shared_write coalesces to
+ // exclusive
+ @Test
+ public void testExSWDb() {
+ LockRequestBuilder bldr = new LockRequestBuilder();
+ LockComponent comp = new LockComponent(LockType.EXCLUSIVE, LockLevel.DB, "mydb");
+ bldr.addLockComponent(comp).setUser("fred");
+ comp = new LockComponent(LockType.SHARED_WRITE, LockLevel.DB, "mydb");
+ bldr.addLockComponent(comp);
+ LockRequest req = bldr.build();
+ List<LockComponent> locks = req.getComponent();
+ Assert.assertEquals(1, locks.size());
+ Assert.assertEquals(LockType.EXCLUSIVE, locks.get(0).getType());
+ }
+
+ // Test that existing exclusive db with new shared_read coalesces to
+ // exclusive
+ @Test
+ public void testExSRDb() {
+ LockRequestBuilder bldr = new LockRequestBuilder();
+ LockComponent comp = new LockComponent(LockType.EXCLUSIVE, LockLevel.DB, "mydb");
+ bldr.addLockComponent(comp).setUser("fred");
+ comp = new LockComponent(LockType.SHARED_READ, LockLevel.DB, "mydb");
+ bldr.addLockComponent(comp);
+ LockRequest req = bldr.build();
+ List<LockComponent> locks = req.getComponent();
+ Assert.assertEquals(1, locks.size());
+ Assert.assertEquals(LockType.EXCLUSIVE, locks.get(0).getType());
+ }
+
+ // Test that existing shared_write db with new exclusive coalesces to
+ // exclusive
+ @Test
+ public void testSWExDb() {
+ LockRequestBuilder bldr = new LockRequestBuilder();
+ LockComponent comp = new LockComponent(LockType.SHARED_WRITE, LockLevel.DB, "mydb");
+ bldr.addLockComponent(comp);
+ comp = new LockComponent(LockType.EXCLUSIVE, LockLevel.DB, "mydb");
+ bldr.addLockComponent(comp).setUser("fred");
+ LockRequest req = bldr.build();
+ List<LockComponent> locks = req.getComponent();
+ Assert.assertEquals(1, locks.size());
+ Assert.assertEquals(LockType.EXCLUSIVE, locks.get(0).getType());
+ }
+
+ // Test that existing shared_write db with new shared_write coalesces to
+ // shared_write
+ @Test
+ public void testSWSWDb() {
+ LockRequestBuilder bldr = new LockRequestBuilder();
+ LockComponent comp = new LockComponent(LockType.SHARED_WRITE, LockLevel.DB, "mydb");
+ bldr.addLockComponent(comp);
+ comp = new LockComponent(LockType.SHARED_WRITE, LockLevel.DB, "mydb");
+ bldr.addLockComponent(comp).setUser("fred");
+ LockRequest req = bldr.build();
+ List<LockComponent> locks = req.getComponent();
+ Assert.assertEquals(1, locks.size());
+ Assert.assertEquals(LockType.SHARED_WRITE, locks.get(0).getType());
+ }
+
+ // Test that existing shared_write db with new shared_read coalesces to
+ // shared_write
+ @Test
+ public void testSWSRDb() {
+ LockRequestBuilder bldr = new LockRequestBuilder();
+ LockComponent comp = new LockComponent(LockType.SHARED_WRITE, LockLevel.DB, "mydb");
+ bldr.addLockComponent(comp);
+ comp = new LockComponent(LockType.SHARED_READ, LockLevel.DB, "mydb");
+ bldr.addLockComponent(comp).setUser("fred");
+ LockRequest req = bldr.build();
+ List<LockComponent> locks = req.getComponent();
+ Assert.assertEquals(1, locks.size());
+ Assert.assertEquals(LockType.SHARED_WRITE, locks.get(0).getType());
+ }
+
+ // Test that existing shared_read db with new exclusive coalesces to
+ // exclusive
+ @Test
+ public void testSRExDb() {
+ LockRequestBuilder bldr = new LockRequestBuilder();
+ LockComponent comp = new LockComponent(LockType.SHARED_READ, LockLevel.DB, "mydb");
+ bldr.addLockComponent(comp);
+ comp = new LockComponent(LockType.EXCLUSIVE, LockLevel.DB, "mydb");
+ bldr.addLockComponent(comp).setUser("fred");
+ LockRequest req = bldr.build();
+ List<LockComponent> locks = req.getComponent();
+ Assert.assertEquals(1, locks.size());
+ Assert.assertEquals(LockType.EXCLUSIVE, locks.get(0).getType());
+ }
+
+ // Test that existing shared_read db with new shared_write coalesces to
+ // shared_write
+ @Test
+ public void testSRSWDb() {
+ LockRequestBuilder bldr = new LockRequestBuilder();
+ LockComponent comp = new LockComponent(LockType.SHARED_READ, LockLevel.DB, "mydb");
+ bldr.addLockComponent(comp);
+ comp = new LockComponent(LockType.SHARED_WRITE, LockLevel.DB, "mydb");
+ bldr.addLockComponent(comp).setUser("fred");
+ LockRequest req = bldr.build();
+ List<LockComponent> locks = req.getComponent();
+ Assert.assertEquals(1, locks.size());
+ Assert.assertEquals(LockType.SHARED_WRITE, locks.get(0).getType());
+ }
+
+ // Test that existing shared_read db with new shared_read coalesces to
+ // shared_read
+ @Test
+ public void testSRSRDb() {
+ LockRequestBuilder bldr = new LockRequestBuilder();
+ LockComponent comp = new LockComponent(LockType.SHARED_READ, LockLevel.DB, "mydb");
+ bldr.addLockComponent(comp);
+ comp = new LockComponent(LockType.SHARED_READ, LockLevel.DB, "mydb");
+ bldr.addLockComponent(comp).setUser("fred");
+ LockRequest req = bldr.build();
+ List<LockComponent> locks = req.getComponent();
+ Assert.assertEquals(1, locks.size());
+ Assert.assertEquals(LockType.SHARED_READ, locks.get(0).getType());
+ }
+
+ // Test that 2 separate tables don't coalesce.
+ @Test
+ public void testTwoSeparateTables() {
+ LockRequestBuilder bldr = new LockRequestBuilder();
+ LockComponent comp = new LockComponent(LockType.EXCLUSIVE, LockLevel.DB, "mydb");
+ comp.setTablename("mytable");
+ bldr.addLockComponent(comp);
+ comp = new LockComponent(LockType.EXCLUSIVE, LockLevel.DB, "mydb");
+ comp.setTablename("yourtable");
+ bldr.addLockComponent(comp).setUser("fred");
+ LockRequest req = bldr.build();
+ List<LockComponent> locks = req.getComponent();
+ Assert.assertEquals(2, locks.size());
+ }
+
+ // Test that 2 exclusive table locks coalesce to one
+ @Test
+ public void testExExTable() {
+ LockRequestBuilder bldr = new LockRequestBuilder();
+ LockComponent comp = new LockComponent(LockType.EXCLUSIVE, LockLevel.DB, "mydb");
+ comp.setTablename("mytable");
+ bldr.addLockComponent(comp);
+ comp = new LockComponent(LockType.EXCLUSIVE, LockLevel.DB, "mydb");
+ comp.setTablename("mytable");
+ bldr.addLockComponent(comp).setUser("fred");
+ LockRequest req = bldr.build();
+ List<LockComponent> locks = req.getComponent();
+ Assert.assertEquals(1, locks.size());
+ }
+
+ // Test that existing exclusive table with new shared_write coalesces to
+ // exclusive
+ @Test
+ public void testExSWTable() {
+ LockRequestBuilder bldr = new LockRequestBuilder();
+ LockComponent comp = new LockComponent(LockType.EXCLUSIVE, LockLevel.DB, "mydb");
+ comp.setTablename("mytable");
+ bldr.addLockComponent(comp);
+ comp = new LockComponent(LockType.SHARED_WRITE, LockLevel.DB, "mydb");
+ comp.setTablename("mytable");
+ bldr.addLockComponent(comp).setUser("fred");
+ LockRequest req = bldr.build();
+ List<LockComponent> locks = req.getComponent();
+ Assert.assertEquals(1, locks.size());
+ Assert.assertEquals(LockType.EXCLUSIVE, locks.get(0).getType());
+ }
+
+ // Test that existing exclusive table with new shared_read coalesces to
+ // exclusive
+ @Test
+ public void testExSRTable() {
+ LockRequestBuilder bldr = new LockRequestBuilder();
+ LockComponent comp = new LockComponent(LockType.EXCLUSIVE, LockLevel.DB, "mydb");
+ comp.setTablename("mytable");
+ bldr.addLockComponent(comp);
+ comp = new LockComponent(LockType.SHARED_READ, LockLevel.DB, "mydb");
+ comp.setTablename("mytable");
+ bldr.addLockComponent(comp).setUser("fred");
+ LockRequest req = bldr.build();
+ List<LockComponent> locks = req.getComponent();
+ Assert.assertEquals(1, locks.size());
+ Assert.assertEquals(LockType.EXCLUSIVE, locks.get(0).getType());
+ }
+
+ // Test that existing shared_write table with new exclusive coalesces to
+ // exclusive
+ @Test
+ public void testSWExTable() {
+ LockRequestBuilder bldr = new LockRequestBuilder();
+ LockComponent comp = new LockComponent(LockType.SHARED_WRITE, LockLevel.DB, "mydb");
+ comp.setTablename("mytable");
+ bldr.addLockComponent(comp);
+ comp = new LockComponent(LockType.EXCLUSIVE, LockLevel.DB, "mydb");
+ comp.setTablename("mytable");
+ bldr.addLockComponent(comp).setUser("fred");
+ LockRequest req = bldr.build();
+ List<LockComponent> locks = req.getComponent();
+ Assert.assertEquals(1, locks.size());
+ Assert.assertEquals(LockType.EXCLUSIVE, locks.get(0).getType());
+ }
+
+ // Test that existing shared_write table with new shared_write coalesces to
+ // shared_write
+ @Test
+ public void testSWSWTable() {
+ LockRequestBuilder bldr = new LockRequestBuilder();
+ LockComponent comp = new LockComponent(LockType.SHARED_WRITE, LockLevel.DB, "mydb");
+ comp.setTablename("mytable");
+ bldr.addLockComponent(comp);
+ comp = new LockComponent(LockType.SHARED_WRITE, LockLevel.DB, "mydb");
+ comp.setTablename("mytable");
+ bldr.addLockComponent(comp).setUser("fred");
+ LockRequest req = bldr.build();
+ List<LockComponent> locks = req.getComponent();
+ Assert.assertEquals(1, locks.size());
+ Assert.assertEquals(LockType.SHARED_WRITE, locks.get(0).getType());
+ }
+
+ // Test that existing shared_write table with new shared_read coalesces to
+ // shared_write
+ @Test
+ public void testSWSRTable() {
+ LockRequestBuilder bldr = new LockRequestBuilder();
+ LockComponent comp = new LockComponent(LockType.SHARED_WRITE, LockLevel.DB, "mydb");
+ comp.setTablename("mytable");
+ bldr.addLockComponent(comp);
+ comp = new LockComponent(LockType.SHARED_READ, LockLevel.DB, "mydb");
+ comp.setTablename("mytable");
+ bldr.addLockComponent(comp).setUser("fred");
+ LockRequest req = bldr.build();
+ List<LockComponent> locks = req.getComponent();
+ Assert.assertEquals(1, locks.size());
+ Assert.assertEquals(LockType.SHARED_WRITE, locks.get(0).getType());
+ }
+
+ // Test that existing shared_read table with new exclusive coalesces to
+ // exclusive
+ @Test
+ public void testSRExTable() {
+ LockRequestBuilder bldr = new LockRequestBuilder();
+ LockComponent comp = new LockComponent(LockType.SHARED_READ, LockLevel.DB, "mydb");
+ comp.setTablename("mytable");
+ bldr.addLockComponent(comp);
+ comp = new LockComponent(LockType.EXCLUSIVE, LockLevel.DB, "mydb");
+ comp.setTablename("mytable");
+ bldr.addLockComponent(comp).setUser("fred");
+ LockRequest req = bldr.build();
+ List<LockComponent> locks = req.getComponent();
+ Assert.assertEquals(1, locks.size());
+ Assert.assertEquals(LockType.EXCLUSIVE, locks.get(0).getType());
+ }
+
+ // Test that existing shared_read table with new shared_write coalesces to
+ // shared_write
+ @Test
+ public void testSRSWTable() {
+ LockRequestBuilder bldr = new LockRequestBuilder();
+ LockComponent comp = new LockComponent(LockType.SHARED_READ, LockLevel.DB, "mydb");
+ comp.setTablename("mytable");
+ bldr.addLockComponent(comp);
+ comp = new LockComponent(LockType.SHARED_WRITE, LockLevel.DB, "mydb");
+ comp.setTablename("mytable");
+ bldr.addLockComponent(comp).setUser("fred");
+ LockRequest req = bldr.build();
+ List<LockComponent> locks = req.getComponent();
+ Assert.assertEquals(1, locks.size());
+ Assert.assertEquals(LockType.SHARED_WRITE, locks.get(0).getType());
+ }
+
+ // Test that existing shared_read table with new shared_read coalesces to
+ // shared_read
+ @Test
+ public void testSRSRTable() {
+ LockRequestBuilder bldr = new LockRequestBuilder();
+ LockComponent comp = new LockComponent(LockType.SHARED_READ, LockLevel.DB, "mydb");
+ comp.setTablename("mytable");
+ bldr.addLockComponent(comp);
+ comp = new LockComponent(LockType.SHARED_READ, LockLevel.DB, "mydb");
+ comp.setTablename("mytable");
+ bldr.addLockComponent(comp).setUser("fred");
+ LockRequest req = bldr.build();
+ List<LockComponent> locks = req.getComponent();
+ Assert.assertEquals(1, locks.size());
+ Assert.assertEquals(LockType.SHARED_READ, locks.get(0).getType());
+ }
+
+ // Test that 2 separate partitions don't coalesce.
+ @Test
+ public void testTwoSeparatePartitions() {
+ LockRequestBuilder bldr = new LockRequestBuilder();
+ LockComponent comp = new LockComponent(LockType.EXCLUSIVE, LockLevel.DB, "mydb");
+ comp.setTablename("mytable");
+ comp.setPartitionname("mypart");
+ bldr.addLockComponent(comp);
+ comp = new LockComponent(LockType.EXCLUSIVE, LockLevel.DB, "mydb");
+ comp.setTablename("mytable");
+ comp.setPartitionname("yourpart");
+ bldr.addLockComponent(comp).setUser("fred");
+ LockRequest req = bldr.build();
+ List<LockComponent> locks = req.getComponent();
+ Assert.assertEquals(2, locks.size());
+ }
+
+ // Test that 2 exclusive partition locks coalesce to one
+ @Test
+ public void testExExPart() {
+ LockRequestBuilder bldr = new LockRequestBuilder();
+ LockComponent comp = new LockComponent(LockType.EXCLUSIVE, LockLevel.DB, "mydb");
+ comp.setTablename("mytable");
+ comp.setPartitionname("mypart");
+ bldr.addLockComponent(comp);
+ comp = new LockComponent(LockType.EXCLUSIVE, LockLevel.DB, "mydb");
+ comp.setTablename("mytable");
+ comp.setPartitionname("mypart");
+ bldr.addLockComponent(comp).setUser("fred");
+ LockRequest req = bldr.build();
+ List<LockComponent> locks = req.getComponent();
+ Assert.assertEquals(1, locks.size());
+ }
+
+ // Test that existing exclusive partition with new shared_write coalesces to
+ // exclusive
+ @Test
+ public void testExSWPart() {
+ LockRequestBuilder bldr = new LockRequestBuilder();
+ LockComponent comp = new LockComponent(LockType.EXCLUSIVE, LockLevel.DB, "mydb");
+ comp.setTablename("mytable");
+ comp.setPartitionname("mypart");
+ bldr.addLockComponent(comp);
+ comp = new LockComponent(LockType.SHARED_WRITE, LockLevel.DB, "mydb");
+ comp.setTablename("mytable");
+ comp.setPartitionname("mypart");
+ bldr.addLockComponent(comp).setUser("fred");
+ LockRequest req = bldr.build();
+ List<LockComponent> locks = req.getComponent();
+ Assert.assertEquals(1, locks.size());
+ Assert.assertEquals(LockType.EXCLUSIVE, locks.get(0).getType());
+ }
+
+ // Test that existing exclusive partition with new shared_read coalesces to
+ // exclusive
+ @Test
+ public void testExSRPart() {
+ LockRequestBuilder bldr = new LockRequestBuilder();
+ LockComponent comp = new LockComponent(LockType.EXCLUSIVE, LockLevel.DB, "mydb");
+ comp.setTablename("mytable");
+ comp.setPartitionname("mypart");
+ bldr.addLockComponent(comp);
+ comp = new LockComponent(LockType.SHARED_READ, LockLevel.DB, "mydb");
+ comp.setTablename("mytable");
+ comp.setPartitionname("mypart");
+ bldr.addLockComponent(comp).setUser("fred");
+ LockRequest req = bldr.build();
+ List<LockComponent> locks = req.getComponent();
+ Assert.assertEquals(1, locks.size());
+ Assert.assertEquals(LockType.EXCLUSIVE, locks.get(0).getType());
+ }
+
+ // Test that existing shared_write partition with new exclusive coalesces to
+ // exclusive
+ @Test
+ public void testSWExPart() {
+ LockRequestBuilder bldr = new LockRequestBuilder();
+ LockComponent comp = new LockComponent(LockType.SHARED_WRITE, LockLevel.DB, "mydb");
+ comp.setTablename("mytable");
+ comp.setPartitionname("mypart");
+ bldr.addLockComponent(comp);
+ comp = new LockComponent(LockType.EXCLUSIVE, LockLevel.DB, "mydb");
+ comp.setTablename("mytable");
+ comp.setPartitionname("mypart");
+ bldr.addLockComponent(comp).setUser("fred");
+ LockRequest req = bldr.build();
+ List<LockComponent> locks = req.getComponent();
+ Assert.assertEquals(1, locks.size());
+ Assert.assertEquals(LockType.EXCLUSIVE, locks.get(0).getType());
+ }
+
+ // Test that existing shared_write partition with new shared_write coalesces to
+ // shared_write
+ @Test
+ public void testSWSWPart() {
+ LockRequestBuilder bldr = new LockRequestBuilder();
+ LockComponent comp = new LockComponent(LockType.SHARED_WRITE, LockLevel.DB, "mydb");
+ comp.setTablename("mytable");
+ comp.setPartitionname("mypart");
+ bldr.addLockComponent(comp);
+ comp = new LockComponent(LockType.SHARED_WRITE, LockLevel.DB, "mydb");
+ comp.setTablename("mytable");
+ comp.setPartitionname("mypart");
+ bldr.addLockComponent(comp).setUser("fred");
+ LockRequest req = bldr.build();
+ List<LockComponent> locks = req.getComponent();
+ Assert.assertEquals(1, locks.size());
+ Assert.assertEquals(LockType.SHARED_WRITE, locks.get(0).getType());
+ }
+
+ // Test that existing shared_write partition with new shared_read coalesces to
+ // shared_write
+ @Test
+ public void testSWSRPart() {
+ LockRequestBuilder bldr = new LockRequestBuilder();
+ LockComponent comp = new LockComponent(LockType.SHARED_WRITE, LockLevel.DB, "mydb");
+ comp.setTablename("mytable");
+ comp.setPartitionname("mypart");
+ bldr.addLockComponent(comp);
+ comp = new LockComponent(LockType.SHARED_READ, LockLevel.DB, "mydb");
+ comp.setTablename("mytable");
+ comp.setPartitionname("mypart");
+ bldr.addLockComponent(comp).setUser("fred");
+ LockRequest req = bldr.build();
+ List<LockComponent> locks = req.getComponent();
+ Assert.assertEquals(1, locks.size());
+ Assert.assertEquals(LockType.SHARED_WRITE, locks.get(0).getType());
+ }
+
+ // Test that existing shared_read partition with new exclusive coalesces to
+ // exclusive
+ @Test
+ public void testSRExPart() {
+ LockRequestBuilder bldr = new LockRequestBuilder();
+ LockComponent comp = new LockComponent(LockType.SHARED_READ, LockLevel.DB, "mydb");
+ comp.setTablename("mytable");
+ comp.setPartitionname("mypart");
+ bldr.addLockComponent(comp);
+ comp = new LockComponent(LockType.EXCLUSIVE, LockLevel.DB, "mydb");
+ comp.setTablename("mytable");
+ comp.setPartitionname("mypart");
+ bldr.addLockComponent(comp).setUser("fred");
+ LockRequest req = bldr.build();
+ List<LockComponent> locks = req.getComponent();
+ Assert.assertEquals(1, locks.size());
+ Assert.assertEquals(LockType.EXCLUSIVE, locks.get(0).getType());
+ }
+
+ // Test that existing shared_read partition with new shared_write coalesces to
+ // shared_write
+ @Test
+ public void testSRSWPart() {
+ LockRequestBuilder bldr = new LockRequestBuilder();
+ LockComponent comp = new LockComponent(LockType.SHARED_READ, LockLevel.DB, "mydb");
+ comp.setTablename("mytable");
+ comp.setPartitionname("mypart");
+ bldr.addLockComponent(comp);
+ comp = new LockComponent(LockType.SHARED_WRITE, LockLevel.DB, "mydb");
+ comp.setTablename("mytable");
+ comp.setPartitionname("mypart");
+ bldr.addLockComponent(comp).setUser("fred");
+ LockRequest req = bldr.build();
+ List<LockComponent> locks = req.getComponent();
+ Assert.assertEquals(1, locks.size());
+ Assert.assertEquals(LockType.SHARED_WRITE, locks.get(0).getType());
+ }
+
+ // Test that existing shared_read partition with new shared_read coalesces to
+ // shared_read
+ @Test
+ public void testSRSRPart() {
+ LockRequestBuilder bldr = new LockRequestBuilder();
+ LockComponent comp = new LockComponent(LockType.SHARED_READ, LockLevel.DB, "mydb");
+ comp.setTablename("mytable");
+ comp.setPartitionname("mypart");
+ bldr.addLockComponent(comp);
+ comp = new LockComponent(LockType.SHARED_READ, LockLevel.DB, "mydb");
+ comp.setTablename("mytable");
+ comp.setPartitionname("mypart");
+ bldr.addLockComponent(comp).setUser("fred");
+ LockRequest req = bldr.build();
+ List<LockComponent> locks = req.getComponent();
+ Assert.assertEquals(1, locks.size());
+ Assert.assertEquals(LockType.SHARED_READ, locks.get(0).getType());
+ }
+}
http://git-wip-us.apache.org/repos/asf/hive/blob/b0b6db73/standalone-metastore/src/test/java/org/apache/hadoop/hive/metastore/TestMetaStoreSchemaFactory.java
----------------------------------------------------------------------
diff --git a/standalone-metastore/src/test/java/org/apache/hadoop/hive/metastore/TestMetaStoreSchemaFactory.java b/standalone-metastore/src/test/java/org/apache/hadoop/hive/metastore/TestMetaStoreSchemaFactory.java
new file mode 100644
index 0000000..1dac3ed
--- /dev/null
+++ b/standalone-metastore/src/test/java/org/apache/hadoop/hive/metastore/TestMetaStoreSchemaFactory.java
@@ -0,0 +1,69 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hive.metastore;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hive.metastore.conf.MetastoreConf;
+import org.apache.hadoop.hive.metastore.conf.MetastoreConf.ConfVars;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Test;
+
+public class TestMetaStoreSchemaFactory {
+ private Configuration conf;
+
+ @Before
+ public void setup() {
+ conf = MetastoreConf.newMetastoreConf();
+ }
+
+ @Test
+ public void testDefaultConfig() {
+ IMetaStoreSchemaInfo metastoreSchemaInfo = MetaStoreSchemaInfoFactory.get(conf);
+ Assert.assertNotNull(metastoreSchemaInfo);
+ }
+
+ @Test
+ public void testWithConfigSet() {
+ MetastoreConf.setVar(conf, ConfVars.SCHEMA_INFO_CLASS,
+ MetaStoreSchemaInfo.class.getCanonicalName());
+ IMetaStoreSchemaInfo metastoreSchemaInfo = MetaStoreSchemaInfoFactory.get(conf);
+ Assert.assertNotNull(metastoreSchemaInfo);
+ Assert.assertTrue("Unexpected instance type of the class MetaStoreSchemaInfo",
+ metastoreSchemaInfo instanceof MetaStoreSchemaInfo);
+ }
+
+ @Test
+ public void testConstructor() {
+ String className = MetastoreConf.getVar(conf, ConfVars.SCHEMA_INFO_CLASS,
+ MetaStoreSchemaInfo.class.getCanonicalName());
+ Class<?> clasz = null;
+ try {
+ clasz = conf.getClassByName(className);
+ clasz.getConstructor(String.class, String.class);
+ } catch (NoSuchMethodException | IllegalArgumentException | ClassNotFoundException e) {
+ throw new IllegalArgumentException(e);
+ }
+ }
+
+ @Test(expected = IllegalArgumentException.class)
+ public void testInvalidClassName() {
+ MetastoreConf.setVar(conf, ConfVars.SCHEMA_INFO_CLASS, "invalid.class.name");
+ MetaStoreSchemaInfoFactory.get(conf);
+ }
+}
http://git-wip-us.apache.org/repos/asf/hive/blob/b0b6db73/standalone-metastore/src/test/java/org/apache/hadoop/hive/metastore/TestMetaStoreSchemaInfo.java
----------------------------------------------------------------------
diff --git a/standalone-metastore/src/test/java/org/apache/hadoop/hive/metastore/TestMetaStoreSchemaInfo.java b/standalone-metastore/src/test/java/org/apache/hadoop/hive/metastore/TestMetaStoreSchemaInfo.java
new file mode 100644
index 0000000..8485e99
--- /dev/null
+++ b/standalone-metastore/src/test/java/org/apache/hadoop/hive/metastore/TestMetaStoreSchemaInfo.java
@@ -0,0 +1,52 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hive.metastore;
+
+import org.apache.hadoop.hive.metastore.conf.MetastoreConf;
+import org.junit.Assert;
+import org.junit.Test;
+
+/**
+ * Test MetaStoreSchemaInfo
+ */
+public class TestMetaStoreSchemaInfo {
+
+ @Test
+ public void testIsVersionCompatible() throws Exception {
+ // first argument is hiveVersion, it is compatible if 2nd argument - dbVersion is
+ // greater than or equal to it
+ // check the compatible case
+ IMetaStoreSchemaInfo metastoreSchemaInfo =
+ MetaStoreSchemaInfoFactory.get(MetastoreConf.newMetastoreConf());
+ Assert.assertTrue(metastoreSchemaInfo.isVersionCompatible("0.0.1", "0.0.1"));
+ Assert.assertTrue(metastoreSchemaInfo.isVersionCompatible("0.0.1", "0.0.2"));
+ Assert.assertTrue(metastoreSchemaInfo.isVersionCompatible("1.0.2", "2.0.1"));
+ Assert.assertTrue(metastoreSchemaInfo.isVersionCompatible("0.0.9", "9.0.0"));
+
+ // check equivalent versions, should be compatible
+ Assert.assertTrue(metastoreSchemaInfo.isVersionCompatible("0.13.0", "0.13.1"));
+ Assert.assertTrue(metastoreSchemaInfo.isVersionCompatible("0.13.1", "0.13.0"));
+
+ // check incompatible versions
+ Assert.assertFalse(metastoreSchemaInfo.isVersionCompatible("0.1.1", "0.1.0"));
+ Assert.assertFalse(metastoreSchemaInfo.isVersionCompatible("4.0.1", "0.1.0"));
+
+ }
+
+}
[4/7] hive git commit: HIVE-17488 Move first set of classes to
standalone metastore. This closes #244. (Alan Gates,
reviewed by Owen O'Malley)
Posted by ga...@apache.org.
http://git-wip-us.apache.org/repos/asf/hive/blob/b0b6db73/metastore/src/test/org/apache/hadoop/hive/metastore/TestAggregateStatsCache.java
----------------------------------------------------------------------
diff --git a/metastore/src/test/org/apache/hadoop/hive/metastore/TestAggregateStatsCache.java b/metastore/src/test/org/apache/hadoop/hive/metastore/TestAggregateStatsCache.java
deleted file mode 100644
index 40700da..0000000
--- a/metastore/src/test/org/apache/hadoop/hive/metastore/TestAggregateStatsCache.java
+++ /dev/null
@@ -1,266 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.hadoop.hive.metastore;
-
-import java.util.ArrayList;
-import java.util.List;
-
-import org.apache.hadoop.hive.conf.HiveConf;
-import org.apache.hadoop.hive.metastore.api.ColumnStatisticsData;
-import org.apache.hadoop.hive.metastore.api.ColumnStatisticsObj;
-import org.apache.hadoop.hive.metastore.api.LongColumnStatsData;
-import org.apache.hadoop.hive.metastore.AggregateStatsCache.AggrColStats;
-import org.apache.hadoop.hive.metastore.AggregateStatsCache.Key;
-import org.apache.hive.common.util.BloomFilter;
-import org.junit.After;
-import org.junit.AfterClass;
-import org.junit.Assert;
-import org.junit.Before;
-import org.junit.BeforeClass;
-import org.junit.Test;
-
-public class TestAggregateStatsCache {
- static String DB_NAME = "db";
- static String TAB_PREFIX = "tab";
- static String PART_PREFIX = "part";
- static String COL_PREFIX = "col";
- static int NUM_TABS = 2;
- static int NUM_PARTS = 20;
- static int NUM_COLS = 5;
- static int MAX_CACHE_NODES = 10;
- static int MAX_PARTITIONS_PER_CACHE_NODE = 10;
- static String TIME_TO_LIVE = "20s";
- static String MAX_WRITER_WAIT = "1s";
- static String MAX_READER_WAIT = "1s";
- static float FALSE_POSITIVE_PROBABILITY = (float) 0.01;
- static float MAX_VARIANCE = (float) 0.5;
- static AggregateStatsCache cache;
- static List<String> tables = new ArrayList<String>();
- static List<String> tabParts = new ArrayList<String>();
- static List<String> tabCols = new ArrayList<String>();
-
- @BeforeClass
- public static void beforeTest() {
- // All data intitializations
- initializeTables();
- initializePartitions();
- initializeColumns();
- }
-
- // tab1, tab2
- private static void initializeTables() {
- for (int i = 1; i <= NUM_TABS; i++) {
- tables.add(TAB_PREFIX + i);
- }
- }
-
- // part1 ... part20
- private static void initializePartitions() {
- for (int i = 1; i <= NUM_PARTS; i++) {
- tabParts.add(PART_PREFIX + i);
- }
- }
-
- // col1 ... col5
- private static void initializeColumns() {
- for (int i = 1; i <= NUM_COLS; i++) {
- tabCols.add(COL_PREFIX + i);
- }
- }
-
- @AfterClass
- public static void afterTest() {
- }
-
- @Before
- public void setUp() {
- HiveConf hiveConf = new HiveConf();
- hiveConf.setIntVar(HiveConf.ConfVars.METASTORE_AGGREGATE_STATS_CACHE_SIZE,
- MAX_CACHE_NODES);
- hiveConf.setIntVar(HiveConf.ConfVars.METASTORE_AGGREGATE_STATS_CACHE_MAX_PARTITIONS,
- MAX_PARTITIONS_PER_CACHE_NODE);
- hiveConf.setFloatVar(
- HiveConf.ConfVars.METASTORE_AGGREGATE_STATS_CACHE_FPP,
- FALSE_POSITIVE_PROBABILITY);
- hiveConf.setFloatVar(HiveConf.ConfVars.METASTORE_AGGREGATE_STATS_CACHE_MAX_VARIANCE,
- MAX_VARIANCE);
- hiveConf.setVar(HiveConf.ConfVars.METASTORE_AGGREGATE_STATS_CACHE_TTL, TIME_TO_LIVE);
- hiveConf.setVar(HiveConf.ConfVars.METASTORE_AGGREGATE_STATS_CACHE_MAX_WRITER_WAIT, MAX_WRITER_WAIT);
- hiveConf.setVar(HiveConf.ConfVars.METASTORE_AGGREGATE_STATS_CACHE_MAX_READER_WAIT, MAX_READER_WAIT);
- cache = AggregateStatsCache.getInstance(hiveConf);
- }
-
- @After
- public void tearDown() {
- }
-
- @Test
- public void testCacheKey() {
- Key k1 = new Key("db", "tbl1", "col");
- Key k2 = new Key("db", "tbl1", "col");
- // k1 equals k2
- Assert.assertEquals(k1, k2);
- Key k3 = new Key("db", "tbl2", "col");
- // k1 not equals k3
- Assert.assertNotEquals(k1, k3);
- }
-
- @Test
- public void testBasicAddAndGet() throws Exception {
- // Partnames: [tab1part1...tab1part9]
- List<String> partNames = preparePartNames(tables.get(0), 1, 9);
- // Prepare the bloom filter
- BloomFilter bloomFilter = prepareBloomFilter(partNames);
- // Add a dummy aggregate stats object for the above parts (part1...part9) of tab1 for col1
- String tblName = tables.get(0);
- String colName = tabCols.get(0);
- int highVal = 100, lowVal = 10, numDVs = 50, numNulls = 5;
- // We'll treat this as the aggregate col stats for part1...part9 of tab1, col1
- ColumnStatisticsObj aggrColStats =
- getDummyLongColStat(colName, highVal, lowVal, numDVs, numNulls);
- // Now add to cache the dummy colstats for these 10 partitions
- cache.add(DB_NAME, tblName, colName, 10, aggrColStats, bloomFilter);
- // Now get from cache
- AggrColStats aggrStatsCached = cache.get(DB_NAME, tblName, colName, partNames);
- Assert.assertNotNull(aggrStatsCached);
-
- ColumnStatisticsObj aggrColStatsCached = aggrStatsCached.getColStats();
- Assert.assertEquals(aggrColStats, aggrColStatsCached);
-
- // Now get a non-existant entry
- aggrStatsCached = cache.get("dbNotThere", tblName, colName, partNames);
- Assert.assertNull(aggrStatsCached);
- }
-
- @Test
- public void testAddGetWithVariance() throws Exception {
- // Partnames: [tab1part1...tab1part9]
- List<String> partNames = preparePartNames(tables.get(0), 1, 9);
- // Prepare the bloom filter
- BloomFilter bloomFilter = prepareBloomFilter(partNames);
- // Add a dummy aggregate stats object for the above parts (part1...part9) of tab1 for col1
- String tblName = tables.get(0);
- String colName = tabCols.get(0);
- int highVal = 100, lowVal = 10, numDVs = 50, numNulls = 5;
- // We'll treat this as the aggregate col stats for part1...part9 of tab1, col1
- ColumnStatisticsObj aggrColStats =
- getDummyLongColStat(colName, highVal, lowVal, numDVs, numNulls);
- // Now add to cache
- cache.add(DB_NAME, tblName, colName, 10, aggrColStats, bloomFilter);
-
- // Now prepare partnames with only 5 partitions: [tab1part1...tab1part5]
- partNames = preparePartNames(tables.get(0), 1, 5);
- // This get should fail because its variance ((10-5)/5) is way past MAX_VARIANCE (0.5)
- AggrColStats aggrStatsCached = cache.get(DB_NAME, tblName, colName, partNames);
- Assert.assertNull(aggrStatsCached);
-
- // Now prepare partnames with 10 partitions: [tab1part11...tab1part20], but with no overlap
- partNames = preparePartNames(tables.get(0), 11, 20);
- // This get should fail because its variance ((10-0)/10) is way past MAX_VARIANCE (0.5)
- aggrStatsCached = cache.get(DB_NAME, tblName, colName, partNames);
- Assert.assertNull(aggrStatsCached);
-
- // Now prepare partnames with 9 partitions: [tab1part1...tab1part8], which are contained in the
- // object that we added to the cache
- partNames = preparePartNames(tables.get(0), 1, 8);
- // This get should succeed because its variance ((10-9)/9) is within past MAX_VARIANCE (0.5)
- aggrStatsCached = cache.get(DB_NAME, tblName, colName, partNames);
- Assert.assertNotNull(aggrStatsCached);
- ColumnStatisticsObj aggrColStatsCached = aggrStatsCached.getColStats();
- Assert.assertEquals(aggrColStats, aggrColStatsCached);
- }
-
- @Test
- public void testTimeToLive() throws Exception {
- // Add a dummy node to cache
- // Partnames: [tab1part1...tab1part9]
- List<String> partNames = preparePartNames(tables.get(0), 1, 9);
- // Prepare the bloom filter
- BloomFilter bloomFilter = prepareBloomFilter(partNames);
- // Add a dummy aggregate stats object for the above parts (part1...part9) of tab1 for col1
- String tblName = tables.get(0);
- String colName = tabCols.get(0);
- int highVal = 100, lowVal = 10, numDVs = 50, numNulls = 5;
- // We'll treat this as the aggregate col stats for part1...part9 of tab1, col1
- ColumnStatisticsObj aggrColStats =
- getDummyLongColStat(colName, highVal, lowVal, numDVs, numNulls);
- // Now add to cache
- cache.add(DB_NAME, tblName, colName, 10, aggrColStats, bloomFilter);
-
- // Sleep for 30 seconds
- Thread.sleep(30000);
-
- // Get should fail now (since TTL is 20s) and we've snoozed for 30 seconds
- AggrColStats aggrStatsCached = cache.get(DB_NAME, tblName, colName, partNames);
- Assert.assertNull(aggrStatsCached);
- }
-
- /**
- * Prepares an array of partition names by getting partitions from minPart ... maxPart and
- * prepending with table name
- * Example: [tab1part1, tab1part2 ...]
- *
- * @param tabName
- * @param minPart
- * @param maxPart
- * @return
- * @throws Exception
- */
- private List<String> preparePartNames(String tabName, int minPart, int maxPart) throws Exception {
- if ((minPart < 1) || (maxPart > NUM_PARTS)) {
- throw new Exception("tabParts does not have these partition numbers");
- }
- List<String> partNames = new ArrayList<String>();
- for (int i = minPart; i <= maxPart; i++) {
- String partName = tabParts.get(i-1);
- partNames.add(tabName + partName);
- }
- return partNames;
- }
-
- /**
- * Prepares a bloom filter from the list of partition names
- * @param partNames
- * @return
- */
- private BloomFilter prepareBloomFilter(List <String> partNames) {
- BloomFilter bloomFilter =
- new BloomFilter(MAX_PARTITIONS_PER_CACHE_NODE, FALSE_POSITIVE_PROBABILITY);
- for (String partName: partNames) {
- bloomFilter.add(partName.getBytes());
- }
- return bloomFilter;
- }
-
- private ColumnStatisticsObj getDummyLongColStat(String colName, int highVal, int lowVal, int numDVs, int numNulls) {
- ColumnStatisticsObj aggrColStats = new ColumnStatisticsObj();
- aggrColStats.setColName(colName);
- aggrColStats.setColType("long");
- LongColumnStatsData longStatsData = new LongColumnStatsData();
- longStatsData.setHighValue(highVal);
- longStatsData.setLowValue(lowVal);
- longStatsData.setNumDVs(numDVs);
- longStatsData.setNumNulls(numNulls);
- ColumnStatisticsData aggrColStatsData = new ColumnStatisticsData();
- aggrColStatsData.setLongStats(longStatsData);
- aggrColStats.setStatsData(aggrColStatsData);
- return aggrColStats;
- }
-}
http://git-wip-us.apache.org/repos/asf/hive/blob/b0b6db73/metastore/src/test/org/apache/hadoop/hive/metastore/TestLockRequestBuilder.java
----------------------------------------------------------------------
diff --git a/metastore/src/test/org/apache/hadoop/hive/metastore/TestLockRequestBuilder.java b/metastore/src/test/org/apache/hadoop/hive/metastore/TestLockRequestBuilder.java
deleted file mode 100644
index 0964826..0000000
--- a/metastore/src/test/org/apache/hadoop/hive/metastore/TestLockRequestBuilder.java
+++ /dev/null
@@ -1,584 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.hive.metastore;
-
-import junit.framework.Assert;
-import org.apache.hadoop.hive.metastore.api.LockComponent;
-import org.apache.hadoop.hive.metastore.api.LockLevel;
-import org.apache.hadoop.hive.metastore.api.LockRequest;
-import org.apache.hadoop.hive.metastore.api.LockType;
-import org.junit.Test;
-
-import java.net.InetAddress;
-import java.util.List;
-
-/**
- * Tests for LockRequestBuilder.
- */
-public class TestLockRequestBuilder {
-
- // Test failure if user not set
- @Test
- public void noUser() {
- LockRequestBuilder bldr = new LockRequestBuilder();
- LockComponent comp = new LockComponent(LockType.EXCLUSIVE, LockLevel.DB, "mydb");
- bldr.addLockComponent(comp);
- comp = new LockComponent(LockType.EXCLUSIVE, LockLevel.DB, "mydb");
- comp.setTablename("mytable");
- bldr.addLockComponent(comp);
- boolean caughtException = false;
- try {
- LockRequest req = bldr.build();
- } catch (RuntimeException e) {
- Assert.assertEquals("Cannot build a lock without giving a user", e.getMessage());
- caughtException = true;
- }
- Assert.assertTrue(caughtException);
- }
-
- // Test that database and table don't coalesce.
- @Test
- public void testDbTable() throws Exception {
- LockRequestBuilder bldr = new LockRequestBuilder();
- LockComponent comp = new LockComponent(LockType.EXCLUSIVE, LockLevel.DB, "mydb");
- bldr.addLockComponent(comp).setUser("fred");
- comp = new LockComponent(LockType.EXCLUSIVE, LockLevel.DB, "mydb");
- comp.setTablename("mytable");
- bldr.addLockComponent(comp);
- LockRequest req = bldr.build();
- List<LockComponent> locks = req.getComponent();
- Assert.assertEquals(2, locks.size());
- Assert.assertEquals("fred", req.getUser());
- Assert.assertEquals(InetAddress.getLocalHost().getHostName(), req.getHostname());
- }
-
- // Test that database and table don't coalesce.
- @Test
- public void testTablePartition() {
- LockRequestBuilder bldr = new LockRequestBuilder();
- LockComponent comp = new LockComponent(LockType.EXCLUSIVE, LockLevel.DB, "mydb");
- comp.setTablename("mytable");
- bldr.addLockComponent(comp).setUser(null);
- comp = new LockComponent(LockType.EXCLUSIVE, LockLevel.DB, "mydb");
- comp.setTablename("mytable");
- comp.setPartitionname("mypartition");
- bldr.addLockComponent(comp);
- LockRequest req = bldr.build();
- List<LockComponent> locks = req.getComponent();
- Assert.assertEquals(2, locks.size());
- Assert.assertEquals("unknown", req.getUser());
- }
-
- // Test that 2 separate databases don't coalesce.
- @Test
- public void testTwoSeparateDbs() {
- LockRequestBuilder bldr = new LockRequestBuilder();
- LockComponent comp = new LockComponent(LockType.EXCLUSIVE, LockLevel.DB, "mydb");
- bldr.addLockComponent(comp).setUser("fred");
- comp = new LockComponent(LockType.EXCLUSIVE, LockLevel.DB, "yourdb");
- bldr.addLockComponent(comp);
- LockRequest req = bldr.build();
- List<LockComponent> locks = req.getComponent();
- Assert.assertEquals(2, locks.size());
- }
-
- // Test that 2 exclusive db locks coalesce to one
- @Test
- public void testExExDb() {
- LockRequestBuilder bldr = new LockRequestBuilder();
- LockComponent comp = new LockComponent(LockType.EXCLUSIVE, LockLevel.DB, "mydb");
- bldr.addLockComponent(comp);
- comp = new LockComponent(LockType.EXCLUSIVE, LockLevel.DB, "mydb");
- bldr.addLockComponent(comp).setUser("fred");
- LockRequest req = bldr.build();
- List<LockComponent> locks = req.getComponent();
- Assert.assertEquals(1, locks.size());
- }
-
- // Test that existing exclusive db with new shared_write coalesces to
- // exclusive
- @Test
- public void testExSWDb() {
- LockRequestBuilder bldr = new LockRequestBuilder();
- LockComponent comp = new LockComponent(LockType.EXCLUSIVE, LockLevel.DB, "mydb");
- bldr.addLockComponent(comp).setUser("fred");
- comp = new LockComponent(LockType.SHARED_WRITE, LockLevel.DB, "mydb");
- bldr.addLockComponent(comp);
- LockRequest req = bldr.build();
- List<LockComponent> locks = req.getComponent();
- Assert.assertEquals(1, locks.size());
- Assert.assertEquals(LockType.EXCLUSIVE, locks.get(0).getType());
- }
-
- // Test that existing exclusive db with new shared_read coalesces to
- // exclusive
- @Test
- public void testExSRDb() {
- LockRequestBuilder bldr = new LockRequestBuilder();
- LockComponent comp = new LockComponent(LockType.EXCLUSIVE, LockLevel.DB, "mydb");
- bldr.addLockComponent(comp).setUser("fred");
- comp = new LockComponent(LockType.SHARED_READ, LockLevel.DB, "mydb");
- bldr.addLockComponent(comp);
- LockRequest req = bldr.build();
- List<LockComponent> locks = req.getComponent();
- Assert.assertEquals(1, locks.size());
- Assert.assertEquals(LockType.EXCLUSIVE, locks.get(0).getType());
- }
-
- // Test that existing shared_write db with new exclusive coalesces to
- // exclusive
- @Test
- public void testSWExDb() {
- LockRequestBuilder bldr = new LockRequestBuilder();
- LockComponent comp = new LockComponent(LockType.SHARED_WRITE, LockLevel.DB, "mydb");
- bldr.addLockComponent(comp);
- comp = new LockComponent(LockType.EXCLUSIVE, LockLevel.DB, "mydb");
- bldr.addLockComponent(comp).setUser("fred");
- LockRequest req = bldr.build();
- List<LockComponent> locks = req.getComponent();
- Assert.assertEquals(1, locks.size());
- Assert.assertEquals(LockType.EXCLUSIVE, locks.get(0).getType());
- }
-
- // Test that existing shared_write db with new shared_write coalesces to
- // shared_write
- @Test
- public void testSWSWDb() {
- LockRequestBuilder bldr = new LockRequestBuilder();
- LockComponent comp = new LockComponent(LockType.SHARED_WRITE, LockLevel.DB, "mydb");
- bldr.addLockComponent(comp);
- comp = new LockComponent(LockType.SHARED_WRITE, LockLevel.DB, "mydb");
- bldr.addLockComponent(comp).setUser("fred");
- LockRequest req = bldr.build();
- List<LockComponent> locks = req.getComponent();
- Assert.assertEquals(1, locks.size());
- Assert.assertEquals(LockType.SHARED_WRITE, locks.get(0).getType());
- }
-
- // Test that existing shared_write db with new shared_read coalesces to
- // shared_write
- @Test
- public void testSWSRDb() {
- LockRequestBuilder bldr = new LockRequestBuilder();
- LockComponent comp = new LockComponent(LockType.SHARED_WRITE, LockLevel.DB, "mydb");
- bldr.addLockComponent(comp);
- comp = new LockComponent(LockType.SHARED_READ, LockLevel.DB, "mydb");
- bldr.addLockComponent(comp).setUser("fred");
- LockRequest req = bldr.build();
- List<LockComponent> locks = req.getComponent();
- Assert.assertEquals(1, locks.size());
- Assert.assertEquals(LockType.SHARED_WRITE, locks.get(0).getType());
- }
-
- // Test that existing shared_read db with new exclusive coalesces to
- // exclusive
- @Test
- public void testSRExDb() {
- LockRequestBuilder bldr = new LockRequestBuilder();
- LockComponent comp = new LockComponent(LockType.SHARED_READ, LockLevel.DB, "mydb");
- bldr.addLockComponent(comp);
- comp = new LockComponent(LockType.EXCLUSIVE, LockLevel.DB, "mydb");
- bldr.addLockComponent(comp).setUser("fred");
- LockRequest req = bldr.build();
- List<LockComponent> locks = req.getComponent();
- Assert.assertEquals(1, locks.size());
- Assert.assertEquals(LockType.EXCLUSIVE, locks.get(0).getType());
- }
-
- // Test that existing shared_read db with new shared_write coalesces to
- // shared_write
- @Test
- public void testSRSWDb() {
- LockRequestBuilder bldr = new LockRequestBuilder();
- LockComponent comp = new LockComponent(LockType.SHARED_READ, LockLevel.DB, "mydb");
- bldr.addLockComponent(comp);
- comp = new LockComponent(LockType.SHARED_WRITE, LockLevel.DB, "mydb");
- bldr.addLockComponent(comp).setUser("fred");
- LockRequest req = bldr.build();
- List<LockComponent> locks = req.getComponent();
- Assert.assertEquals(1, locks.size());
- Assert.assertEquals(LockType.SHARED_WRITE, locks.get(0).getType());
- }
-
- // Test that existing shared_read db with new shared_read coalesces to
- // shared_read
- @Test
- public void testSRSRDb() {
- LockRequestBuilder bldr = new LockRequestBuilder();
- LockComponent comp = new LockComponent(LockType.SHARED_READ, LockLevel.DB, "mydb");
- bldr.addLockComponent(comp);
- comp = new LockComponent(LockType.SHARED_READ, LockLevel.DB, "mydb");
- bldr.addLockComponent(comp).setUser("fred");
- LockRequest req = bldr.build();
- List<LockComponent> locks = req.getComponent();
- Assert.assertEquals(1, locks.size());
- Assert.assertEquals(LockType.SHARED_READ, locks.get(0).getType());
- }
-
- // Test that 2 separate tables don't coalesce.
- @Test
- public void testTwoSeparateTables() {
- LockRequestBuilder bldr = new LockRequestBuilder();
- LockComponent comp = new LockComponent(LockType.EXCLUSIVE, LockLevel.DB, "mydb");
- comp.setTablename("mytable");
- bldr.addLockComponent(comp);
- comp = new LockComponent(LockType.EXCLUSIVE, LockLevel.DB, "mydb");
- comp.setTablename("yourtable");
- bldr.addLockComponent(comp).setUser("fred");
- LockRequest req = bldr.build();
- List<LockComponent> locks = req.getComponent();
- Assert.assertEquals(2, locks.size());
- }
-
- // Test that 2 exclusive table locks coalesce to one
- @Test
- public void testExExTable() {
- LockRequestBuilder bldr = new LockRequestBuilder();
- LockComponent comp = new LockComponent(LockType.EXCLUSIVE, LockLevel.DB, "mydb");
- comp.setTablename("mytable");
- bldr.addLockComponent(comp);
- comp = new LockComponent(LockType.EXCLUSIVE, LockLevel.DB, "mydb");
- comp.setTablename("mytable");
- bldr.addLockComponent(comp).setUser("fred");
- LockRequest req = bldr.build();
- List<LockComponent> locks = req.getComponent();
- Assert.assertEquals(1, locks.size());
- }
-
- // Test that existing exclusive table with new shared_write coalesces to
- // exclusive
- @Test
- public void testExSWTable() {
- LockRequestBuilder bldr = new LockRequestBuilder();
- LockComponent comp = new LockComponent(LockType.EXCLUSIVE, LockLevel.DB, "mydb");
- comp.setTablename("mytable");
- bldr.addLockComponent(comp);
- comp = new LockComponent(LockType.SHARED_WRITE, LockLevel.DB, "mydb");
- comp.setTablename("mytable");
- bldr.addLockComponent(comp).setUser("fred");
- LockRequest req = bldr.build();
- List<LockComponent> locks = req.getComponent();
- Assert.assertEquals(1, locks.size());
- Assert.assertEquals(LockType.EXCLUSIVE, locks.get(0).getType());
- }
-
- // Test that existing exclusive table with new shared_read coalesces to
- // exclusive
- @Test
- public void testExSRTable() {
- LockRequestBuilder bldr = new LockRequestBuilder();
- LockComponent comp = new LockComponent(LockType.EXCLUSIVE, LockLevel.DB, "mydb");
- comp.setTablename("mytable");
- bldr.addLockComponent(comp);
- comp = new LockComponent(LockType.SHARED_READ, LockLevel.DB, "mydb");
- comp.setTablename("mytable");
- bldr.addLockComponent(comp).setUser("fred");
- LockRequest req = bldr.build();
- List<LockComponent> locks = req.getComponent();
- Assert.assertEquals(1, locks.size());
- Assert.assertEquals(LockType.EXCLUSIVE, locks.get(0).getType());
- }
-
- // Test that existing shared_write table with new exclusive coalesces to
- // exclusive
- @Test
- public void testSWExTable() {
- LockRequestBuilder bldr = new LockRequestBuilder();
- LockComponent comp = new LockComponent(LockType.SHARED_WRITE, LockLevel.DB, "mydb");
- comp.setTablename("mytable");
- bldr.addLockComponent(comp);
- comp = new LockComponent(LockType.EXCLUSIVE, LockLevel.DB, "mydb");
- comp.setTablename("mytable");
- bldr.addLockComponent(comp).setUser("fred");
- LockRequest req = bldr.build();
- List<LockComponent> locks = req.getComponent();
- Assert.assertEquals(1, locks.size());
- Assert.assertEquals(LockType.EXCLUSIVE, locks.get(0).getType());
- }
-
- // Test that existing shared_write table with new shared_write coalesces to
- // shared_write
- @Test
- public void testSWSWTable() {
- LockRequestBuilder bldr = new LockRequestBuilder();
- LockComponent comp = new LockComponent(LockType.SHARED_WRITE, LockLevel.DB, "mydb");
- comp.setTablename("mytable");
- bldr.addLockComponent(comp);
- comp = new LockComponent(LockType.SHARED_WRITE, LockLevel.DB, "mydb");
- comp.setTablename("mytable");
- bldr.addLockComponent(comp).setUser("fred");
- LockRequest req = bldr.build();
- List<LockComponent> locks = req.getComponent();
- Assert.assertEquals(1, locks.size());
- Assert.assertEquals(LockType.SHARED_WRITE, locks.get(0).getType());
- }
-
- // Test that existing shared_write table with new shared_read coalesces to
- // shared_write
- @Test
- public void testSWSRTable() {
- LockRequestBuilder bldr = new LockRequestBuilder();
- LockComponent comp = new LockComponent(LockType.SHARED_WRITE, LockLevel.DB, "mydb");
- comp.setTablename("mytable");
- bldr.addLockComponent(comp);
- comp = new LockComponent(LockType.SHARED_READ, LockLevel.DB, "mydb");
- comp.setTablename("mytable");
- bldr.addLockComponent(comp).setUser("fred");
- LockRequest req = bldr.build();
- List<LockComponent> locks = req.getComponent();
- Assert.assertEquals(1, locks.size());
- Assert.assertEquals(LockType.SHARED_WRITE, locks.get(0).getType());
- }
-
- // Test that existing shared_read table with new exclusive coalesces to
- // exclusive
- @Test
- public void testSRExTable() {
- LockRequestBuilder bldr = new LockRequestBuilder();
- LockComponent comp = new LockComponent(LockType.SHARED_READ, LockLevel.DB, "mydb");
- comp.setTablename("mytable");
- bldr.addLockComponent(comp);
- comp = new LockComponent(LockType.EXCLUSIVE, LockLevel.DB, "mydb");
- comp.setTablename("mytable");
- bldr.addLockComponent(comp).setUser("fred");
- LockRequest req = bldr.build();
- List<LockComponent> locks = req.getComponent();
- Assert.assertEquals(1, locks.size());
- Assert.assertEquals(LockType.EXCLUSIVE, locks.get(0).getType());
- }
-
- // Test that existing shared_read table with new shared_write coalesces to
- // shared_write
- @Test
- public void testSRSWTable() {
- LockRequestBuilder bldr = new LockRequestBuilder();
- LockComponent comp = new LockComponent(LockType.SHARED_READ, LockLevel.DB, "mydb");
- comp.setTablename("mytable");
- bldr.addLockComponent(comp);
- comp = new LockComponent(LockType.SHARED_WRITE, LockLevel.DB, "mydb");
- comp.setTablename("mytable");
- bldr.addLockComponent(comp).setUser("fred");
- LockRequest req = bldr.build();
- List<LockComponent> locks = req.getComponent();
- Assert.assertEquals(1, locks.size());
- Assert.assertEquals(LockType.SHARED_WRITE, locks.get(0).getType());
- }
-
- // Test that existing shared_read table with new shared_read coalesces to
- // shared_read
- @Test
- public void testSRSRTable() {
- LockRequestBuilder bldr = new LockRequestBuilder();
- LockComponent comp = new LockComponent(LockType.SHARED_READ, LockLevel.DB, "mydb");
- comp.setTablename("mytable");
- bldr.addLockComponent(comp);
- comp = new LockComponent(LockType.SHARED_READ, LockLevel.DB, "mydb");
- comp.setTablename("mytable");
- bldr.addLockComponent(comp).setUser("fred");
- LockRequest req = bldr.build();
- List<LockComponent> locks = req.getComponent();
- Assert.assertEquals(1, locks.size());
- Assert.assertEquals(LockType.SHARED_READ, locks.get(0).getType());
- }
-
- // Test that 2 separate partitions don't coalesce.
- @Test
- public void testTwoSeparatePartitions() {
- LockRequestBuilder bldr = new LockRequestBuilder();
- LockComponent comp = new LockComponent(LockType.EXCLUSIVE, LockLevel.DB, "mydb");
- comp.setTablename("mytable");
- comp.setPartitionname("mypart");
- bldr.addLockComponent(comp);
- comp = new LockComponent(LockType.EXCLUSIVE, LockLevel.DB, "mydb");
- comp.setTablename("mytable");
- comp.setPartitionname("yourpart");
- bldr.addLockComponent(comp).setUser("fred");
- LockRequest req = bldr.build();
- List<LockComponent> locks = req.getComponent();
- Assert.assertEquals(2, locks.size());
- }
-
- // Test that 2 exclusive partition locks coalesce to one
- @Test
- public void testExExPart() {
- LockRequestBuilder bldr = new LockRequestBuilder();
- LockComponent comp = new LockComponent(LockType.EXCLUSIVE, LockLevel.DB, "mydb");
- comp.setTablename("mytable");
- comp.setPartitionname("mypart");
- bldr.addLockComponent(comp);
- comp = new LockComponent(LockType.EXCLUSIVE, LockLevel.DB, "mydb");
- comp.setTablename("mytable");
- comp.setPartitionname("mypart");
- bldr.addLockComponent(comp).setUser("fred");
- LockRequest req = bldr.build();
- List<LockComponent> locks = req.getComponent();
- Assert.assertEquals(1, locks.size());
- }
-
- // Test that existing exclusive partition with new shared_write coalesces to
- // exclusive
- @Test
- public void testExSWPart() {
- LockRequestBuilder bldr = new LockRequestBuilder();
- LockComponent comp = new LockComponent(LockType.EXCLUSIVE, LockLevel.DB, "mydb");
- comp.setTablename("mytable");
- comp.setPartitionname("mypart");
- bldr.addLockComponent(comp);
- comp = new LockComponent(LockType.SHARED_WRITE, LockLevel.DB, "mydb");
- comp.setTablename("mytable");
- comp.setPartitionname("mypart");
- bldr.addLockComponent(comp).setUser("fred");
- LockRequest req = bldr.build();
- List<LockComponent> locks = req.getComponent();
- Assert.assertEquals(1, locks.size());
- Assert.assertEquals(LockType.EXCLUSIVE, locks.get(0).getType());
- }
-
- // Test that existing exclusive partition with new shared_read coalesces to
- // exclusive
- @Test
- public void testExSRPart() {
- LockRequestBuilder bldr = new LockRequestBuilder();
- LockComponent comp = new LockComponent(LockType.EXCLUSIVE, LockLevel.DB, "mydb");
- comp.setTablename("mytable");
- comp.setPartitionname("mypart");
- bldr.addLockComponent(comp);
- comp = new LockComponent(LockType.SHARED_READ, LockLevel.DB, "mydb");
- comp.setTablename("mytable");
- comp.setPartitionname("mypart");
- bldr.addLockComponent(comp).setUser("fred");
- LockRequest req = bldr.build();
- List<LockComponent> locks = req.getComponent();
- Assert.assertEquals(1, locks.size());
- Assert.assertEquals(LockType.EXCLUSIVE, locks.get(0).getType());
- }
-
- // Test that existing shared_write partition with new exclusive coalesces to
- // exclusive
- @Test
- public void testSWExPart() {
- LockRequestBuilder bldr = new LockRequestBuilder();
- LockComponent comp = new LockComponent(LockType.SHARED_WRITE, LockLevel.DB, "mydb");
- comp.setTablename("mytable");
- comp.setPartitionname("mypart");
- bldr.addLockComponent(comp);
- comp = new LockComponent(LockType.EXCLUSIVE, LockLevel.DB, "mydb");
- comp.setTablename("mytable");
- comp.setPartitionname("mypart");
- bldr.addLockComponent(comp).setUser("fred");
- LockRequest req = bldr.build();
- List<LockComponent> locks = req.getComponent();
- Assert.assertEquals(1, locks.size());
- Assert.assertEquals(LockType.EXCLUSIVE, locks.get(0).getType());
- }
-
- // Test that existing shared_write partition with new shared_write coalesces to
- // shared_write
- @Test
- public void testSWSWPart() {
- LockRequestBuilder bldr = new LockRequestBuilder();
- LockComponent comp = new LockComponent(LockType.SHARED_WRITE, LockLevel.DB, "mydb");
- comp.setTablename("mytable");
- comp.setPartitionname("mypart");
- bldr.addLockComponent(comp);
- comp = new LockComponent(LockType.SHARED_WRITE, LockLevel.DB, "mydb");
- comp.setTablename("mytable");
- comp.setPartitionname("mypart");
- bldr.addLockComponent(comp).setUser("fred");
- LockRequest req = bldr.build();
- List<LockComponent> locks = req.getComponent();
- Assert.assertEquals(1, locks.size());
- Assert.assertEquals(LockType.SHARED_WRITE, locks.get(0).getType());
- }
-
- // Test that existing shared_write partition with new shared_read coalesces to
- // shared_write
- @Test
- public void testSWSRPart() {
- LockRequestBuilder bldr = new LockRequestBuilder();
- LockComponent comp = new LockComponent(LockType.SHARED_WRITE, LockLevel.DB, "mydb");
- comp.setTablename("mytable");
- comp.setPartitionname("mypart");
- bldr.addLockComponent(comp);
- comp = new LockComponent(LockType.SHARED_READ, LockLevel.DB, "mydb");
- comp.setTablename("mytable");
- comp.setPartitionname("mypart");
- bldr.addLockComponent(comp).setUser("fred");
- LockRequest req = bldr.build();
- List<LockComponent> locks = req.getComponent();
- Assert.assertEquals(1, locks.size());
- Assert.assertEquals(LockType.SHARED_WRITE, locks.get(0).getType());
- }
-
- // Test that existing shared_read partition with new exclusive coalesces to
- // exclusive
- @Test
- public void testSRExPart() {
- LockRequestBuilder bldr = new LockRequestBuilder();
- LockComponent comp = new LockComponent(LockType.SHARED_READ, LockLevel.DB, "mydb");
- comp.setTablename("mytable");
- comp.setPartitionname("mypart");
- bldr.addLockComponent(comp);
- comp = new LockComponent(LockType.EXCLUSIVE, LockLevel.DB, "mydb");
- comp.setTablename("mytable");
- comp.setPartitionname("mypart");
- bldr.addLockComponent(comp).setUser("fred");
- LockRequest req = bldr.build();
- List<LockComponent> locks = req.getComponent();
- Assert.assertEquals(1, locks.size());
- Assert.assertEquals(LockType.EXCLUSIVE, locks.get(0).getType());
- }
-
- // Test that existing shared_read partition with new shared_write coalesces to
- // shared_write
- @Test
- public void testSRSWPart() {
- LockRequestBuilder bldr = new LockRequestBuilder();
- LockComponent comp = new LockComponent(LockType.SHARED_READ, LockLevel.DB, "mydb");
- comp.setTablename("mytable");
- comp.setPartitionname("mypart");
- bldr.addLockComponent(comp);
- comp = new LockComponent(LockType.SHARED_WRITE, LockLevel.DB, "mydb");
- comp.setTablename("mytable");
- comp.setPartitionname("mypart");
- bldr.addLockComponent(comp).setUser("fred");
- LockRequest req = bldr.build();
- List<LockComponent> locks = req.getComponent();
- Assert.assertEquals(1, locks.size());
- Assert.assertEquals(LockType.SHARED_WRITE, locks.get(0).getType());
- }
-
- // Test that existing shared_read partition with new shared_read coalesces to
- // shared_read
- @Test
- public void testSRSRPart() {
- LockRequestBuilder bldr = new LockRequestBuilder();
- LockComponent comp = new LockComponent(LockType.SHARED_READ, LockLevel.DB, "mydb");
- comp.setTablename("mytable");
- comp.setPartitionname("mypart");
- bldr.addLockComponent(comp);
- comp = new LockComponent(LockType.SHARED_READ, LockLevel.DB, "mydb");
- comp.setTablename("mytable");
- comp.setPartitionname("mypart");
- bldr.addLockComponent(comp).setUser("fred");
- LockRequest req = bldr.build();
- List<LockComponent> locks = req.getComponent();
- Assert.assertEquals(1, locks.size());
- Assert.assertEquals(LockType.SHARED_READ, locks.get(0).getType());
- }
-}
http://git-wip-us.apache.org/repos/asf/hive/blob/b0b6db73/metastore/src/test/org/apache/hadoop/hive/metastore/TestMetaStoreSchemaFactory.java
----------------------------------------------------------------------
diff --git a/metastore/src/test/org/apache/hadoop/hive/metastore/TestMetaStoreSchemaFactory.java b/metastore/src/test/org/apache/hadoop/hive/metastore/TestMetaStoreSchemaFactory.java
deleted file mode 100644
index 99eec40..0000000
--- a/metastore/src/test/org/apache/hadoop/hive/metastore/TestMetaStoreSchemaFactory.java
+++ /dev/null
@@ -1,67 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.hive.metastore;
-
-import org.apache.hadoop.hive.conf.HiveConf;
-import org.junit.Assert;
-import org.junit.Before;
-import org.junit.Test;
-
-public class TestMetaStoreSchemaFactory {
- private HiveConf conf;
-
- @Before
- public void setup() {
- conf = new HiveConf(this.getClass());
- }
-
- @Test
- public void testDefaultConfig() {
- IMetaStoreSchemaInfo metastoreSchemaInfo = MetaStoreSchemaInfoFactory.get(conf);
- Assert.assertNotNull(metastoreSchemaInfo);
- }
-
- @Test
- public void testWithConfigSet() {
- conf.set(HiveConf.ConfVars.METASTORE_SCHEMA_INFO_CLASS.varname,
- MetaStoreSchemaInfo.class.getCanonicalName());
- IMetaStoreSchemaInfo metastoreSchemaInfo = MetaStoreSchemaInfoFactory.get(conf);
- Assert.assertNotNull(metastoreSchemaInfo);
- Assert.assertTrue("Unexpected instance type of the class MetaStoreSchemaInfo",
- metastoreSchemaInfo instanceof MetaStoreSchemaInfo);
- }
-
- @Test
- public void testConstructor() {
- String className = conf.get(HiveConf.ConfVars.METASTORE_SCHEMA_INFO_CLASS.varname,
- MetaStoreSchemaInfo.class.getCanonicalName());
- Class<?> clasz = null;
- try {
- clasz = conf.getClassByName(className);
- clasz.getConstructor(String.class, String.class);
- } catch (NoSuchMethodException | IllegalArgumentException | ClassNotFoundException e) {
- throw new IllegalArgumentException(e);
- }
- }
-
- @Test(expected = IllegalArgumentException.class)
- public void testInvalidClassName() {
- conf.set(HiveConf.ConfVars.METASTORE_SCHEMA_INFO_CLASS.varname, "invalid.class.name");
- MetaStoreSchemaInfoFactory.get(conf);
- }
-}
http://git-wip-us.apache.org/repos/asf/hive/blob/b0b6db73/metastore/src/test/org/apache/hadoop/hive/metastore/TestMetaStoreSchemaInfo.java
----------------------------------------------------------------------
diff --git a/metastore/src/test/org/apache/hadoop/hive/metastore/TestMetaStoreSchemaInfo.java b/metastore/src/test/org/apache/hadoop/hive/metastore/TestMetaStoreSchemaInfo.java
deleted file mode 100644
index 35e862f..0000000
--- a/metastore/src/test/org/apache/hadoop/hive/metastore/TestMetaStoreSchemaInfo.java
+++ /dev/null
@@ -1,52 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.hive.metastore;
-
-import org.apache.hadoop.hive.conf.HiveConf;
-import org.junit.Assert;
-import org.junit.Test;
-
-/**
- * Test MetaStoreSchemaInfo
- */
-public class TestMetaStoreSchemaInfo {
-
- @Test
- public void testIsVersionCompatible() throws Exception {
- // first argument is hiveVersion, it is compatible if 2nd argument - dbVersion is
- // greater than or equal to it
- // check the compatible case
- IMetaStoreSchemaInfo metastoreSchemaInfo =
- MetaStoreSchemaInfoFactory.get(new HiveConf(TestMetaStoreSchemaInfo.class));
- Assert.assertTrue(metastoreSchemaInfo.isVersionCompatible("0.0.1", "0.0.1"));
- Assert.assertTrue(metastoreSchemaInfo.isVersionCompatible("0.0.1", "0.0.2"));
- Assert.assertTrue(metastoreSchemaInfo.isVersionCompatible("1.0.2", "2.0.1"));
- Assert.assertTrue(metastoreSchemaInfo.isVersionCompatible("0.0.9", "9.0.0"));
-
- // check equivalent versions, should be compatible
- Assert.assertTrue(metastoreSchemaInfo.isVersionCompatible("0.13.0", "0.13.1"));
- Assert.assertTrue(metastoreSchemaInfo.isVersionCompatible("0.13.1", "0.13.0"));
-
- // check incompatible versions
- Assert.assertFalse(metastoreSchemaInfo.isVersionCompatible("0.1.1", "0.1.0"));
- Assert.assertFalse(metastoreSchemaInfo.isVersionCompatible("4.0.1", "0.1.0"));
-
- }
-
-}
http://git-wip-us.apache.org/repos/asf/hive/blob/b0b6db73/ql/src/java/org/apache/hadoop/hive/ql/security/authorization/plugin/AuthorizationMetaStoreFilterHook.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/security/authorization/plugin/AuthorizationMetaStoreFilterHook.java b/ql/src/java/org/apache/hadoop/hive/ql/security/authorization/plugin/AuthorizationMetaStoreFilterHook.java
index b08c63d..96f1b9e 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/security/authorization/plugin/AuthorizationMetaStoreFilterHook.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/security/authorization/plugin/AuthorizationMetaStoreFilterHook.java
@@ -20,10 +20,10 @@ package org.apache.hadoop.hive.ql.security.authorization.plugin;
import java.util.ArrayList;
import java.util.List;
+import org.apache.hadoop.conf.Configuration;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.apache.hadoop.hive.common.classification.InterfaceAudience.Private;
-import org.apache.hadoop.hive.conf.HiveConf;
import org.apache.hadoop.hive.metastore.DefaultMetaStoreFilterHookImpl;
import org.apache.hadoop.hive.metastore.api.MetaException;
import org.apache.hadoop.hive.ql.security.authorization.plugin.HivePrivilegeObject.HivePrivilegeObjectType;
@@ -38,7 +38,7 @@ public class AuthorizationMetaStoreFilterHook extends DefaultMetaStoreFilterHook
public static final Logger LOG = LoggerFactory.getLogger(AuthorizationMetaStoreFilterHook.class);
- public AuthorizationMetaStoreFilterHook(HiveConf conf) {
+ public AuthorizationMetaStoreFilterHook(Configuration conf) {
super(conf);
}
http://git-wip-us.apache.org/repos/asf/hive/blob/b0b6db73/ql/src/java/org/apache/hadoop/hive/ql/txn/compactor/CompactorThread.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/txn/compactor/CompactorThread.java b/ql/src/java/org/apache/hadoop/hive/ql/txn/compactor/CompactorThread.java
index 4d6e24e..2016485 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/txn/compactor/CompactorThread.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/txn/compactor/CompactorThread.java
@@ -1,4 +1,4 @@
-/**
+/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
@@ -17,6 +17,7 @@
*/
package org.apache.hadoop.hive.ql.txn.compactor;
+import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
@@ -28,6 +29,7 @@ import org.apache.hadoop.hive.metastore.api.MetaException;
import org.apache.hadoop.hive.metastore.api.Partition;
import org.apache.hadoop.hive.metastore.api.StorageDescriptor;
import org.apache.hadoop.hive.metastore.api.Table;
+import org.apache.hadoop.hive.metastore.conf.MetastoreConf;
import org.apache.hadoop.hive.metastore.txn.CompactionInfo;
import org.apache.hadoop.hive.metastore.txn.TxnStore;
import org.apache.hadoop.hive.metastore.txn.TxnUtils;
@@ -58,8 +60,17 @@ abstract class CompactorThread extends Thread implements MetaStoreThread {
protected AtomicBoolean looped;
@Override
- public void setHiveConf(HiveConf conf) {
- this.conf = conf;
+ public void setConf(Configuration configuration) {
+ // TODO MS-SPLIT for now, keep a copy of HiveConf around as we need to call other methods with
+ // it. This should be changed to Configuration once everything that this calls that requires
+ // HiveConf is moved to the standalone metastore.
+ conf = (configuration instanceof HiveConf) ? (HiveConf)configuration :
+ new HiveConf(configuration, HiveConf.class);
+ }
+
+ @Override
+ public Configuration getConf() {
+ return conf;
}
@Override
@@ -80,7 +91,7 @@ abstract class CompactorThread extends Thread implements MetaStoreThread {
// Get our own connection to the database so we can get table and partition information.
rs = RawStoreProxy.getProxy(conf, conf,
- conf.getVar(HiveConf.ConfVars.METASTORE_RAW_STORE_IMPL), threadId);
+ MetastoreConf.getVar(conf, MetastoreConf.ConfVars.RAW_STORE_IMPL), threadId);
}
/**
@@ -107,7 +118,7 @@ abstract class CompactorThread extends Thread implements MetaStoreThread {
*/
protected Partition resolvePartition(CompactionInfo ci) throws Exception {
if (ci.partName != null) {
- List<Partition> parts = null;
+ List<Partition> parts;
try {
parts = rs.getPartitionsByNames(ci.dbname, ci.tableName,
Collections.singletonList(ci.partName));
@@ -163,7 +174,7 @@ abstract class CompactorThread extends Thread implements MetaStoreThread {
LOG.debug("Unable to stat file as current user, trying as table owner");
// Now, try it as the table owner and see if we get better luck.
- final List<String> wrapper = new ArrayList<String>(1);
+ final List<String> wrapper = new ArrayList<>(1);
UserGroupInformation ugi = UserGroupInformation.createProxyUser(t.getOwner(),
UserGroupInformation.getLoginUser());
ugi.doAs(new PrivilegedExceptionAction<Object>() {
http://git-wip-us.apache.org/repos/asf/hive/blob/b0b6db73/ql/src/test/org/apache/hadoop/hive/ql/TestTxnCommands2.java
----------------------------------------------------------------------
diff --git a/ql/src/test/org/apache/hadoop/hive/ql/TestTxnCommands2.java b/ql/src/test/org/apache/hadoop/hive/ql/TestTxnCommands2.java
index 21b4a2c..d08371d 100644
--- a/ql/src/test/org/apache/hadoop/hive/ql/TestTxnCommands2.java
+++ b/ql/src/test/org/apache/hadoop/hive/ql/TestTxnCommands2.java
@@ -228,7 +228,7 @@ public class TestTxnCommands2 {
runStatementOnDriver("alter table " + Table.ACIDTBL + " compact 'MAJOR'");
Worker t = new Worker();
t.setThreadId((int) t.getId());
- t.setHiveConf(hiveConf);
+ t.setConf(hiveConf);
AtomicBoolean stop = new AtomicBoolean();
AtomicBoolean looped = new AtomicBoolean();
stop.set(true);
@@ -275,7 +275,7 @@ public class TestTxnCommands2 {
runStatementOnDriver("alter table "+ Table.ACIDTBL + " compact 'MAJOR'");
Worker t = new Worker();
t.setThreadId((int) t.getId());
- t.setHiveConf(hiveConf);
+ t.setConf(hiveConf);
AtomicBoolean stop = new AtomicBoolean();
AtomicBoolean looped = new AtomicBoolean();
stop.set(true);
@@ -1010,7 +1010,7 @@ public class TestTxnCommands2 {
//this should not schedule a new compaction due to prior failures, but will create Attempted entry
Initiator init = new Initiator();
init.setThreadId((int)init.getId());
- init.setHiveConf(hiveConf);
+ init.setConf(hiveConf);
init.init(stop, new AtomicBoolean());
init.run();
int numAttemptedCompactions = 1;
@@ -1160,7 +1160,7 @@ public class TestTxnCommands2 {
AtomicBoolean stop = new AtomicBoolean(true);
Worker t = new Worker();
t.setThreadId((int) t.getId());
- t.setHiveConf(hiveConf);
+ t.setConf(hiveConf);
AtomicBoolean looped = new AtomicBoolean();
t.init(stop, looped);
t.run();
@@ -1169,7 +1169,7 @@ public class TestTxnCommands2 {
AtomicBoolean stop = new AtomicBoolean(true);
Cleaner t = new Cleaner();
t.setThreadId((int) t.getId());
- t.setHiveConf(hiveConf);
+ t.setConf(hiveConf);
AtomicBoolean looped = new AtomicBoolean();
t.init(stop, looped);
t.run();
@@ -1220,7 +1220,7 @@ public class TestTxnCommands2 {
txnHandler.compact(new CompactionRequest("default", tblName, CompactionType.MINOR));
Worker t = new Worker();
t.setThreadId((int) t.getId());
- t.setHiveConf(hiveConf);
+ t.setConf(hiveConf);
AtomicBoolean stop = new AtomicBoolean(true);
AtomicBoolean looped = new AtomicBoolean();
t.init(stop, looped);
@@ -1241,14 +1241,14 @@ public class TestTxnCommands2 {
//run Cleaner
Cleaner c = new Cleaner();
c.setThreadId((int)c.getId());
- c.setHiveConf(hiveConf);
+ c.setConf(hiveConf);
c.init(stop, new AtomicBoolean());
c.run();
//this seems odd, but we wan to make sure that to run CompactionTxnHandler.cleanEmptyAbortedTxns()
Initiator i = new Initiator();
i.setThreadId((int)i.getId());
- i.setHiveConf(hiveConf);
+ i.setConf(hiveConf);
i.init(stop, new AtomicBoolean());
i.run();
@@ -1318,7 +1318,7 @@ public class TestTxnCommands2 {
runStatementOnDriver("alter table "+ Table.ACIDTBL + " compact 'MAJOR'");
Worker t = new Worker();
t.setThreadId((int) t.getId());
- t.setHiveConf(hiveConf);
+ t.setConf(hiveConf);
AtomicBoolean stop = new AtomicBoolean();
AtomicBoolean looped = new AtomicBoolean();
stop.set(true);
http://git-wip-us.apache.org/repos/asf/hive/blob/b0b6db73/ql/src/test/org/apache/hadoop/hive/ql/txn/compactor/CompactorTest.java
----------------------------------------------------------------------
diff --git a/ql/src/test/org/apache/hadoop/hive/ql/txn/compactor/CompactorTest.java b/ql/src/test/org/apache/hadoop/hive/ql/txn/compactor/CompactorTest.java
index 7455fa8..af8a63d 100644
--- a/ql/src/test/org/apache/hadoop/hive/ql/txn/compactor/CompactorTest.java
+++ b/ql/src/test/org/apache/hadoop/hive/ql/txn/compactor/CompactorTest.java
@@ -280,7 +280,7 @@ public abstract class CompactorTest {
default: throw new RuntimeException("Huh? Unknown thread type.");
}
t.setThreadId((int) t.getId());
- t.setHiveConf(conf);
+ t.setConf(conf);
stop.set(stopAfterOne);
t.init(stop, looped);
if (stopAfterOne) t.run();
http://git-wip-us.apache.org/repos/asf/hive/blob/b0b6db73/standalone-metastore/pom.xml
----------------------------------------------------------------------
diff --git a/standalone-metastore/pom.xml b/standalone-metastore/pom.xml
index e38ef37..5cdbbcb 100644
--- a/standalone-metastore/pom.xml
+++ b/standalone-metastore/pom.xml
@@ -43,6 +43,11 @@
<version>${guava.version}</version>
</dependency>
<dependency>
+ <groupId>com.google.protobuf</groupId>
+ <artifactId>protobuf-java</artifactId>
+ <version>${protobuf.version}</version>
+ </dependency>
+ <dependency>
<groupId>io.dropwizard.metrics</groupId>
<artifactId>metrics-core</artifactId>
<version>${dropwizard.version}</version>
@@ -190,6 +195,26 @@
<plugins>
<!-- plugins are always listed in sorted order by groupId, artifectId -->
<plugin>
+ <groupId>com.github.os72</groupId>
+ <artifactId>protoc-jar-maven-plugin</artifactId>
+ <version>3.0.0-a3</version>
+ <executions>
+ <execution>
+ <phase>generate-sources</phase>
+ <goals>
+ <goal>run</goal>
+ </goals>
+ <configuration>
+ <protocVersion>2.5.0</protocVersion>
+ <addSources>none</addSources>
+ <inputDirectories>
+ <include>${basedir}/src/main/protobuf/org/apache/hadoop/hive/metastore</include>
+ </inputDirectories>
+ </configuration>
+ </execution>
+ </executions>
+ </plugin>
+ <plugin>
<groupId>org.codehaus.mojo</groupId>
<artifactId>build-helper-maven-plugin</artifactId>
<executions>
@@ -202,6 +227,8 @@
<configuration>
<sources>
<source>src/gen/thrift/gen-javabean</source>
+ <source>${project.build.directory}/generated-sources</source>
+ <source>src/gen/org</source>
</sources>
</configuration>
</execution>
@@ -225,6 +252,29 @@
</execution>
</executions>
</plugin>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-antrun-plugin</artifactId>
+ <executions>
+ <execution>
+ <id>generate-version-annotation</id>
+ <phase>generate-sources</phase>
+ <configuration>
+ <target>
+ <exec executable="bash" failonerror="true">
+ <arg value="${basedir}/src/main/resources/saveVersion.sh"/>
+ <arg value="${project.version}"/>
+ <arg value="${hive.version.shortname}"/>
+ <arg value="${basedir}/src"/>
+ </exec>
+ </target>
+ </configuration>
+ <goals>
+ <goal>run</goal>
+ </goals>
+ </execution>
+ </executions>
+ </plugin>
</plugins>
</build>
http://git-wip-us.apache.org/repos/asf/hive/blob/b0b6db73/standalone-metastore/src/main/java/org/apache/hadoop/hive/common/classification/RetrySemantics.java
----------------------------------------------------------------------
diff --git a/standalone-metastore/src/main/java/org/apache/hadoop/hive/common/classification/RetrySemantics.java b/standalone-metastore/src/main/java/org/apache/hadoop/hive/common/classification/RetrySemantics.java
new file mode 100644
index 0000000..10d21ee
--- /dev/null
+++ b/standalone-metastore/src/main/java/org/apache/hadoop/hive/common/classification/RetrySemantics.java
@@ -0,0 +1,57 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hive.common.classification;
+
+
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
+
+import java.lang.annotation.ElementType;
+import java.lang.annotation.Retention;
+import java.lang.annotation.RetentionPolicy;
+import java.lang.annotation.Target;
+
+/**
+ * These annotations are meant to indicate how to handle retry logic.
+ * Initially meant for Metastore API when made across a network, i.e. asynchronously where
+ * the response may not reach the caller and thus it cannot know if the operation was actually
+ * performed on the server.
+ */
+@InterfaceStability.Evolving
+@InterfaceAudience.LimitedPrivate("Hive developer")
+public class RetrySemantics {
+ @Retention(RetentionPolicy.RUNTIME)
+ @Target(ElementType.METHOD)
+ public @interface Idempotent {
+ String[] value() default "";
+ int maxRetryCount() default Integer.MAX_VALUE;
+ int delayMs() default 100;
+ }
+ @Retention(RetentionPolicy.RUNTIME)
+ @Target(ElementType.METHOD)
+ public @interface ReadOnly {/*trivially retry-able*/}
+ @Retention(RetentionPolicy.RUNTIME)
+ @Target(ElementType.METHOD)
+ public @interface CannotRetry {}
+ @Retention(RetentionPolicy.RUNTIME)
+ @Target(ElementType.METHOD)
+ public @interface SafeToRetry {
+ /*may not be Idempotent but is safe to retry*/
+ String[] value() default "";
+ }
+}
http://git-wip-us.apache.org/repos/asf/hive/blob/b0b6db73/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/AggregateStatsCache.java
----------------------------------------------------------------------
diff --git a/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/AggregateStatsCache.java b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/AggregateStatsCache.java
new file mode 100644
index 0000000..bdac161
--- /dev/null
+++ b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/AggregateStatsCache.java
@@ -0,0 +1,567 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.hadoop.hive.metastore;
+
+import org.apache.hadoop.hive.metastore.conf.MetastoreConf;
+import org.apache.hadoop.hive.metastore.conf.MetastoreConf.ConfVars;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hive.metastore.api.ColumnStatisticsObj;
+import org.apache.hive.common.util.BloomFilter;
+
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicInteger;
+import java.util.concurrent.atomic.AtomicLong;
+import java.util.concurrent.locks.Lock;
+import java.util.concurrent.locks.ReadWriteLock;
+import java.util.concurrent.locks.ReentrantReadWriteLock;
+
+public class AggregateStatsCache {
+
+ private static final Logger LOG = LoggerFactory.getLogger(AggregateStatsCache.class.getName());
+ private static AggregateStatsCache self = null;
+
+ // Backing store for this cache
+ private final ConcurrentHashMap<Key, AggrColStatsList> cacheStore;
+ // Cache size
+ private final int maxCacheNodes;
+ // Current nodes in the cache
+ private final AtomicInteger currentNodes = new AtomicInteger(0);
+ // Run the cleaner thread when the cache is maxFull% full
+ private final double maxFull;
+ // Run the cleaner thread until cache is cleanUntil% occupied
+ private final double cleanUntil;
+ // Nodes go stale after this
+ private final long timeToLiveMs;
+ // Max time when waiting for write locks on node list
+ private final long maxWriterWaitTime;
+ // Max time when waiting for read locks on node list
+ private final long maxReaderWaitTime;
+ // Maximum number of paritions aggregated per cache node
+ private final int maxPartsPerCacheNode;
+ // Bloom filter false positive probability
+ private final double falsePositiveProbability;
+ // Max tolerable variance for matches
+ private final double maxVariance;
+ // Used to determine if cleaner thread is already running
+ private boolean isCleaning = false;
+ private final AtomicLong cacheHits = new AtomicLong(0);
+ private final AtomicLong cacheMisses = new AtomicLong(0);
+ // To track cleaner metrics
+ int numRemovedTTL = 0, numRemovedLRU = 0;
+
+ private AggregateStatsCache(int maxCacheNodes, int maxPartsPerCacheNode, long timeToLiveMs,
+ double falsePositiveProbability, double maxVariance, long maxWriterWaitTime,
+ long maxReaderWaitTime, double maxFull, double cleanUntil) {
+ this.maxCacheNodes = maxCacheNodes;
+ this.maxPartsPerCacheNode = maxPartsPerCacheNode;
+ this.timeToLiveMs = timeToLiveMs;
+ this.falsePositiveProbability = falsePositiveProbability;
+ this.maxVariance = maxVariance;
+ this.maxWriterWaitTime = maxWriterWaitTime;
+ this.maxReaderWaitTime = maxReaderWaitTime;
+ this.maxFull = maxFull;
+ this.cleanUntil = cleanUntil;
+ this.cacheStore = new ConcurrentHashMap<>();
+ }
+
+ public static synchronized AggregateStatsCache getInstance(Configuration conf) {
+ if (self == null) {
+ int maxCacheNodes =
+ MetastoreConf.getIntVar(conf, ConfVars.AGGREGATE_STATS_CACHE_SIZE);
+ // The number of partitions aggregated per cache node
+ // If the number of partitions requested is > this value, we'll fetch directly from Metastore
+ int maxPartitionsPerCacheNode =
+ MetastoreConf.getIntVar(conf, ConfVars.AGGREGATE_STATS_CACHE_MAX_PARTITIONS);
+ long timeToLiveMs = MetastoreConf.getTimeVar(conf, ConfVars.AGGREGATE_STATS_CACHE_TTL,
+ TimeUnit.SECONDS)*1000;
+ // False positives probability we are ready to tolerate for the underlying bloom filter
+ double falsePositiveProbability =
+ MetastoreConf.getDoubleVar(conf, ConfVars.AGGREGATE_STATS_CACHE_FPP);
+ // Maximum tolerable variance in number of partitions between cached node and our request
+ double maxVariance =
+ MetastoreConf.getDoubleVar(conf, ConfVars.AGGREGATE_STATS_CACHE_MAX_VARIANCE);
+ long maxWriterWaitTime = MetastoreConf.getTimeVar(conf,
+ ConfVars.AGGREGATE_STATS_CACHE_MAX_WRITER_WAIT, TimeUnit.MILLISECONDS);
+ long maxReaderWaitTime = MetastoreConf.getTimeVar(conf,
+ ConfVars.AGGREGATE_STATS_CACHE_MAX_READER_WAIT, TimeUnit.MILLISECONDS);
+ double maxFull =
+ MetastoreConf.getDoubleVar(conf, ConfVars.AGGREGATE_STATS_CACHE_MAX_FULL);
+ double cleanUntil =
+ MetastoreConf.getDoubleVar(conf, ConfVars.AGGREGATE_STATS_CACHE_CLEAN_UNTIL);
+ self =
+ new AggregateStatsCache(maxCacheNodes, maxPartitionsPerCacheNode, timeToLiveMs,
+ falsePositiveProbability, maxVariance, maxWriterWaitTime, maxReaderWaitTime, maxFull,
+ cleanUntil);
+ }
+ return self;
+ }
+
+ public int getMaxCacheNodes() {
+ return maxCacheNodes;
+ }
+
+ public int getCurrentNodes() {
+ return currentNodes.intValue();
+ }
+
+ public float getFullPercent() {
+ return (currentNodes.intValue() / (float) maxCacheNodes) * 100;
+ }
+
+ public int getMaxPartsPerCacheNode() {
+ return maxPartsPerCacheNode;
+ }
+
+ public double getFalsePositiveProbability() {
+ return falsePositiveProbability;
+ }
+
+ public Float getHitRatio() {
+ if (cacheHits.longValue() + cacheMisses.longValue() > 0) {
+ return (float) (cacheHits.longValue()) / (cacheHits.longValue() + cacheMisses.longValue());
+ }
+ return null;
+ }
+
+ /**
+ * Return aggregate stats for a column from the cache or null.
+ * While reading from the nodelist for a key, we wait maxReaderWaitTime to acquire the lock,
+ * failing which we return a cache miss (i.e. null)
+ *
+ * @param dbName
+ * @param tblName
+ * @param colName
+ * @param partNames
+ * @return
+ */
+ public AggrColStats get(String dbName, String tblName, String colName, List<String> partNames) {
+ // Cache key
+ Key key = new Key(dbName, tblName, colName);
+ AggrColStatsList candidateList = cacheStore.get(key);
+ // No key, or no nodes in candidate list
+ if ((candidateList == null) || (candidateList.nodes.size() == 0)) {
+ LOG.debug("No aggregate stats cached for " + key.toString());
+ return null;
+ }
+ // Find the value object
+ // Update the timestamp of the key,value if value matches the criteria
+ // Return the value
+ AggrColStats match = null;
+ boolean isLocked = false;
+ try {
+ // Try to readlock the candidateList; timeout after maxReaderWaitTime
+ isLocked = candidateList.readLock.tryLock(maxReaderWaitTime, TimeUnit.MILLISECONDS);
+ if (isLocked) {
+ match = findBestMatch(partNames, candidateList.nodes);
+ }
+ if (match != null) {
+ // Ok to not lock the list for this and use a volatile lastAccessTime instead
+ candidateList.updateLastAccessTime();
+ cacheHits.incrementAndGet();
+ LOG.info("Returning aggregate stats from the cache; total hits: " + cacheHits.longValue()
+ + ", total misses: " + cacheMisses.longValue() + ", hit ratio: " + getHitRatio());
+ }
+ else {
+ cacheMisses.incrementAndGet();
+ }
+ } catch (InterruptedException e) {
+ LOG.debug("Interrupted Exception ignored ",e);
+ } finally {
+ if (isLocked) {
+ candidateList.readLock.unlock();
+ }
+ }
+ return match;
+ }
+
+ /**
+ * Find the best match using the configurable error tolerance and time to live value
+ *
+ * @param partNames
+ * @param candidates
+ * @return best matched node or null
+ */
+ private AggrColStats findBestMatch(List<String> partNames, List<AggrColStats> candidates) {
+ // Hits, misses tracked for a candidate node
+ MatchStats matchStats;
+ // MatchStats for each candidate
+ Map<AggrColStats, MatchStats> candidateMatchStats = new HashMap<>();
+ // The final match we intend to return
+ AggrColStats bestMatch = null;
+ // To compare among potentially multiple matches
+ int bestMatchHits = 0;
+ int numPartsRequested = partNames.size();
+ // 1st pass at marking invalid candidates
+ // Checks based on variance and TTL
+ // Note: we're not creating a copy of the list for saving memory
+ for (AggrColStats candidate : candidates) {
+ // Variance check
+ if (Math.abs((candidate.getNumPartsCached() - numPartsRequested) / numPartsRequested)
+ > maxVariance) {
+ continue;
+ }
+ // TTL check
+ if (isExpired(candidate)) {
+ continue;
+ } else {
+ candidateMatchStats.put(candidate, new MatchStats(0, 0));
+ }
+ }
+ // We'll count misses as we iterate
+ int maxMisses = (int) maxVariance * numPartsRequested;
+ for (String partName : partNames) {
+ for (Iterator<Map.Entry<AggrColStats, MatchStats>> iterator = candidateMatchStats.entrySet().iterator(); iterator.hasNext();) {
+ Map.Entry<AggrColStats, MatchStats> entry = iterator.next();
+ AggrColStats candidate = entry.getKey();
+ matchStats = entry.getValue();
+ if (candidate.getBloomFilter().test(partName.getBytes())) {
+ ++matchStats.hits;
+ } else {
+ ++matchStats.misses;
+ }
+ // 2nd pass at removing invalid candidates
+ // If misses so far exceed max tolerable misses
+ if (matchStats.misses > maxMisses) {
+ iterator.remove();
+ continue;
+ }
+ // Check if this is the best match so far
+ if (matchStats.hits > bestMatchHits) {
+ bestMatch = candidate;
+ }
+ }
+ }
+ if (bestMatch != null) {
+ // Update the last access time for this node
+ bestMatch.updateLastAccessTime();
+ }
+ return bestMatch;
+ }
+
+ /**
+ * Add a new node to the cache; may trigger the cleaner thread if the cache is near full capacity.
+ * We'll however add the node even if we temporaily exceed maxCacheNodes, because the cleaner
+ * will eventually create space from expired nodes or by removing LRU nodes.
+ *
+ * @param dbName
+ * @param tblName
+ * @param colName
+ * @param numPartsCached
+ * @param colStats
+ * @param bloomFilter
+ */
+ // TODO: make add asynchronous: add shouldn't block the higher level calls
+ public void add(String dbName, String tblName, String colName, long numPartsCached,
+ ColumnStatisticsObj colStats, BloomFilter bloomFilter) {
+ // If we have no space in the cache, run cleaner thread
+ if (getCurrentNodes() / maxCacheNodes > maxFull) {
+ spawnCleaner();
+ }
+ // Cache key
+ Key key = new Key(dbName, tblName, colName);
+ // Add new node to the cache
+ AggrColStats node = new AggrColStats(numPartsCached, bloomFilter, colStats);
+ AggrColStatsList nodeList;
+ AggrColStatsList newNodeList = new AggrColStatsList();
+ newNodeList.nodes = new ArrayList<>();
+ nodeList = cacheStore.putIfAbsent(key, newNodeList);
+ if (nodeList == null) {
+ nodeList = newNodeList;
+ }
+ boolean isLocked = false;
+ try {
+ isLocked = nodeList.writeLock.tryLock(maxWriterWaitTime, TimeUnit.MILLISECONDS);
+ if (isLocked) {
+ nodeList.nodes.add(node);
+ node.updateLastAccessTime();
+ nodeList.updateLastAccessTime();
+ currentNodes.getAndIncrement();
+ }
+ } catch (InterruptedException e) {
+ LOG.debug("Interrupted Exception ignored ", e);
+ } finally {
+ if (isLocked) {
+ nodeList.writeLock.unlock();
+ }
+ }
+ }
+
+ /**
+ * Cleans the expired nodes or removes LRU nodes of the cache,
+ * until the cache size reduces to cleanUntil% full.
+ */
+ private void spawnCleaner() {
+ // This spawns a separate thread to walk through the cache and removes expired nodes.
+ // Only one cleaner thread should be running at any point.
+ synchronized (this) {
+ if (isCleaning) {
+ return;
+ }
+ isCleaning = true;
+ }
+ Thread cleaner = new Thread("AggregateStatsCache-CleanerThread") {
+ @Override
+ public void run() {
+ numRemovedTTL = 0;
+ numRemovedLRU = 0;
+ long cleanerStartTime = System.currentTimeMillis();
+ LOG.info("AggregateStatsCache is " + getFullPercent() + "% full, with "
+ + getCurrentNodes() + " nodes; starting cleaner thread");
+ try {
+ Iterator<Map.Entry<Key, AggrColStatsList>> mapIterator = cacheStore.entrySet().iterator();
+ while (mapIterator.hasNext()) {
+ Map.Entry<Key, AggrColStatsList> pair =
+ mapIterator.next();
+ AggrColStats node;
+ AggrColStatsList candidateList = pair.getValue();
+ List<AggrColStats> nodes = candidateList.nodes;
+ if (nodes.size() == 0) {
+ mapIterator.remove();
+ continue;
+ }
+ boolean isLocked = false;
+ try {
+ isLocked = candidateList.writeLock.tryLock(maxWriterWaitTime, TimeUnit.MILLISECONDS);
+ if (isLocked) {
+ for (Iterator<AggrColStats> listIterator = nodes.iterator(); listIterator.hasNext();) {
+ node = listIterator.next();
+ // Remove the node if it has expired
+ if (isExpired(node)) {
+ listIterator.remove();
+ numRemovedTTL++;
+ currentNodes.getAndDecrement();
+ }
+ }
+ }
+ } catch (InterruptedException e) {
+ LOG.debug("Interrupted Exception ignored ",e);
+ } finally {
+ if (isLocked) {
+ candidateList.writeLock.unlock();
+ }
+ }
+ // We want to make sure this runs at a low priority in the background
+ Thread.yield();
+ }
+ // If the expired nodes did not result in cache being cleanUntil% in size,
+ // start removing LRU nodes
+ while (getCurrentNodes() / maxCacheNodes > cleanUntil) {
+ evictOneNode();
+ }
+ } finally {
+ isCleaning = false;
+ LOG.info("Stopping cleaner thread; AggregateStatsCache is now " + getFullPercent()
+ + "% full, with " + getCurrentNodes() + " nodes");
+ LOG.info("Number of expired nodes removed: " + numRemovedTTL);
+ LOG.info("Number of LRU nodes removed: " + numRemovedLRU);
+ LOG.info("Cleaner ran for: " + (System.currentTimeMillis() - cleanerStartTime) + "ms");
+ }
+ }
+ };
+ cleaner.setPriority(Thread.MIN_PRIORITY);
+ cleaner.setDaemon(true);
+ cleaner.start();
+ }
+
+ /**
+ * Evict an LRU node or expired node whichever we find first
+ */
+ private void evictOneNode() {
+ // Get the LRU key, value
+ Key lruKey = null;
+ AggrColStatsList lruValue = null;
+ for (Map.Entry<Key, AggrColStatsList> entry : cacheStore.entrySet()) {
+ Key key = entry.getKey();
+ AggrColStatsList value = entry.getValue();
+ if (lruKey == null) {
+ lruKey = key;
+ lruValue = value;
+ continue;
+ }
+ if ((value.lastAccessTime < lruValue.lastAccessTime) && !(value.nodes.isEmpty())) {
+ lruKey = key;
+ lruValue = value;
+ }
+ }
+ // Now delete a node for this key's list
+ AggrColStatsList candidateList = cacheStore.get(lruKey);
+ boolean isLocked = false;
+ try {
+ isLocked = candidateList.writeLock.tryLock(maxWriterWaitTime, TimeUnit.MILLISECONDS);
+ if (isLocked) {
+ AggrColStats candidate;
+ AggrColStats lruNode = null;
+ int currentIndex = 0;
+ int deleteIndex = 0;
+ for (Iterator<AggrColStats> iterator = candidateList.nodes.iterator(); iterator.hasNext();) {
+ candidate = iterator.next();
+ // Since we have to create space for 1, if we find an expired node we will remove it &
+ // return
+ if (isExpired(candidate)) {
+ iterator.remove();
+ currentNodes.getAndDecrement();
+ numRemovedTTL++;
+ return;
+ }
+ // Sorry, too many ifs but this form looks optimal
+ // Update the LRU node from what we've seen so far
+ if (lruNode == null) {
+ lruNode = candidate;
+ ++currentIndex;
+ continue;
+ }
+ if (lruNode != null) {
+ if (candidate.lastAccessTime < lruNode.lastAccessTime) {
+ lruNode = candidate;
+ deleteIndex = currentIndex;
+ }
+ }
+ }
+ candidateList.nodes.remove(deleteIndex);
+ currentNodes.getAndDecrement();
+ numRemovedLRU++;
+ }
+ } catch (InterruptedException e) {
+ LOG.debug("Interrupted Exception ignored ",e);
+ } finally {
+ if (isLocked) {
+ candidateList.writeLock.unlock();
+ }
+ }
+ }
+
+ private boolean isExpired(AggrColStats aggrColStats) {
+ return (System.currentTimeMillis() - aggrColStats.lastAccessTime) > timeToLiveMs;
+ }
+
+ /**
+ * Key object for the stats cache hashtable
+ */
+ static class Key {
+ private final String dbName;
+ private final String tblName;
+ private final String colName;
+
+ Key(String db, String table, String col) {
+ // Don't construct an illegal cache key
+ if ((db == null) || (table == null) || (col == null)) {
+ throw new IllegalArgumentException("dbName, tblName, colName can't be null");
+ }
+ dbName = db;
+ tblName = table;
+ colName = col;
+ }
+
+ @Override
+ public boolean equals(Object other) {
+ if ((other == null) || !(other instanceof Key)) {
+ return false;
+ }
+ Key that = (Key) other;
+ return dbName.equals(that.dbName) && tblName.equals(that.tblName)
+ && colName.equals(that.colName);
+ }
+
+ @Override
+ public int hashCode() {
+ return dbName.hashCode() * 31 + tblName.hashCode() * 31 + colName.hashCode();
+ }
+
+ @Override
+ public String toString() {
+ return "database:" + dbName + ", table:" + tblName + ", column:" + colName;
+ }
+
+ }
+
+ static class AggrColStatsList {
+ // TODO: figure out a better data structure for node list(?)
+ private List<AggrColStats> nodes = new ArrayList<>();
+ private final ReadWriteLock lock = new ReentrantReadWriteLock();
+ // Read lock for get operation
+ private final Lock readLock = lock.readLock();
+ // Write lock for add, evict and clean operation
+ private final Lock writeLock = lock.writeLock();
+ // Using volatile instead of locking updates to this variable,
+ // since we can rely on approx lastAccessTime but don't want a performance hit
+ private volatile long lastAccessTime = 0;
+
+ List<AggrColStats> getNodes() {
+ return nodes;
+ }
+
+ void updateLastAccessTime() {
+ this.lastAccessTime = System.currentTimeMillis();
+ }
+ }
+
+ public static class AggrColStats {
+ private final long numPartsCached;
+ private final BloomFilter bloomFilter;
+ private final ColumnStatisticsObj colStats;
+ private volatile long lastAccessTime;
+
+ public AggrColStats(long numPartsCached, BloomFilter bloomFilter,
+ ColumnStatisticsObj colStats) {
+ this.numPartsCached = numPartsCached;
+ this.bloomFilter = bloomFilter;
+ this.colStats = colStats;
+ this.lastAccessTime = System.currentTimeMillis();
+ }
+
+ public long getNumPartsCached() {
+ return numPartsCached;
+ }
+
+ public ColumnStatisticsObj getColStats() {
+ return colStats;
+ }
+
+ public BloomFilter getBloomFilter() {
+ return bloomFilter;
+ }
+
+ void updateLastAccessTime() {
+ this.lastAccessTime = System.currentTimeMillis();
+ }
+ }
+
+ /**
+ * Intermediate object, used to collect hits & misses for each cache node that is evaluate for an
+ * incoming request
+ */
+ private static class MatchStats {
+ private int hits = 0;
+ private int misses = 0;
+
+ MatchStats(int hits, int misses) {
+ this.hits = hits;
+ this.misses = misses;
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/hive/blob/b0b6db73/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/DatabaseProduct.java
----------------------------------------------------------------------
diff --git a/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/DatabaseProduct.java b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/DatabaseProduct.java
new file mode 100644
index 0000000..03a5e4a
--- /dev/null
+++ b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/DatabaseProduct.java
@@ -0,0 +1,75 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hive.metastore;
+
+import java.sql.SQLException;
+import java.sql.SQLTransactionRollbackException;
+
+/** Database product infered via JDBC. */
+public enum DatabaseProduct {
+ DERBY, MYSQL, POSTGRES, ORACLE, SQLSERVER, OTHER;
+
+ /**
+ * Determine the database product type
+ * @param productName string to defer database connection
+ * @return database product type
+ */
+ public static DatabaseProduct determineDatabaseProduct(String productName) throws SQLException {
+ if (productName == null) {
+ return OTHER;
+ }
+ productName = productName.toLowerCase();
+ if (productName.contains("derby")) {
+ return DERBY;
+ } else if (productName.contains("microsoft sql server")) {
+ return SQLSERVER;
+ } else if (productName.contains("mysql")) {
+ return MYSQL;
+ } else if (productName.contains("oracle")) {
+ return ORACLE;
+ } else if (productName.contains("postgresql")) {
+ return POSTGRES;
+ } else {
+ return OTHER;
+ }
+ }
+
+ public static boolean isDeadlock(DatabaseProduct dbProduct, SQLException e) {
+ return e instanceof SQLTransactionRollbackException
+ || ((dbProduct == MYSQL || dbProduct == POSTGRES || dbProduct == SQLSERVER)
+ && e.getSQLState().equals("40001"))
+ || (dbProduct == POSTGRES && e.getSQLState().equals("40P01"))
+ || (dbProduct == ORACLE && (e.getMessage().contains("deadlock detected")
+ || e.getMessage().contains("can't serialize access for this transaction")));
+ }
+
+ /**
+ * Whether the RDBMS has restrictions on IN list size (explicit, or poor perf-based).
+ */
+ public static boolean needsInBatching(DatabaseProduct dbType) {
+ return dbType == ORACLE || dbType == SQLSERVER;
+ }
+
+ /**
+ * Whether the RDBMS has a bug in join and filter operation order described in DERBY-6358.
+ */
+ public static boolean hasJoinOperationOrderBug(DatabaseProduct dbType) {
+ return dbType == DERBY || dbType == ORACLE;
+ }
+}
http://git-wip-us.apache.org/repos/asf/hive/blob/b0b6db73/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/DefaultHiveMetaHook.java
----------------------------------------------------------------------
diff --git a/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/DefaultHiveMetaHook.java b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/DefaultHiveMetaHook.java
new file mode 100644
index 0000000..0957945
--- /dev/null
+++ b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/DefaultHiveMetaHook.java
@@ -0,0 +1,51 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hive.metastore;
+
+import org.apache.hadoop.hive.metastore.api.MetaException;
+import org.apache.hadoop.hive.metastore.api.Table;
+
+public abstract class DefaultHiveMetaHook implements HiveMetaHook {
+ /**
+ * Called after successfully after INSERT [OVERWRITE] statement is executed.
+ * @param table table definition
+ * @param overwrite true if it is INSERT OVERWRITE
+ *
+ * @throws MetaException
+ */
+ public abstract void commitInsertTable(Table table, boolean overwrite) throws MetaException;
+
+ /**
+ * called before commit insert method is called
+ * @param table table definition
+ * @param overwrite true if it is INSERT OVERWRITE
+ *
+ * @throws MetaException
+ */
+ public abstract void preInsertTable(Table table, boolean overwrite) throws MetaException;
+
+ /**
+ * called in case pre commit or commit insert fail.
+ * @param table table definition
+ * @param overwrite true if it is INSERT OVERWRITE
+ *
+ * @throws MetaException
+ */
+ public abstract void rollbackInsertTable(Table table, boolean overwrite) throws MetaException;
+}
http://git-wip-us.apache.org/repos/asf/hive/blob/b0b6db73/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/DefaultMetaStoreFilterHookImpl.java
----------------------------------------------------------------------
diff --git a/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/DefaultMetaStoreFilterHookImpl.java b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/DefaultMetaStoreFilterHookImpl.java
new file mode 100644
index 0000000..020bea8
--- /dev/null
+++ b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/DefaultMetaStoreFilterHookImpl.java
@@ -0,0 +1,102 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hive.metastore;
+
+import java.util.List;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hive.metastore.api.Database;
+import org.apache.hadoop.hive.metastore.api.Index;
+import org.apache.hadoop.hive.metastore.api.MetaException;
+import org.apache.hadoop.hive.metastore.api.NoSuchObjectException;
+import org.apache.hadoop.hive.metastore.api.Partition;
+import org.apache.hadoop.hive.metastore.api.PartitionSpec;
+import org.apache.hadoop.hive.metastore.api.Table;
+
+/**
+ * Default no-op implementation of the MetaStoreFilterHook that returns the result as is
+ */
+public class DefaultMetaStoreFilterHookImpl implements MetaStoreFilterHook {
+
+ public DefaultMetaStoreFilterHookImpl(Configuration conf) {
+ }
+
+ @Override
+ public List<String> filterDatabases(List<String> dbList) throws MetaException {
+ return dbList;
+ }
+
+ @Override
+ public Database filterDatabase(Database dataBase) throws NoSuchObjectException {
+ return dataBase;
+ }
+
+ @Override
+ public List<String> filterTableNames(String dbName, List<String> tableList) throws MetaException {
+ return tableList;
+ }
+
+ @Override
+ public Table filterTable(Table table) throws NoSuchObjectException {
+ return table;
+ }
+
+ @Override
+ public List<Table> filterTables(List<Table> tableList) throws MetaException {
+ return tableList;
+ }
+
+ @Override
+ public List<Partition> filterPartitions(List<Partition> partitionList) throws MetaException {
+ return partitionList;
+ }
+
+ @Override
+ public List<PartitionSpec> filterPartitionSpecs(
+ List<PartitionSpec> partitionSpecList) throws MetaException {
+ return partitionSpecList;
+ }
+
+ @Override
+ public Partition filterPartition(Partition partition) throws NoSuchObjectException {
+ return partition;
+ }
+
+ @Override
+ public List<String> filterPartitionNames(String dbName, String tblName,
+ List<String> partitionNames) throws MetaException {
+ return partitionNames;
+ }
+
+ @Override
+ public Index filterIndex(Index index) throws NoSuchObjectException {
+ return index;
+ }
+
+ @Override
+ public List<String> filterIndexNames(String dbName, String tblName,
+ List<String> indexList) throws MetaException {
+ return indexList;
+ }
+
+ @Override
+ public List<Index> filterIndexes(List<Index> indexeList) throws MetaException {
+ return indexeList;
+ }
+}
[3/7] hive git commit: HIVE-17488 Move first set of classes to
standalone metastore. This closes #244. (Alan Gates,
reviewed by Owen O'Malley)
Posted by ga...@apache.org.
http://git-wip-us.apache.org/repos/asf/hive/blob/b0b6db73/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/FileFormatProxy.java
----------------------------------------------------------------------
diff --git a/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/FileFormatProxy.java b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/FileFormatProxy.java
new file mode 100644
index 0000000..14ff187
--- /dev/null
+++ b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/FileFormatProxy.java
@@ -0,0 +1,64 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hive.metastore;
+
+import java.io.IOException;
+import java.nio.ByteBuffer;
+import java.util.List;
+
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hive.metastore.Metastore.SplitInfos;
+import org.apache.hadoop.hive.ql.io.sarg.SearchArgument;
+
+/**
+ * Same as PartitionExpressionProxy, but for file format specific methods for metadata cache.
+ */
+public interface FileFormatProxy {
+
+ /**
+ * Applies SARG to file metadata, and produces some result for this file.
+ * @param sarg SARG
+ * @param fileMetadata File metadata from metastore cache.
+ * @return The result to return to client for this file, or null if file is eliminated.
+ */
+ SplitInfos applySargToMetadata(SearchArgument sarg, ByteBuffer fileMetadata) throws IOException;
+
+ /**
+ * @param fs The filesystem of the file.
+ * @param path The file path.
+ * @param addedVals Output parameter; additional column values for columns returned by
+ * getAddedColumnsToCache to cache in MS.
+ * @return The ORC file metadata for a given file.
+ */
+ ByteBuffer getMetadataToCache(
+ FileSystem fs, Path path, ByteBuffer[] addedVals) throws IOException;
+
+ /**
+ * @return Additional column names to cache in MS for this format.
+ */
+ ByteBuffer[] getAddedColumnsToCache();
+
+ /**
+ * @param metadata File metadatas.
+ * @return Additional values for columns returned by getAddedColumnsToCache to cache in MS
+ * for respective metadatas.
+ */
+ ByteBuffer[][] getAddedValuesToCache(List<ByteBuffer> metadata);
+
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hive/blob/b0b6db73/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/HiveMetaException.java
----------------------------------------------------------------------
diff --git a/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/HiveMetaException.java b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/HiveMetaException.java
new file mode 100644
index 0000000..5bd5a70
--- /dev/null
+++ b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/HiveMetaException.java
@@ -0,0 +1,42 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hive.metastore;
+
+public class HiveMetaException extends Exception {
+
+ public HiveMetaException() {
+ // TODO Auto-generated constructor stub
+ }
+
+ public HiveMetaException(String message) {
+ super(message);
+ // TODO Auto-generated constructor stub
+ }
+
+ public HiveMetaException(Throwable cause) {
+ super(cause);
+ // TODO Auto-generated constructor stub
+ }
+
+ public HiveMetaException(String message, Throwable cause) {
+ super(message, cause);
+ // TODO Auto-generated constructor stub
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/hive/blob/b0b6db73/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/HiveMetaHook.java
----------------------------------------------------------------------
diff --git a/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/HiveMetaHook.java b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/HiveMetaHook.java
new file mode 100644
index 0000000..2534fa2
--- /dev/null
+++ b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/HiveMetaHook.java
@@ -0,0 +1,95 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hive.metastore;
+
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
+import org.apache.hadoop.hive.metastore.api.MetaException;
+import org.apache.hadoop.hive.metastore.api.Table;
+
+/**
+ * HiveMetaHook defines notification methods which are invoked as part
+ * of transactions against the metastore, allowing external catalogs
+ * such as HBase to be kept in sync with Hive's metastore.
+ *
+ *<p>
+ *
+ * Implementations can use {@link MetaStoreUtils#isExternalTable} to
+ * distinguish external tables from managed tables.
+ */
+@InterfaceAudience.Public
+@InterfaceStability.Stable
+public interface HiveMetaHook {
+ /**
+ * Called before a new table definition is added to the metastore
+ * during CREATE TABLE.
+ *
+ * @param table new table definition
+ */
+ public void preCreateTable(Table table)
+ throws MetaException;
+
+ /**
+ * Called after failure adding a new table definition to the metastore
+ * during CREATE TABLE.
+ *
+ * @param table new table definition
+ */
+ public void rollbackCreateTable(Table table)
+ throws MetaException;
+
+ /**
+ * Called after successfully adding a new table definition to the metastore
+ * during CREATE TABLE.
+ *
+ * @param table new table definition
+ */
+ public void commitCreateTable(Table table)
+ throws MetaException;
+
+ /**
+ * Called before a table definition is removed from the metastore
+ * during DROP TABLE.
+ *
+ * @param table table definition
+ */
+ public void preDropTable(Table table)
+ throws MetaException;
+
+ /**
+ * Called after failure removing a table definition from the metastore
+ * during DROP TABLE.
+ *
+ * @param table table definition
+ */
+ public void rollbackDropTable(Table table)
+ throws MetaException;
+
+ /**
+ * Called after successfully removing a table definition from the metastore
+ * during DROP TABLE.
+ *
+ * @param table table definition
+ *
+ * @param deleteData whether to delete data as well; this should typically
+ * be ignored in the case of an external table
+ */
+ public void commitDropTable(Table table, boolean deleteData)
+ throws MetaException;
+}
http://git-wip-us.apache.org/repos/asf/hive/blob/b0b6db73/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/HiveMetaHookLoader.java
----------------------------------------------------------------------
diff --git a/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/HiveMetaHookLoader.java b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/HiveMetaHookLoader.java
new file mode 100644
index 0000000..1cdae9b
--- /dev/null
+++ b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/HiveMetaHookLoader.java
@@ -0,0 +1,39 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hive.metastore;
+
+import org.apache.hadoop.hive.metastore.api.MetaException;
+import org.apache.hadoop.hive.metastore.api.Table;
+
+/**
+ * HiveMetaHookLoader is responsible for loading a {@link HiveMetaHook}
+ * for a given table.
+ */
+public interface HiveMetaHookLoader {
+ /**
+ * Loads a hook for the specified table.
+ *
+ * @param tbl table of interest
+ *
+ * @return hook, or null if none registered
+ */
+ public HiveMetaHook getHook(Table tbl) throws MetaException;
+}
+
+// End HiveMetaHookLoader.java
http://git-wip-us.apache.org/repos/asf/hive/blob/b0b6db73/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/HiveMetaStoreFsImpl.java
----------------------------------------------------------------------
diff --git a/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/HiveMetaStoreFsImpl.java b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/HiveMetaStoreFsImpl.java
new file mode 100644
index 0000000..8322cb8
--- /dev/null
+++ b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/HiveMetaStoreFsImpl.java
@@ -0,0 +1,53 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hive.metastore;
+
+import java.io.FileNotFoundException;
+
+import org.apache.hadoop.hive.metastore.utils.FileUtils;
+import org.apache.hadoop.hive.metastore.utils.MetaStoreUtils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hive.metastore.api.MetaException;
+
+public class HiveMetaStoreFsImpl implements MetaStoreFS {
+
+ public static final Logger LOG = LoggerFactory
+ .getLogger("hive.metastore.hivemetastoreFsimpl");
+
+ @Override
+ public boolean deleteDir(FileSystem fs, Path f, boolean recursive,
+ boolean ifPurge, Configuration conf) throws MetaException {
+ try {
+ FileUtils.moveToTrash(fs, f, conf, ifPurge);
+ if (fs.exists(f)) {
+ throw new MetaException("Unable to delete directory: " + f);
+ }
+ return true;
+ } catch (FileNotFoundException e) {
+ return true; // ok even if there is not data
+ } catch (Exception e) {
+ MetaStoreUtils.logAndThrowMetaException(e);
+ }
+ return false;
+ }
+}
http://git-wip-us.apache.org/repos/asf/hive/blob/b0b6db73/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/IExtrapolatePartStatus.java
----------------------------------------------------------------------
diff --git a/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/IExtrapolatePartStatus.java b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/IExtrapolatePartStatus.java
new file mode 100644
index 0000000..aa50e91
--- /dev/null
+++ b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/IExtrapolatePartStatus.java
@@ -0,0 +1,85 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hive.metastore;
+
+import java.util.HashMap;
+import java.util.Map;
+
+public interface IExtrapolatePartStatus {
+ /**
+ * The sequence of colStatNames.
+ */
+ static String[] colStatNames = new String[] { "LONG_LOW_VALUE", "LONG_HIGH_VALUE",
+ "DOUBLE_LOW_VALUE", "DOUBLE_HIGH_VALUE", "BIG_DECIMAL_LOW_VALUE", "BIG_DECIMAL_HIGH_VALUE",
+ "NUM_NULLS", "NUM_DISTINCTS", "AVG_COL_LEN", "MAX_COL_LEN", "NUM_TRUES", "NUM_FALSES",
+ "AVG_NDV_LONG", "AVG_NDV_DOUBLE", "AVG_NDV_DECIMAL", "SUM_NUM_DISTINCTS" };
+
+ /**
+ * The indexes for colstats.
+ */
+ static HashMap<String, Integer[]> indexMaps = new HashMap<String, Integer[]>() {
+ {
+ put("bigint", new Integer[] { 0, 1, 6, 7, 12, 15 });
+ put("int", new Integer[] { 0, 1, 6, 7, 12, 15 });
+ put("smallint", new Integer[] { 0, 1, 6, 7, 12, 15 });
+ put("tinyint", new Integer[] { 0, 1, 6, 7, 12, 15 });
+ put("date", new Integer[] { 0, 1, 6, 7, 12, 15 });
+ put("timestamp", new Integer[] { 0, 1, 6, 7, 12, 15 });
+ put("long", new Integer[] { 0, 1, 6, 7, 12, 15 });
+ put("double", new Integer[] { 2, 3, 6, 7, 13, 15 });
+ put("float", new Integer[] { 2, 3, 6, 7, 13, 15 });
+ put("varchar", new Integer[] { 8, 9, 6, 7, 15 });
+ put("char", new Integer[] { 8, 9, 6, 7, 15 });
+ put("string", new Integer[] { 8, 9, 6, 7, 15 });
+ put("boolean", new Integer[] { 10, 11, 6, 15 });
+ put("binary", new Integer[] { 8, 9, 6, 15 });
+ put("decimal", new Integer[] { 4, 5, 6, 7, 14, 15 });
+ put("default", new Integer[] { 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 15 });
+ }
+ };
+
+ /**
+ * The sequence of colStatTypes.
+ */
+ static enum ColStatType {
+ Long, Double, Decimal
+ }
+
+ static ColStatType[] colStatTypes = new ColStatType[] { ColStatType.Long, ColStatType.Long,
+ ColStatType.Double, ColStatType.Double, ColStatType.Decimal, ColStatType.Decimal,
+ ColStatType.Long, ColStatType.Long, ColStatType.Double, ColStatType.Long, ColStatType.Long,
+ ColStatType.Long, ColStatType.Double, ColStatType.Double, ColStatType.Double,
+ ColStatType.Long };
+
+ /**
+ * The sequence of aggregation function on colStats.
+ */
+ static enum AggrType {
+ Min, Max, Sum, Avg
+ }
+
+ static AggrType[] aggrTypes = new AggrType[] { AggrType.Min, AggrType.Max, AggrType.Min,
+ AggrType.Max, AggrType.Min, AggrType.Max, AggrType.Sum, AggrType.Max, AggrType.Max,
+ AggrType.Max, AggrType.Sum, AggrType.Sum, AggrType.Avg, AggrType.Avg, AggrType.Avg,
+ AggrType.Sum };
+
+ public Object extrapolate(Object[] min, Object[] max, int colStatIndex,
+ Map<String, Integer> indexMap);
+
+}
http://git-wip-us.apache.org/repos/asf/hive/blob/b0b6db73/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/IHMSHandler.java
----------------------------------------------------------------------
diff --git a/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/IHMSHandler.java b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/IHMSHandler.java
new file mode 100644
index 0000000..633b3c7
--- /dev/null
+++ b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/IHMSHandler.java
@@ -0,0 +1,28 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hive.metastore;
+
+import org.apache.hadoop.conf.Configurable;
+import org.apache.hadoop.hive.metastore.api.MetaException;
+import org.apache.hadoop.hive.metastore.api.ThriftHiveMetastore;
+
+public interface IHMSHandler extends ThriftHiveMetastore.Iface, Configurable {
+
+ void init() throws MetaException;
+}
http://git-wip-us.apache.org/repos/asf/hive/blob/b0b6db73/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/IMetaStoreSchemaInfo.java
----------------------------------------------------------------------
diff --git a/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/IMetaStoreSchemaInfo.java b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/IMetaStoreSchemaInfo.java
new file mode 100644
index 0000000..be89f9b
--- /dev/null
+++ b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/IMetaStoreSchemaInfo.java
@@ -0,0 +1,108 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hive.metastore;
+
+
+import java.sql.Connection;
+import java.util.List;
+
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.hive.metastore.tools.HiveSchemaHelper;
+
+/**
+ * Defines the method which must be implemented to be used using schema tool to support metastore
+ * schema upgrades. The configuration hive.metastore.schema.info.class is used to create instances
+ * of this type by SchemaTool.
+ *
+ * Instances of this interface should be created using MetaStoreSchemaInfoFactory class which uses
+ * two Strings argument constructor to instantiate the implementations of this interface
+ */
+@InterfaceAudience.Private
+public interface IMetaStoreSchemaInfo {
+ String SQL_FILE_EXTENSION = ".sql";
+
+ /***
+ * Get the list of sql scripts required to upgrade from the give version to current.
+ *
+ * @param fromVersion
+ * @return
+ * @throws HiveMetaException
+ */
+ List<String> getUpgradeScripts(String fromVersion) throws HiveMetaException;
+
+ /***
+ * Get the name of the script to initialize the schema for given version
+ *
+ * @param toVersion Target version. If it's null, then the current server version is used
+ * @return
+ * @throws HiveMetaException
+ */
+ String generateInitFileName(String toVersion) throws HiveMetaException;
+
+ /**
+ * Find the directory of metastore scripts
+ *
+ * @return the path of directory where the sql scripts are
+ */
+ String getMetaStoreScriptDir();
+
+ /**
+ * Get the pre-upgrade script for a given script name. Schema tool runs the pre-upgrade scripts
+ * returned by this method before running any upgrade scripts. These scripts could contain setup
+ * statements may fail on some database versions and failure is ignorable.
+ *
+ * @param index - index number of the file. The preupgrade script name is derived using the given
+ * index
+ * @param scriptName - upgrade script name
+ * @return name of the pre-upgrade script to be run before running upgrade script
+ */
+ String getPreUpgradeScriptName(int index, String scriptName);
+
+ /**
+ * Get hive distribution schema version. Schematool uses this version to identify
+ * the Hive version. It compares this version with the version found in metastore database
+ * to determine the upgrade or initialization scripts
+ * @return Hive schema version
+ */
+ String getHiveSchemaVersion();
+
+ /**
+ * Get the schema version from the backend database. This version is used by SchemaTool to to
+ * compare the version returned by getHiveSchemaVersion and determine the upgrade order and
+ * scripts needed to upgrade the metastore schema
+ *
+ * @param metastoreDbConnectionInfo Connection information needed to connect to the backend
+ * database
+ * @return
+ * @throws HiveMetaException when unable to fetch the schema version
+ */
+ String getMetaStoreSchemaVersion(
+ HiveSchemaHelper.MetaStoreConnectionInfo metastoreDbConnectionInfo) throws HiveMetaException;
+ /**
+ * A dbVersion is compatible with hive version if it is greater or equal to the hive version. This
+ * is result of the db schema upgrade design principles followed in hive project. The state where
+ * db schema version is ahead of hive software version is often seen when a 'rolling upgrade' or
+ * 'rolling downgrade' is happening. This is a state where hive is functional and returning non
+ * zero status for it is misleading.
+ *
+ * @param productVersion version of hive software
+ * @param dbVersion version of metastore rdbms schema
+ * @return true if versions are compatible
+ */
+ boolean isVersionCompatible(String productVersion, String dbVersion);
+}
http://git-wip-us.apache.org/repos/asf/hive/blob/b0b6db73/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/LinearExtrapolatePartStatus.java
----------------------------------------------------------------------
diff --git a/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/LinearExtrapolatePartStatus.java b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/LinearExtrapolatePartStatus.java
new file mode 100644
index 0000000..526b55b
--- /dev/null
+++ b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/LinearExtrapolatePartStatus.java
@@ -0,0 +1,106 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hive.metastore;
+
+import java.math.BigDecimal;
+import java.util.Map;
+
+public class LinearExtrapolatePartStatus implements IExtrapolatePartStatus {
+
+ @Override
+ public Object extrapolate(Object[] min, Object[] max, int colStatIndex,
+ Map<String, Integer> indexMap) {
+ int rightBorderInd = indexMap.size() - 1;
+ int minInd = indexMap.get((String) min[1]);
+ int maxInd = indexMap.get((String) max[1]);
+ if (minInd == maxInd) {
+ return min[0];
+ }
+ //note that recent metastore stores decimal in string.
+ double decimalmin= 0;
+ double decimalmax = 0;
+ if (colStatTypes[colStatIndex] == ColStatType.Decimal) {
+ BigDecimal bdmin = new BigDecimal(min[0].toString());
+ decimalmin = bdmin.doubleValue();
+ BigDecimal bdmax = new BigDecimal(max[0].toString());
+ decimalmax = bdmax.doubleValue();
+ }
+ if (aggrTypes[colStatIndex] == AggrType.Max) {
+ if (minInd < maxInd) {
+ // right border is the max
+ if (colStatTypes[colStatIndex] == ColStatType.Long) {
+ return (Long) ((Long) min[0] + (((Long) max[0] - (Long) min[0])
+ * (rightBorderInd - minInd) / (maxInd - minInd)));
+ } else if (colStatTypes[colStatIndex] == ColStatType.Double) {
+ return (Double) ((Double) min[0] + (((Double) max[0] - (Double) min[0])
+ * (rightBorderInd - minInd) / (maxInd - minInd)));
+ } else {
+ double ret = decimalmin + (decimalmax - decimalmin)
+ * (rightBorderInd - minInd) / (maxInd - minInd);
+ return String.valueOf(ret);
+ }
+ } else {
+ // left border is the max
+ if (colStatTypes[colStatIndex] == ColStatType.Long) {
+ return (Long) ((Long) min[0] + ((Long) max[0] - (Long) min[0])
+ * minInd / (minInd - maxInd));
+ } else if (colStatTypes[colStatIndex] == ColStatType.Double) {
+ return (Double) ((Double) min[0] + ((Double) max[0] - (Double) min[0])
+ * minInd / (minInd - maxInd));
+ } else {
+ double ret = decimalmin + (decimalmax - decimalmin) * minInd
+ / (minInd - maxInd);
+ return String.valueOf(ret);
+ }
+ }
+ } else {
+ if (minInd < maxInd) {
+ // left border is the min
+ if (colStatTypes[colStatIndex] == ColStatType.Long) {
+ Long ret = (Long) max[0] - ((Long) max[0] - (Long) min[0]) * maxInd
+ / (maxInd - minInd);
+ return ret;
+ } else if (colStatTypes[colStatIndex] == ColStatType.Double) {
+ Double ret = (Double) max[0] - ((Double) max[0] - (Double) min[0])
+ * maxInd / (maxInd - minInd);
+ return ret;
+ } else {
+ double ret = decimalmax - (decimalmax - decimalmin) * maxInd
+ / (maxInd - minInd);
+ return String.valueOf(ret);
+ }
+ } else {
+ // right border is the min
+ if (colStatTypes[colStatIndex] == ColStatType.Long) {
+ Long ret = (Long) max[0] - ((Long) max[0] - (Long) min[0])
+ * (rightBorderInd - maxInd) / (minInd - maxInd);
+ return ret;
+ } else if (colStatTypes[colStatIndex] == ColStatType.Double) {
+ Double ret = (Double) max[0] - ((Double) max[0] - (Double) min[0])
+ * (rightBorderInd - maxInd) / (minInd - maxInd);
+ return ret;
+ } else {
+ double ret = decimalmax - (decimalmax - decimalmin)
+ * (rightBorderInd - maxInd) / (minInd - maxInd);
+ return String.valueOf(ret);
+ }
+ }
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/hive/blob/b0b6db73/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/LockComponentBuilder.java
----------------------------------------------------------------------
diff --git a/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/LockComponentBuilder.java b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/LockComponentBuilder.java
new file mode 100644
index 0000000..de6c718
--- /dev/null
+++ b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/LockComponentBuilder.java
@@ -0,0 +1,121 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hive.metastore;
+
+import org.apache.hadoop.hive.metastore.api.DataOperationType;
+import org.apache.hadoop.hive.metastore.api.LockComponent;
+import org.apache.hadoop.hive.metastore.api.LockLevel;
+import org.apache.hadoop.hive.metastore.api.LockType;
+
+/**
+ * A builder for {@link LockComponent}s
+ */
+public class LockComponentBuilder {
+ private LockComponent component;
+ private boolean tableNameSet;
+ private boolean partNameSet;
+
+ public LockComponentBuilder() {
+ component = new LockComponent();
+ tableNameSet = partNameSet = false;
+ }
+
+ /**
+ * Set the lock to be exclusive.
+ * @return reference to this builder
+ */
+ public LockComponentBuilder setExclusive() {
+ component.setType(LockType.EXCLUSIVE);
+ return this;
+ }
+
+ /**
+ * Set the lock to be semi-shared.
+ * @return reference to this builder
+ */
+ public LockComponentBuilder setSemiShared() {
+ component.setType(LockType.SHARED_WRITE);
+ return this;
+ }
+
+ /**
+ * Set the lock to be shared.
+ * @return reference to this builder
+ */
+ public LockComponentBuilder setShared() {
+ component.setType(LockType.SHARED_READ);
+ return this;
+ }
+
+ /**
+ * Set the database name.
+ * @param dbName database name
+ * @return reference to this builder
+ */
+ public LockComponentBuilder setDbName(String dbName) {
+ component.setDbname(dbName);
+ return this;
+ }
+
+ public LockComponentBuilder setOperationType(DataOperationType dop) {
+ component.setOperationType(dop);
+ return this;
+ }
+
+ public LockComponentBuilder setIsAcid(boolean t) {
+ component.setIsAcid(t);
+ return this;
+ }
+ /**
+ * Set the table name.
+ * @param tableName table name
+ * @return reference to this builder
+ */
+ public LockComponentBuilder setTableName(String tableName) {
+ component.setTablename(tableName);
+ tableNameSet = true;
+ return this;
+ }
+
+ /**
+ * Set the partition name.
+ * @param partitionName partition name
+ * @return reference to this builder
+ */
+ public LockComponentBuilder setPartitionName(String partitionName) {
+ component.setPartitionname(partitionName);
+ partNameSet = true;
+ return this;
+ }
+ public LockComponentBuilder setIsDynamicPartitionWrite(boolean t) {
+ component.setIsDynamicPartitionWrite(t);
+ return this;
+ }
+
+ /**
+ * Get the constructed lock component.
+ * @return lock component.
+ */
+ public LockComponent build() {
+ LockLevel level = LockLevel.DB;
+ if (tableNameSet) level = LockLevel.TABLE;
+ if (partNameSet) level = LockLevel.PARTITION;
+ component.setLevel(level);
+ return component;
+ }
+}
http://git-wip-us.apache.org/repos/asf/hive/blob/b0b6db73/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/LockRequestBuilder.java
----------------------------------------------------------------------
diff --git a/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/LockRequestBuilder.java b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/LockRequestBuilder.java
new file mode 100644
index 0000000..d03c73a
--- /dev/null
+++ b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/LockRequestBuilder.java
@@ -0,0 +1,168 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hive.metastore;
+
+import org.apache.hadoop.hive.metastore.api.LockComponent;
+import org.apache.hadoop.hive.metastore.api.LockRequest;
+import org.apache.hadoop.hive.metastore.api.LockType;
+
+import java.net.InetAddress;
+import java.net.UnknownHostException;
+import java.util.LinkedHashMap;
+import java.util.Map;
+
+/**
+ * Builder class to make constructing {@link LockRequest} easier.
+ */
+public class LockRequestBuilder {
+
+ private LockRequest req;
+ private LockTrie trie;
+ private boolean userSet;
+
+ /**
+ * @deprecated
+ */
+ public LockRequestBuilder() {
+ this(null);
+ }
+ public LockRequestBuilder(String agentInfo) {
+ req = new LockRequest();
+ trie = new LockTrie();
+ userSet = false;
+ if(agentInfo != null) {
+ req.setAgentInfo(agentInfo);
+ }
+ }
+
+ /**
+ * Get the constructed LockRequest.
+ * @return lock request
+ */
+ public LockRequest build() {
+ if (!userSet) {
+ throw new RuntimeException("Cannot build a lock without giving a user");
+ }
+ trie.addLocksToRequest(req);
+ try {
+ req.setHostname(InetAddress.getLocalHost().getHostName());
+ } catch (UnknownHostException e) {
+ throw new RuntimeException("Unable to determine our local host!");
+ }
+ return req;
+ }
+
+ /**
+ * Set the transaction id.
+ * @param txnid transaction id
+ * @return reference to this builder
+ */
+ public LockRequestBuilder setTransactionId(long txnid) {
+ req.setTxnid(txnid);
+ return this;
+ }
+
+ public LockRequestBuilder setUser(String user) {
+ if (user == null) user = "unknown";
+ req.setUser(user);
+ userSet = true;
+ return this;
+ }
+
+ /**
+ * Add a lock component to the lock request
+ * @param component to add
+ * @return reference to this builder
+ */
+ public LockRequestBuilder addLockComponent(LockComponent component) {
+ trie.add(component);
+ return this;
+ }
+
+ // For reasons that are completely incomprehensible to me the semantic
+ // analyzers often ask for multiple locks on the same entity (for example
+ // a shared_read and an exlcusive lock). The db locking system gets confused
+ // by this and dead locks on it. To resolve that, we'll make sure in the
+ // request that multiple locks are coalesced and promoted to the higher
+ // level of locking. To do this we put all locks components in trie based
+ // on dbname, tablename, partition name and handle the promotion as new
+ // requests come in. This structure depends on the fact that null is a
+ // valid key in a LinkedHashMap. So a database lock will map to (dbname, null,
+ // null).
+ private static class LockTrie {
+ Map<String, TableTrie> trie;
+
+ LockTrie() {
+ trie = new LinkedHashMap<>();
+ }
+
+ public void add(LockComponent comp) {
+ TableTrie tabs = trie.get(comp.getDbname());
+ if (tabs == null) {
+ tabs = new TableTrie();
+ trie.put(comp.getDbname(), tabs);
+ }
+ setTable(comp, tabs);
+ }
+
+ public void addLocksToRequest(LockRequest request) {
+ for (TableTrie tab : trie.values()) {
+ for (PartTrie part : tab.values()) {
+ for (LockComponent lock : part.values()) {
+ request.addToComponent(lock);
+ }
+ }
+ }
+ }
+
+ private void setTable(LockComponent comp, TableTrie tabs) {
+ PartTrie parts = tabs.get(comp.getTablename());
+ if (parts == null) {
+ parts = new PartTrie();
+ tabs.put(comp.getTablename(), parts);
+ }
+ setPart(comp, parts);
+ }
+
+ private void setPart(LockComponent comp, PartTrie parts) {
+ LockComponent existing = parts.get(comp.getPartitionname());
+ if (existing == null) {
+ // No existing lock for this partition.
+ parts.put(comp.getPartitionname(), comp);
+ } else if (existing.getType() != LockType.EXCLUSIVE &&
+ (comp.getType() == LockType.EXCLUSIVE ||
+ comp.getType() == LockType.SHARED_WRITE)) {
+ // We only need to promote if comp.type is > existing.type. For
+ // efficiency we check if existing is exclusive (in which case we
+ // need never promote) or if comp is exclusive or shared_write (in
+ // which case we can promote even though they may both be shared
+ // write). If comp is shared_read there's never a need to promote.
+ parts.put(comp.getPartitionname(), comp);
+ }
+ }
+
+ private static class TableTrie extends LinkedHashMap<String, PartTrie> {
+ }
+
+ private static class PartTrie extends LinkedHashMap<String, LockComponent> {
+ }
+
+
+
+ }
+}
http://git-wip-us.apache.org/repos/asf/hive/blob/b0b6db73/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/MetaStoreEndFunctionContext.java
----------------------------------------------------------------------
diff --git a/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/MetaStoreEndFunctionContext.java b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/MetaStoreEndFunctionContext.java
new file mode 100644
index 0000000..998531f
--- /dev/null
+++ b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/MetaStoreEndFunctionContext.java
@@ -0,0 +1,59 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hive.metastore;
+
+/**
+ * Base class which provides context to implementations of MetaStoreEndFunctionListener
+ */
+
+public class MetaStoreEndFunctionContext {
+
+ /**
+ * whether method was successful or not.
+ */
+ private final boolean success;
+ private final Exception e;
+ private final String inputTableName;
+
+ public MetaStoreEndFunctionContext(boolean success, Exception e, String inputTableName) {
+ this.success = success;
+ this.e = e;
+ this.inputTableName = inputTableName;
+ }
+
+ public MetaStoreEndFunctionContext(boolean success) {
+ this(success, null, null);
+ }
+
+ /**
+ * @return whether or not the method succeeded.
+ */
+ public boolean isSuccess() {
+ return success;
+ }
+
+ public Exception getException() {
+ return e;
+ }
+
+ public String getInputTableName() {
+ return inputTableName;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/hive/blob/b0b6db73/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/MetaStoreEndFunctionListener.java
----------------------------------------------------------------------
diff --git a/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/MetaStoreEndFunctionListener.java b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/MetaStoreEndFunctionListener.java
new file mode 100644
index 0000000..bc7e0be
--- /dev/null
+++ b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/MetaStoreEndFunctionListener.java
@@ -0,0 +1,58 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hive.metastore;
+
+import java.util.AbstractMap;
+
+import org.apache.hadoop.conf.Configurable;
+import org.apache.hadoop.conf.Configuration;
+
+/**
+ * This abstract class needs to be extended to provide implementation of actions that need
+ * to be performed when a function ends. These methods are called whenever a function ends.
+ *
+ * It also provides a way to add fb303 counters through the exportCounters method.
+ */
+
+public abstract class MetaStoreEndFunctionListener implements Configurable {
+
+ private Configuration conf;
+
+ public MetaStoreEndFunctionListener(Configuration config){
+ this.conf = config;
+ }
+
+ public abstract void onEndFunction(String functionName, MetaStoreEndFunctionContext context);
+
+ // Unless this is overridden, it does nothing
+ public void exportCounters(AbstractMap<String, Long> counters) {
+ }
+
+ @Override
+ public Configuration getConf() {
+ return this.conf;
+ }
+
+ @Override
+ public void setConf(Configuration config) {
+ this.conf = config;
+ }
+
+
+}
http://git-wip-us.apache.org/repos/asf/hive/blob/b0b6db73/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/MetaStoreFS.java
----------------------------------------------------------------------
diff --git a/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/MetaStoreFS.java b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/MetaStoreFS.java
new file mode 100644
index 0000000..ddcda4c
--- /dev/null
+++ b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/MetaStoreFS.java
@@ -0,0 +1,43 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hive.metastore;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hive.metastore.api.MetaException;
+
+/**
+ * Define a set of APIs that may vary in different environments
+ */
+public interface MetaStoreFS {
+
+ /**
+ * delete a directory
+ *
+ * @param f
+ * @param ifPurge
+ * @param recursive
+ * @return true on success
+ * @throws MetaException
+ */
+ public boolean deleteDir(FileSystem fs, Path f, boolean recursive,
+ boolean ifPurge, Configuration conf) throws MetaException;
+
+}
http://git-wip-us.apache.org/repos/asf/hive/blob/b0b6db73/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/MetaStoreFilterHook.java
----------------------------------------------------------------------
diff --git a/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/MetaStoreFilterHook.java b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/MetaStoreFilterHook.java
new file mode 100644
index 0000000..fb341b2
--- /dev/null
+++ b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/MetaStoreFilterHook.java
@@ -0,0 +1,132 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hive.metastore;
+
+import java.util.List;
+
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
+import org.apache.hadoop.hive.metastore.api.Database;
+import org.apache.hadoop.hive.metastore.api.Index;
+import org.apache.hadoop.hive.metastore.api.MetaException;
+import org.apache.hadoop.hive.metastore.api.NoSuchObjectException;
+import org.apache.hadoop.hive.metastore.api.Partition;
+import org.apache.hadoop.hive.metastore.api.PartitionSpec;
+import org.apache.hadoop.hive.metastore.api.Table;
+
+/**
+ * Metadata filter hook for metastore client. This will be useful for authorization
+ * plugins on hiveserver2 to filter metadata results, especially in case of
+ * non-impersonation mode where the metastore doesn't know the end user's identity.
+ */
+@InterfaceAudience.LimitedPrivate(value = {"Apache Sentry (Incubating)" })
+@InterfaceStability.Evolving
+public interface MetaStoreFilterHook {
+
+ /**
+ * Filter given list of databases
+ * @param dbList
+ * @return List of filtered Db names
+ */
+ public List<String> filterDatabases(List<String> dbList) throws MetaException;
+
+ /**
+ * filter to given database object if applicable
+ * @param dataBase
+ * @return the same database if it's not filtered out
+ * @throws NoSuchObjectException
+ */
+ public Database filterDatabase(Database dataBase) throws MetaException, NoSuchObjectException;
+
+ /**
+ * Filter given list of tables
+ * @param dbName
+ * @param tableList
+ * @return List of filtered table names
+ */
+ public List<String> filterTableNames(String dbName, List<String> tableList) throws MetaException;
+
+ /**
+ * filter to given table object if applicable
+ * @param table
+ * @return the same table if it's not filtered out
+ * @throws NoSuchObjectException
+ */
+ public Table filterTable(Table table) throws MetaException, NoSuchObjectException;
+
+ /**
+ * Filter given list of tables
+ * @param tableList
+ * @return List of filtered table names
+ */
+ public List<Table> filterTables(List<Table> tableList) throws MetaException;
+
+ /**
+ * Filter given list of partitions
+ * @param partitionList
+ * @return
+ */
+ public List<Partition> filterPartitions(List<Partition> partitionList) throws MetaException;
+
+ /**
+ * Filter given list of partition specs
+ * @param partitionSpecList
+ * @return
+ */
+ public List<PartitionSpec> filterPartitionSpecs(List<PartitionSpec> partitionSpecList)
+ throws MetaException;
+
+ /**
+ * filter to given partition object if applicable
+ * @param partition
+ * @return the same partition object if it's not filtered out
+ * @throws NoSuchObjectException
+ */
+ public Partition filterPartition(Partition partition) throws MetaException, NoSuchObjectException;
+
+ /**
+ * Filter given list of partition names
+ * @param dbName
+ * @param tblName
+ * @param partitionNames
+ * @return
+ */
+ public List<String> filterPartitionNames(String dbName, String tblName,
+ List<String> partitionNames) throws MetaException;
+
+ public Index filterIndex(Index index) throws MetaException, NoSuchObjectException;
+
+ /**
+ * Filter given list of index names
+ * @param dbName
+ * @param tblName
+ * @param indexList
+ * @return
+ */
+ public List<String> filterIndexNames(String dbName, String tblName,
+ List<String> indexList) throws MetaException;
+
+ /**
+ * Filter given list of index objects
+ * @param indexeList
+ * @return
+ */
+ public List<Index> filterIndexes(List<Index> indexeList) throws MetaException;
+}
+
http://git-wip-us.apache.org/repos/asf/hive/blob/b0b6db73/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/MetaStoreInit.java
----------------------------------------------------------------------
diff --git a/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/MetaStoreInit.java b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/MetaStoreInit.java
new file mode 100644
index 0000000..d3eee85
--- /dev/null
+++ b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/MetaStoreInit.java
@@ -0,0 +1,109 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hive.metastore;
+
+import org.apache.hadoop.hive.metastore.conf.MetastoreConf;
+import org.apache.hadoop.hive.metastore.conf.MetastoreConf.ConfVars;
+import org.apache.hadoop.hive.metastore.utils.JavaUtils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hive.metastore.api.MetaException;
+import org.apache.hadoop.hive.metastore.hooks.JDOConnectionURLHook;
+import org.apache.hadoop.util.ReflectionUtils;
+
+/**
+ * MetaStoreInit defines functions to init/update MetaStore connection url.
+ *
+ */
+public class MetaStoreInit {
+
+ private static final Logger LOG = LoggerFactory.getLogger(MetaStoreInit.class);
+
+ static class MetaStoreInitData {
+ JDOConnectionURLHook urlHook = null;
+ String urlHookClassName = "";
+ }
+
+ /**
+ * Updates the connection URL in hiveConf using the hook (if a hook has been
+ * set using hive.metastore.ds.connection.url.hook property)
+ * @param originalConf - original configuration used to look up hook settings
+ * @param activeConf - the configuration file in use for looking up db url
+ * @param badUrl
+ * @param updateData - hook information
+ * @return true if a new connection URL was loaded into the thread local
+ * configuration
+ * @throws MetaException
+ */
+ static boolean updateConnectionURL(Configuration originalConf, Configuration activeConf,
+ String badUrl, MetaStoreInitData updateData)
+ throws MetaException {
+ String connectUrl = null;
+ String currentUrl = MetaStoreInit.getConnectionURL(activeConf);
+ try {
+ // We always call init because the hook name in the configuration could
+ // have changed.
+ MetaStoreInit.initConnectionUrlHook(originalConf, updateData);
+ if (updateData.urlHook != null) {
+ if (badUrl != null) {
+ updateData.urlHook.notifyBadConnectionUrl(badUrl);
+ }
+ connectUrl = updateData.urlHook.getJdoConnectionUrl(originalConf);
+ }
+ } catch (Exception e) {
+ LOG.error("Exception while getting connection URL from the hook: " +
+ e);
+ }
+
+ if (connectUrl != null && !connectUrl.equals(currentUrl)) {
+ LOG.error(
+ String.format("Overriding %s with %s",
+ MetastoreConf.ConfVars.CONNECTURLKEY.toString(),
+ connectUrl));
+ MetastoreConf.setVar(activeConf, ConfVars.CONNECTURLKEY, connectUrl);
+ return true;
+ }
+ return false;
+ }
+
+ static String getConnectionURL(Configuration conf) {
+ return MetastoreConf.getVar(conf, ConfVars.CONNECTURLKEY, "");
+ }
+
+ // Multiple threads could try to initialize at the same time.
+ synchronized private static void initConnectionUrlHook(Configuration conf,
+ MetaStoreInitData updateData) throws ClassNotFoundException {
+
+ String className = MetastoreConf.getVar(conf, ConfVars.CONNECTURLHOOK, "").trim();
+ if (className.equals("")) {
+ updateData.urlHookClassName = "";
+ updateData.urlHook = null;
+ return;
+ }
+ boolean urlHookChanged = !updateData.urlHookClassName.equals(className);
+ if (updateData.urlHook == null || urlHookChanged) {
+ updateData.urlHookClassName = className.trim();
+
+ Class<?> urlHookClass = Class.forName(updateData.urlHookClassName, true,
+ JavaUtils.getClassLoader());
+ updateData.urlHook = (JDOConnectionURLHook) ReflectionUtils.newInstance(urlHookClass, null);
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/hive/blob/b0b6db73/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/MetaStoreSchemaInfo.java
----------------------------------------------------------------------
diff --git a/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/MetaStoreSchemaInfo.java b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/MetaStoreSchemaInfo.java
new file mode 100644
index 0000000..0c36855
--- /dev/null
+++ b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/MetaStoreSchemaInfo.java
@@ -0,0 +1,234 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hive.metastore;
+
+import java.io.BufferedReader;
+import java.io.File;
+import java.io.FileNotFoundException;
+import java.io.FileReader;
+import java.io.IOException;
+import java.sql.Connection;
+import java.sql.ResultSet;
+import java.sql.SQLException;
+import java.sql.Statement;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Map;
+
+import org.apache.hadoop.hive.metastore.tools.HiveSchemaHelper;
+import org.apache.hadoop.hive.metastore.tools.HiveSchemaHelper.MetaStoreConnectionInfo;
+
+import com.google.common.collect.ImmutableMap;
+import org.apache.hadoop.hive.metastore.utils.MetastoreVersionInfo;
+
+
+public class MetaStoreSchemaInfo implements IMetaStoreSchemaInfo {
+ protected static final String UPGRADE_FILE_PREFIX = "upgrade-";
+ private static final String INIT_FILE_PREFIX = "hive-schema-";
+ private static final String VERSION_UPGRADE_LIST = "upgrade.order";
+ private static final String PRE_UPGRADE_PREFIX = "pre-";
+ protected final String dbType;
+ private String[] hiveSchemaVersions;
+ private final String hiveHome;
+
+ // Some version upgrades often don't change schema. So they are equivalent to
+ // a version
+ // that has a corresponding schema. eg "0.13.1" is equivalent to "0.13.0"
+ private static final Map<String, String> EQUIVALENT_VERSIONS =
+ ImmutableMap.of("0.13.1", "0.13.0",
+ "1.0.0", "0.14.0",
+ "1.0.1", "1.0.0",
+ "1.1.1", "1.1.0",
+ "1.2.1", "1.2.0"
+ );
+
+ public MetaStoreSchemaInfo(String hiveHome, String dbType) throws HiveMetaException {
+ this.hiveHome = hiveHome;
+ this.dbType = dbType;
+ }
+
+ private void loadAllUpgradeScripts(String dbType) throws HiveMetaException {
+ // load upgrade order for the given dbType
+ List<String> upgradeOrderList = new ArrayList<>();
+ String upgradeListFile = getMetaStoreScriptDir() + File.separator +
+ VERSION_UPGRADE_LIST + "." + dbType;
+ try (FileReader fr = new FileReader(upgradeListFile);
+ BufferedReader bfReader = new BufferedReader(fr)) {
+ String currSchemaVersion;
+ while ((currSchemaVersion = bfReader.readLine()) != null) {
+ upgradeOrderList.add(currSchemaVersion.trim());
+ }
+ } catch (FileNotFoundException e) {
+ throw new HiveMetaException("File " + upgradeListFile + "not found ", e);
+ } catch (IOException e) {
+ throw new HiveMetaException("Error reading " + upgradeListFile, e);
+ }
+ hiveSchemaVersions = upgradeOrderList.toArray(new String[0]);
+ }
+
+ /***
+ * Get the list of sql scripts required to upgrade from the give version to current
+ * @param fromVersion
+ * @return
+ * @throws HiveMetaException
+ */
+ @Override
+ public List<String> getUpgradeScripts(String fromVersion)
+ throws HiveMetaException {
+ List <String> upgradeScriptList = new ArrayList<>();
+
+ // check if we are already at current schema level
+ if (getHiveSchemaVersion().equals(fromVersion)) {
+ return upgradeScriptList;
+ }
+ loadAllUpgradeScripts(dbType);
+ // Find the list of scripts to execute for this upgrade
+ int firstScript = hiveSchemaVersions.length;
+ for (int i=0; i < hiveSchemaVersions.length; i++) {
+ if (hiveSchemaVersions[i].startsWith(fromVersion)) {
+ firstScript = i;
+ }
+ }
+ if (firstScript == hiveSchemaVersions.length) {
+ throw new HiveMetaException("Unknown version specified for upgrade " +
+ fromVersion + " Metastore schema may be too old or newer");
+ }
+
+ for (int i=firstScript; i < hiveSchemaVersions.length; i++) {
+ String scriptFile = generateUpgradeFileName(hiveSchemaVersions[i]);
+ upgradeScriptList.add(scriptFile);
+ }
+ return upgradeScriptList;
+ }
+
+ /***
+ * Get the name of the script to initialize the schema for given version
+ * @param toVersion Target version. If it's null, then the current server version is used
+ * @return
+ * @throws HiveMetaException
+ */
+ @Override
+ public String generateInitFileName(String toVersion) throws HiveMetaException {
+ if (toVersion == null) {
+ toVersion = getHiveSchemaVersion();
+ }
+ String initScriptName = INIT_FILE_PREFIX + toVersion + "." +
+ dbType + SQL_FILE_EXTENSION;
+ // check if the file exists
+ if (!(new File(getMetaStoreScriptDir() + File.separatorChar +
+ initScriptName).exists())) {
+ throw new HiveMetaException("Unknown version specified for initialization: " + toVersion);
+ }
+ return initScriptName;
+ }
+
+ /**
+ * Find the directory of metastore scripts
+ * @return
+ */
+ @Override
+ public String getMetaStoreScriptDir() {
+ return hiveHome + File.separatorChar +
+ "scripts" + File.separatorChar + "metastore" +
+ File.separatorChar + "upgrade" + File.separatorChar + dbType;
+ }
+
+ // format the upgrade script name eg upgrade-x-y-dbType.sql
+ private String generateUpgradeFileName(String fileVersion) {
+ return UPGRADE_FILE_PREFIX + fileVersion + "." + dbType + SQL_FILE_EXTENSION;
+ }
+
+ @Override
+ public String getPreUpgradeScriptName(int index, String upgradeScriptName) {
+ return PRE_UPGRADE_PREFIX + index + "-" + upgradeScriptName;
+ }
+
+ @Override
+ public String getHiveSchemaVersion() {
+ String hiveVersion = MetastoreVersionInfo.getShortVersion();
+ return getEquivalentVersion(hiveVersion);
+ }
+
+ private static String getEquivalentVersion(String hiveVersion) {
+ // if there is an equivalent version, return that, else return this version
+ String equivalentVersion = EQUIVALENT_VERSIONS.get(hiveVersion);
+ if (equivalentVersion != null) {
+ return equivalentVersion;
+ } else {
+ return hiveVersion;
+ }
+ }
+
+ @Override
+ public boolean isVersionCompatible(String hiveVersion, String dbVersion) {
+ hiveVersion = getEquivalentVersion(hiveVersion);
+ dbVersion = getEquivalentVersion(dbVersion);
+ if (hiveVersion.equals(dbVersion)) {
+ return true;
+ }
+ String[] hiveVerParts = hiveVersion.split("\\.");
+ String[] dbVerParts = dbVersion.split("\\.");
+ if (hiveVerParts.length != 3 || dbVerParts.length != 3) {
+ // these are non standard version numbers. can't perform the
+ // comparison on these, so assume that they are incompatible
+ return false;
+ }
+
+ for (int i = 0; i < dbVerParts.length; i++) {
+ int dbVerPart = Integer.parseInt(dbVerParts[i]);
+ int hiveVerPart = Integer.parseInt(hiveVerParts[i]);
+ if (dbVerPart > hiveVerPart) {
+ return true;
+ } else if (dbVerPart < hiveVerPart) {
+ return false;
+ } else {
+ continue; // compare next part
+ }
+ }
+
+ return true;
+ }
+
+ @Override
+ public String getMetaStoreSchemaVersion(MetaStoreConnectionInfo connectionInfo)
+ throws HiveMetaException {
+ String versionQuery;
+ boolean needsQuotedIdentifier =
+ HiveSchemaHelper.getDbCommandParser(connectionInfo.getDbType()).needsQuotedIdentifier();
+ if (needsQuotedIdentifier) {
+ versionQuery = "select t.\"SCHEMA_VERSION\" from \"VERSION\" t";
+ } else {
+ versionQuery = "select t.SCHEMA_VERSION from VERSION t";
+ }
+ try (Connection metastoreDbConnection =
+ HiveSchemaHelper.getConnectionToMetastore(connectionInfo); Statement stmt =
+ metastoreDbConnection.createStatement()) {
+ ResultSet res = stmt.executeQuery(versionQuery);
+ if (!res.next()) {
+ throw new HiveMetaException("Could not find version info in metastore VERSION table.");
+ }
+ String currentSchemaVersion = res.getString(1);
+ if (res.next()) {
+ throw new HiveMetaException("Multiple versions were found in metastore.");
+ }
+ return currentSchemaVersion;
+ } catch (SQLException e) {
+ throw new HiveMetaException("Failed to get schema version, Cause:" + e.getMessage());
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/hive/blob/b0b6db73/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/MetaStoreSchemaInfoFactory.java
----------------------------------------------------------------------
diff --git a/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/MetaStoreSchemaInfoFactory.java b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/MetaStoreSchemaInfoFactory.java
new file mode 100644
index 0000000..d03b7ae
--- /dev/null
+++ b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/MetaStoreSchemaInfoFactory.java
@@ -0,0 +1,64 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hive.metastore;
+
+import java.lang.reflect.Constructor;
+import java.lang.reflect.InvocationTargetException;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hive.metastore.conf.MetastoreConf;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * Factory class implementation to create instances of IMetaStoreSchemaInfo
+ * based on the provided configuration
+ */
+public class MetaStoreSchemaInfoFactory {
+ public static final Logger LOG = LoggerFactory.getLogger(MetaStoreSchemaInfoFactory.class);
+
+ public static IMetaStoreSchemaInfo get(Configuration conf) {
+ String hiveHome = System.getenv("HIVE_HOME");
+ if (hiveHome == null) {
+ LOG.debug("HIVE_HOME is not set. Using current directory instead");
+ hiveHome = ".";
+ }
+ return get(conf, hiveHome, null);
+ }
+
+ public static IMetaStoreSchemaInfo get(Configuration conf, String hiveHome, String dbType) {
+ String className = MetastoreConf.getVar(conf, MetastoreConf.ConfVars.SCHEMA_INFO_CLASS);
+ Class<?> clasz;
+ try {
+ clasz = conf.getClassByName(className);
+ } catch (ClassNotFoundException e) {
+ LOG.error("Unable to load class " + className, e);
+ throw new IllegalArgumentException(e);
+ }
+ Constructor<?> constructor;
+ try {
+ constructor = clasz.getConstructor(String.class, String.class);
+ constructor.setAccessible(true);
+ return (IMetaStoreSchemaInfo) constructor.newInstance(hiveHome, dbType);
+ } catch (NoSuchMethodException | InstantiationException | IllegalAccessException
+ | IllegalArgumentException | InvocationTargetException e) {
+ LOG.error("Unable to create instance of class " + className, e);
+ throw new IllegalArgumentException(e);
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/hive/blob/b0b6db73/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/MetaStoreThread.java
----------------------------------------------------------------------
diff --git a/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/MetaStoreThread.java b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/MetaStoreThread.java
new file mode 100644
index 0000000..10f7732
--- /dev/null
+++ b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/MetaStoreThread.java
@@ -0,0 +1,57 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hive.metastore;
+
+import org.apache.hadoop.conf.Configurable;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hive.metastore.api.MetaException;
+
+import java.util.concurrent.atomic.AtomicBoolean;
+
+/**
+ * A thread that runs in the metastore, separate from the threads in the thrift service.
+ */
+public interface MetaStoreThread extends Configurable {
+
+ /**
+ * Set the id for this thread.
+ * @param threadId id of the thread
+ */
+ void setThreadId(int threadId);
+
+ /**
+ * Initialize the thread. This must not be called until after
+ * {@link #setConf(Configuration)} and {@link #setThreadId(int)}
+ * have been called.
+ * @param stop a flag to watch for when to stop. If this value is set to true,
+ * the thread will terminate the next time through its main loop.
+ * @param looped a flag that is set to true everytime a thread goes through it's main loop.
+ * This is purely for testing so that tests can assure themselves that the thread
+ * has run through it's loop once. The test can set this value to false. The
+ * thread should then assure that the loop has been gone completely through at
+ * least once.
+ */
+ void init(AtomicBoolean stop, AtomicBoolean looped) throws MetaException;
+
+ /**
+ * Run the thread in the background. This must not be called until
+ * {@link MetaStoreThread#init(java.util.concurrent.atomic.AtomicBoolean,java.util.concurrent.atomic.AtomicBoolean)} has
+ * been called.
+ */
+ void start();
+}
http://git-wip-us.apache.org/repos/asf/hive/blob/b0b6db73/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/PartitionDropOptions.java
----------------------------------------------------------------------
diff --git a/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/PartitionDropOptions.java b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/PartitionDropOptions.java
new file mode 100644
index 0000000..40018c9
--- /dev/null
+++ b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/PartitionDropOptions.java
@@ -0,0 +1,54 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hive.metastore;
+
+/**
+ * Class to generalize the switches for dropPartitions().
+ */
+public class PartitionDropOptions {
+
+ public boolean deleteData = true;
+ public boolean ifExists = false;
+ public boolean returnResults = true;
+ public boolean purgeData = false;
+
+ public static PartitionDropOptions instance() { return new PartitionDropOptions(); }
+
+ public PartitionDropOptions deleteData(boolean deleteData) {
+ this.deleteData = deleteData;
+ return this;
+ }
+
+ public PartitionDropOptions ifExists(boolean ifExists) {
+ this.ifExists = ifExists;
+ return this;
+ }
+
+ public PartitionDropOptions returnResults(boolean returnResults) {
+ this.returnResults = returnResults;
+ return this;
+ }
+
+ public PartitionDropOptions purgeData(boolean purgeData) {
+ this.purgeData = purgeData;
+ return this;
+ }
+
+} // class PartitionDropSwitches;
+
http://git-wip-us.apache.org/repos/asf/hive/blob/b0b6db73/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/PartitionExpressionProxy.java
----------------------------------------------------------------------
diff --git a/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/PartitionExpressionProxy.java b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/PartitionExpressionProxy.java
new file mode 100644
index 0000000..af0a6bd
--- /dev/null
+++ b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/PartitionExpressionProxy.java
@@ -0,0 +1,73 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hive.metastore;
+
+import java.util.List;
+
+import org.apache.hadoop.hive.metastore.api.FieldSchema;
+import org.apache.hadoop.hive.metastore.api.FileMetadataExprType;
+import org.apache.hadoop.hive.metastore.api.MetaException;
+import org.apache.hadoop.hive.ql.io.sarg.SearchArgument;
+
+/**
+ * The proxy interface that metastore uses for variety of QL operations (metastore can't depend
+ * on QL because QL depends on metastore; creating metastore-client module would be a proper way
+ * to solve this problem).
+ */
+public interface PartitionExpressionProxy {
+
+ /**
+ * Converts serialized Hive expression into filter in the format suitable for Filter.g.
+ * @param expr Serialized expression.
+ * @return The filter string.
+ */
+ public String convertExprToFilter(byte[] expr) throws MetaException;
+
+ /**
+ * Filters the partition names via serialized Hive expression.
+ * @param partColumns Partition columns in the underlying table.
+ * @param expr Serialized expression.
+ * @param defaultPartitionName Default partition name from job or server configuration.
+ * @param partitionNames Partition names; the list is modified in place.
+ * @return Whether there were any unknown partitions preserved in the name list.
+ */
+ boolean filterPartitionsByExpr(List<FieldSchema> partColumns,
+ byte[] expr, String defaultPartitionName, List<String> partitionNames) throws MetaException;
+
+ /**
+ * Determines the file metadata type from input format of the source table or partition.
+ * @param inputFormat Input format name.
+ * @return The file metadata type.
+ */
+ FileMetadataExprType getMetadataType(String inputFormat);
+
+ /**
+ * Gets a separate proxy that can be used to call file-format-specific methods.
+ * @param type The file metadata type.
+ * @return The proxy.
+ */
+ FileFormatProxy getFileFormatProxy(FileMetadataExprType type);
+
+ /**
+ * Creates SARG from serialized representation.
+ * @param expr SARG, serialized as Kryo.
+ * @return SARG.
+ */
+ SearchArgument createSarg(byte[] expr);
+}
http://git-wip-us.apache.org/repos/asf/hive/blob/b0b6db73/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/TServerSocketKeepAlive.java
----------------------------------------------------------------------
diff --git a/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/TServerSocketKeepAlive.java b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/TServerSocketKeepAlive.java
new file mode 100644
index 0000000..95bd76e
--- /dev/null
+++ b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/TServerSocketKeepAlive.java
@@ -0,0 +1,47 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hive.metastore;
+
+import java.net.SocketException;
+
+import org.apache.thrift.transport.TServerSocket;
+import org.apache.thrift.transport.TSocket;
+import org.apache.thrift.transport.TTransportException;
+
+/**
+ * TServerSocketKeepAlive - like TServerSocket, but will enable keepalive for
+ * accepted sockets.
+ *
+ */
+public class TServerSocketKeepAlive extends TServerSocket {
+ public TServerSocketKeepAlive(TServerSocket serverSocket) throws TTransportException {
+ super(serverSocket.getServerSocket());
+ }
+
+ @Override
+ protected TSocket acceptImpl() throws TTransportException {
+ TSocket ts = super.acceptImpl();
+ try {
+ ts.getSocket().setKeepAlive(true);
+ } catch (SocketException e) {
+ throw new TTransportException(e);
+ }
+ return ts;
+ }
+}
http://git-wip-us.apache.org/repos/asf/hive/blob/b0b6db73/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/TableType.java
----------------------------------------------------------------------
diff --git a/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/TableType.java b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/TableType.java
new file mode 100644
index 0000000..e9e16d7
--- /dev/null
+++ b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/TableType.java
@@ -0,0 +1,26 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hive.metastore;
+
+/**
+ * Typesafe enum for types of tables described by the metastore.
+ */
+public enum TableType {
+ MANAGED_TABLE, EXTERNAL_TABLE, VIRTUAL_VIEW, INDEX_TABLE, MATERIALIZED_VIEW
+}
http://git-wip-us.apache.org/repos/asf/hive/blob/b0b6db73/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/annotation/MetastoreVersionAnnotation.java
----------------------------------------------------------------------
diff --git a/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/annotation/MetastoreVersionAnnotation.java b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/annotation/MetastoreVersionAnnotation.java
new file mode 100644
index 0000000..0542546
--- /dev/null
+++ b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/annotation/MetastoreVersionAnnotation.java
@@ -0,0 +1,85 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hive.metastore.annotation;
+
+import org.apache.hadoop.classification.InterfaceStability;
+
+import java.lang.annotation.ElementType;
+import java.lang.annotation.Retention;
+import java.lang.annotation.RetentionPolicy;
+import java.lang.annotation.Target;
+
+
+/**
+ * HiveVersionAnnotation.
+ *
+ */
+@Retention(RetentionPolicy.RUNTIME)
+@Target(ElementType.PACKAGE)
+@InterfaceStability.Unstable
+public @interface MetastoreVersionAnnotation {
+
+ /**
+ * Get the Hive version
+ * @return the version string "0.6.3-dev"
+ */
+ String version();
+
+ /**
+ * Get the Hive short version containing major/minor/change version numbers
+ * @return the short version string "0.6.3"
+ */
+ String shortVersion();
+
+ /**
+ * Get the username that compiled Hive.
+ */
+ String user();
+
+ /**
+ * Get the date when Hive was compiled.
+ * @return the date in unix 'date' format
+ */
+ String date();
+
+ /**
+ * Get the url for the git repository.
+ */
+ String url();
+
+ /**
+ * Get the git revision.
+ * @return the revision number as a string (eg. "451451")
+ */
+ String revision();
+
+ /**
+ * Get the branch from which this was compiled.
+ * @return The branch name, e.g. "trunk" or "branches/branch-0.20"
+ */
+ String branch();
+
+ /**
+ * Get a checksum of the source files from which
+ * Hive was compiled.
+ * @return a string that uniquely identifies the source
+ **/
+ String srcChecksum();
+
+}
http://git-wip-us.apache.org/repos/asf/hive/blob/b0b6db73/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/annotation/NoReconnect.java
----------------------------------------------------------------------
diff --git a/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/annotation/NoReconnect.java b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/annotation/NoReconnect.java
new file mode 100644
index 0000000..edf0831
--- /dev/null
+++ b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/annotation/NoReconnect.java
@@ -0,0 +1,29 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.hadoop.hive.metastore.annotation;
+
+import java.lang.annotation.ElementType;
+import java.lang.annotation.Retention;
+import java.lang.annotation.RetentionPolicy;
+import java.lang.annotation.Target;
+
+@Target({ElementType.METHOD})
+@Retention(RetentionPolicy.RUNTIME)
+public @interface NoReconnect {
+}
http://git-wip-us.apache.org/repos/asf/hive/blob/b0b6db73/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/hooks/JDOConnectionURLHook.java
----------------------------------------------------------------------
diff --git a/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/hooks/JDOConnectionURLHook.java b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/hooks/JDOConnectionURLHook.java
new file mode 100644
index 0000000..65084bd
--- /dev/null
+++ b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/hooks/JDOConnectionURLHook.java
@@ -0,0 +1,53 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hive.metastore.hooks;
+
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
+import org.apache.hadoop.conf.Configuration;
+
+/**
+ * JDOConnectURLHook is used to get the URL that JDO uses to connect to the
+ * database that stores the metastore data. Classes implementing this must be
+ * thread-safe (for Thrift server).
+ */
+@InterfaceAudience.Public
+@InterfaceStability.Stable
+public interface JDOConnectionURLHook {
+
+ /**
+ * Gets the connection URL to supply to JDO. In addition to initialization,
+ * this method will be called after a connection failure for each reconnect
+ * attempt.
+ *
+ * @param conf The configuration used to initialize this instance of the HMS
+ * @return the connection URL
+ * @throws Exception
+ */
+ public String getJdoConnectionUrl(Configuration conf)
+ throws Exception;
+
+ /**
+ * Alerts this that the connection URL was bad. Can be used to collect stats,
+ * etc.
+ *
+ * @param url
+ */
+ public void notifyBadConnectionUrl(String url);
+}
[7/7] hive git commit: HIVE-17488 Move first set of classes to
standalone metastore. This closes #244. (Alan Gates,
reviewed by Owen O'Malley)
Posted by ga...@apache.org.
HIVE-17488 Move first set of classes to standalone metastore. This closes #244. (Alan Gates, reviewed by Owen O'Malley)
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/b0b6db73
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/b0b6db73
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/b0b6db73
Branch: refs/heads/master
Commit: b0b6db7307767710d58bdf59234959ffc17445b1
Parents: 1c84e0c
Author: Alan Gates <ga...@hortonworks.com>
Authored: Thu Sep 14 15:08:53 2017 -0700
Committer: Alan Gates <ga...@hortonworks.com>
Committed: Thu Sep 14 15:08:53 2017 -0700
----------------------------------------------------------------------
.gitignore | 1 +
.../org/apache/hive/beeline/HiveSchemaTool.java | 5 +-
.../apache/hive/beeline/TestHiveSchemaTool.java | 9 +-
.../common/classification/RetrySemantics.java | 54 -
.../hive/hcatalog/streaming/TestStreaming.java | 2 +-
.../hadoop/hive/metastore/TestFilterHooks.java | 3 +-
.../hive/ql/txn/compactor/TestCompactor.java | 40 +-
metastore/pom.xml | 39 -
.../apache/hadoop/hive/metastore/Metastore.java | 1331 ------------------
.../hive/metastore/AggregateStatsCache.java | 573 --------
.../hadoop/hive/metastore/DatabaseProduct.java | 75 -
.../hive/metastore/DefaultHiveMetaHook.java | 51 -
.../DefaultMetaStoreFilterHookImpl.java | 102 --
.../hadoop/hive/metastore/FileFormatProxy.java | 64 -
.../hive/metastore/HiveMetaException.java | 42 -
.../hadoop/hive/metastore/HiveMetaHook.java | 96 --
.../hive/metastore/HiveMetaHookLoader.java | 39 -
.../hadoop/hive/metastore/HiveMetaStore.java | 2 +-
.../hive/metastore/HiveMetaStoreClient.java | 3 +-
.../hive/metastore/HiveMetaStoreFsImpl.java | 52 -
.../hive/metastore/IExtrapolatePartStatus.java | 85 --
.../hadoop/hive/metastore/IHMSHandler.java | 28 -
.../hive/metastore/IMetaStoreSchemaInfo.java | 107 --
.../metastore/LinearExtrapolatePartStatus.java | 106 --
.../hive/metastore/LockComponentBuilder.java | 121 --
.../hive/metastore/LockRequestBuilder.java | 168 ---
.../hive/metastore/MetaStoreDirectSql.java | 4 +-
.../metastore/MetaStoreEndFunctionContext.java | 59 -
.../metastore/MetaStoreEndFunctionListener.java | 58 -
.../hadoop/hive/metastore/MetaStoreFS.java | 43 -
.../hive/metastore/MetaStoreFilterHook.java | 132 --
.../hadoop/hive/metastore/MetaStoreInit.java | 112 --
.../hive/metastore/MetaStoreSchemaInfo.java | 234 ---
.../metastore/MetaStoreSchemaInfoFactory.java | 65 -
.../hadoop/hive/metastore/MetaStoreThread.java | 62 -
.../hive/metastore/PartitionDropOptions.java | 54 -
.../metastore/PartitionExpressionProxy.java | 73 -
.../hive/metastore/TServerSocketKeepAlive.java | 47 -
.../apache/hadoop/hive/metastore/TableType.java | 26 -
.../hive/metastore/annotation/NoReconnect.java | 29 -
.../metastore/hooks/JDOConnectionURLHook.java | 53 -
.../spec/CompositePartitionSpecProxy.java | 228 ---
.../spec/PartitionListComposingSpecProxy.java | 171 ---
.../partition/spec/PartitionSpecProxy.java | 199 ---
.../spec/PartitionSpecWithSharedSDProxy.java | 172 ---
.../hive/metastore/tools/HiveSchemaHelper.java | 640 ---------
.../hadoop/hive/metastore/metastore.proto | 29 -
.../hive/metastore/TestAggregateStatsCache.java | 266 ----
.../hive/metastore/TestLockRequestBuilder.java | 584 --------
.../metastore/TestMetaStoreSchemaFactory.java | 67 -
.../hive/metastore/TestMetaStoreSchemaInfo.java | 52 -
.../AuthorizationMetaStoreFilterHook.java | 4 +-
.../hive/ql/txn/compactor/CompactorThread.java | 23 +-
.../apache/hadoop/hive/ql/TestTxnCommands2.java | 18 +-
.../hive/ql/txn/compactor/CompactorTest.java | 2 +-
standalone-metastore/pom.xml | 50 +
.../common/classification/RetrySemantics.java | 57 +
.../hive/metastore/AggregateStatsCache.java | 567 ++++++++
.../hadoop/hive/metastore/DatabaseProduct.java | 75 +
.../hive/metastore/DefaultHiveMetaHook.java | 51 +
.../DefaultMetaStoreFilterHookImpl.java | 102 ++
.../hadoop/hive/metastore/FileFormatProxy.java | 64 +
.../hive/metastore/HiveMetaException.java | 42 +
.../hadoop/hive/metastore/HiveMetaHook.java | 95 ++
.../hive/metastore/HiveMetaHookLoader.java | 39 +
.../hive/metastore/HiveMetaStoreFsImpl.java | 53 +
.../hive/metastore/IExtrapolatePartStatus.java | 85 ++
.../hadoop/hive/metastore/IHMSHandler.java | 28 +
.../hive/metastore/IMetaStoreSchemaInfo.java | 108 ++
.../metastore/LinearExtrapolatePartStatus.java | 106 ++
.../hive/metastore/LockComponentBuilder.java | 121 ++
.../hive/metastore/LockRequestBuilder.java | 168 +++
.../metastore/MetaStoreEndFunctionContext.java | 59 +
.../metastore/MetaStoreEndFunctionListener.java | 58 +
.../hadoop/hive/metastore/MetaStoreFS.java | 43 +
.../hive/metastore/MetaStoreFilterHook.java | 132 ++
.../hadoop/hive/metastore/MetaStoreInit.java | 109 ++
.../hive/metastore/MetaStoreSchemaInfo.java | 234 +++
.../metastore/MetaStoreSchemaInfoFactory.java | 64 +
.../hadoop/hive/metastore/MetaStoreThread.java | 57 +
.../hive/metastore/PartitionDropOptions.java | 54 +
.../metastore/PartitionExpressionProxy.java | 73 +
.../hive/metastore/TServerSocketKeepAlive.java | 47 +
.../apache/hadoop/hive/metastore/TableType.java | 26 +
.../annotation/MetastoreVersionAnnotation.java | 85 ++
.../hive/metastore/annotation/NoReconnect.java | 29 +
.../metastore/hooks/JDOConnectionURLHook.java | 53 +
.../spec/CompositePartitionSpecProxy.java | 226 +++
.../spec/PartitionListComposingSpecProxy.java | 171 +++
.../partition/spec/PartitionSpecProxy.java | 199 +++
.../spec/PartitionSpecWithSharedSDProxy.java | 172 +++
.../hive/metastore/tools/HiveSchemaHelper.java | 641 +++++++++
.../hadoop/hive/metastore/utils/FileUtils.java | 66 +
.../hadoop/hive/metastore/utils/JavaUtils.java | 37 +
.../hive/metastore/utils/MetaStoreUtils.java | 41 +
.../metastore/utils/MetastoreVersionInfo.java | 133 ++
.../hadoop/hive/metastore/metastore.proto | 29 +
.../src/main/resources/saveVersion.sh | 91 ++
.../hive/metastore/TestAggregateStatsCache.java | 267 ++++
.../hive/metastore/TestLockRequestBuilder.java | 584 ++++++++
.../metastore/TestMetaStoreSchemaFactory.java | 69 +
.../hive/metastore/TestMetaStoreSchemaInfo.java | 52 +
102 files changed, 5778 insertions(+), 6638 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/b0b6db73/.gitignore
----------------------------------------------------------------------
diff --git a/.gitignore b/.gitignore
index 89169b0..d56f31f 100644
--- a/.gitignore
+++ b/.gitignore
@@ -30,3 +30,4 @@ conf/hive-default.xml.template
itests/hive-blobstore/src/test/resources/blobstore-conf.xml
.DS_Store
patchprocess
+standalone-metastore/src/gen/org/apache/hadoop/hive/metastore/annotation/package-info.java
http://git-wip-us.apache.org/repos/asf/hive/blob/b0b6db73/beeline/src/java/org/apache/hive/beeline/HiveSchemaTool.java
----------------------------------------------------------------------
diff --git a/beeline/src/java/org/apache/hive/beeline/HiveSchemaTool.java b/beeline/src/java/org/apache/hive/beeline/HiveSchemaTool.java
index 4d7373a..84963af 100644
--- a/beeline/src/java/org/apache/hive/beeline/HiveSchemaTool.java
+++ b/beeline/src/java/org/apache/hive/beeline/HiveSchemaTool.java
@@ -38,6 +38,7 @@ import org.apache.hadoop.hive.metastore.IMetaStoreSchemaInfo;
import org.apache.hadoop.hive.metastore.MetaStoreSchemaInfoFactory;
import org.apache.hadoop.hive.metastore.TableType;
import org.apache.hadoop.hive.metastore.api.MetaException;
+import org.apache.hadoop.hive.metastore.conf.MetastoreConf;
import org.apache.hadoop.hive.metastore.tools.HiveSchemaHelper;
import org.apache.hadoop.hive.metastore.tools.HiveSchemaHelper.MetaStoreConnectionInfo;
import org.apache.hadoop.hive.metastore.tools.HiveSchemaHelper.NestedScriptParser;
@@ -1016,8 +1017,8 @@ public class HiveSchemaTool {
private String[] argsWith(String password) throws IOException {
return new String[]
{
- "-u", url == null ? HiveSchemaHelper.getValidConfVar(ConfVars.METASTORECONNECTURLKEY, hiveConf) : url,
- "-d", driver == null ? HiveSchemaHelper.getValidConfVar(ConfVars.METASTORE_CONNECTION_DRIVER, hiveConf) : driver,
+ "-u", url == null ? HiveSchemaHelper.getValidConfVar(MetastoreConf.ConfVars.CONNECTURLKEY, hiveConf) : url,
+ "-d", driver == null ? HiveSchemaHelper.getValidConfVar(MetastoreConf.ConfVars.CONNECTION_DRIVER, hiveConf) : driver,
"-n", userName,
"-p", password,
"-f", sqlScriptFile
http://git-wip-us.apache.org/repos/asf/hive/blob/b0b6db73/beeline/src/test/org/apache/hive/beeline/TestHiveSchemaTool.java
----------------------------------------------------------------------
diff --git a/beeline/src/test/org/apache/hive/beeline/TestHiveSchemaTool.java b/beeline/src/test/org/apache/hive/beeline/TestHiveSchemaTool.java
index d9b8f42..03c9f7c 100644
--- a/beeline/src/test/org/apache/hive/beeline/TestHiveSchemaTool.java
+++ b/beeline/src/test/org/apache/hive/beeline/TestHiveSchemaTool.java
@@ -18,6 +18,7 @@
package org.apache.hive.beeline;
import org.apache.hadoop.hive.conf.HiveConf;
+import org.apache.hadoop.hive.metastore.conf.MetastoreConf;
import org.apache.hadoop.hive.metastore.tools.HiveSchemaHelper;
import org.junit.After;
import org.junit.Before;
@@ -55,10 +56,10 @@ public class TestHiveSchemaTool {
public void setup() throws IOException {
mockStatic(HiveSchemaHelper.class);
when(HiveSchemaHelper
- .getValidConfVar(eq(HiveConf.ConfVars.METASTORECONNECTURLKEY), same(hiveConf)))
+ .getValidConfVar(eq(MetastoreConf.ConfVars.CONNECTURLKEY), same(hiveConf)))
.thenReturn("someURL");
when(HiveSchemaHelper
- .getValidConfVar(eq(HiveConf.ConfVars.METASTORE_CONNECTION_DRIVER), same(hiveConf)))
+ .getValidConfVar(eq(MetastoreConf.ConfVars.CONNECTION_DRIVER), same(hiveConf)))
.thenReturn("someDriver");
File file = new File(scriptFile);
@@ -71,9 +72,9 @@ public class TestHiveSchemaTool {
@After
public void globalAssert() throws IOException {
verifyStatic();
- HiveSchemaHelper.getValidConfVar(eq(HiveConf.ConfVars.METASTORECONNECTURLKEY), same(hiveConf));
+ HiveSchemaHelper.getValidConfVar(eq(MetastoreConf.ConfVars.CONNECTURLKEY), same(hiveConf));
HiveSchemaHelper
- .getValidConfVar(eq(HiveConf.ConfVars.METASTORE_CONNECTION_DRIVER), same(hiveConf));
+ .getValidConfVar(eq(MetastoreConf.ConfVars.CONNECTION_DRIVER), same(hiveConf));
new File(scriptFile).delete();
}
http://git-wip-us.apache.org/repos/asf/hive/blob/b0b6db73/common/src/java/org/apache/hadoop/hive/common/classification/RetrySemantics.java
----------------------------------------------------------------------
diff --git a/common/src/java/org/apache/hadoop/hive/common/classification/RetrySemantics.java b/common/src/java/org/apache/hadoop/hive/common/classification/RetrySemantics.java
deleted file mode 100644
index f1c3946..0000000
--- a/common/src/java/org/apache/hadoop/hive/common/classification/RetrySemantics.java
+++ /dev/null
@@ -1,54 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.hive.common.classification;
-
-
-import java.lang.annotation.ElementType;
-import java.lang.annotation.Retention;
-import java.lang.annotation.RetentionPolicy;
-import java.lang.annotation.Target;
-
-/**
- * These annotations are meant to indicate how to handle retry logic.
- * Initially meant for Metastore API when made across a network, i.e. asynchronously where
- * the response may not reach the caller and thus it cannot know if the operation was actually
- * performed on the server.
- */
-@InterfaceStability.Evolving
-@InterfaceAudience.LimitedPrivate("Hive developer")
-public class RetrySemantics {
- @Retention(RetentionPolicy.RUNTIME)
- @Target(ElementType.METHOD)
- public @interface Idempotent {
- String[] value() default "";
- int maxRetryCount() default Integer.MAX_VALUE;
- int delayMs() default 100;
- }
- @Retention(RetentionPolicy.RUNTIME)
- @Target(ElementType.METHOD)
- public @interface ReadOnly {/*trivially retry-able*/}
- @Retention(RetentionPolicy.RUNTIME)
- @Target(ElementType.METHOD)
- public @interface CannotRetry {}
- @Retention(RetentionPolicy.RUNTIME)
- @Target(ElementType.METHOD)
- public @interface SafeToRetry {
- /*may not be Idempotent but is safe to retry*/
- String[] value() default "";
- }
-}
http://git-wip-us.apache.org/repos/asf/hive/blob/b0b6db73/hcatalog/streaming/src/test/org/apache/hive/hcatalog/streaming/TestStreaming.java
----------------------------------------------------------------------
diff --git a/hcatalog/streaming/src/test/org/apache/hive/hcatalog/streaming/TestStreaming.java b/hcatalog/streaming/src/test/org/apache/hive/hcatalog/streaming/TestStreaming.java
index 49520ef..1e73a4b 100644
--- a/hcatalog/streaming/src/test/org/apache/hive/hcatalog/streaming/TestStreaming.java
+++ b/hcatalog/streaming/src/test/org/apache/hive/hcatalog/streaming/TestStreaming.java
@@ -414,7 +414,7 @@ public class TestStreaming {
AtomicBoolean stop = new AtomicBoolean(true);
Worker t = new Worker();
t.setThreadId((int) t.getId());
- t.setHiveConf(hiveConf);
+ t.setConf(hiveConf);
AtomicBoolean looped = new AtomicBoolean();
t.init(stop, looped);
t.run();
http://git-wip-us.apache.org/repos/asf/hive/blob/b0b6db73/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestFilterHooks.java
----------------------------------------------------------------------
diff --git a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestFilterHooks.java b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestFilterHooks.java
index fc85d86..69efc98 100644
--- a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestFilterHooks.java
+++ b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestFilterHooks.java
@@ -24,6 +24,7 @@ import static org.junit.Assert.fail;
import java.util.ArrayList;
import java.util.List;
+import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hive.UtilsForTest;
import org.apache.hadoop.hive.cli.CliSessionState;
import org.apache.hadoop.hive.conf.HiveConf;
@@ -49,7 +50,7 @@ public class TestFilterHooks {
public static class DummyMetaStoreFilterHookImpl extends DefaultMetaStoreFilterHookImpl {
public static boolean blockResults = false;
- public DummyMetaStoreFilterHookImpl(HiveConf conf) {
+ public DummyMetaStoreFilterHookImpl(Configuration conf) {
super(conf);
}
http://git-wip-us.apache.org/repos/asf/hive/blob/b0b6db73/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/txn/compactor/TestCompactor.java
----------------------------------------------------------------------
diff --git a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/txn/compactor/TestCompactor.java b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/txn/compactor/TestCompactor.java
index 44b244b..4bfc5be 100644
--- a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/txn/compactor/TestCompactor.java
+++ b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/txn/compactor/TestCompactor.java
@@ -200,7 +200,7 @@ public class TestCompactor {
Initiator initiator = new Initiator();
initiator.setThreadId((int)initiator.getId());
conf.setIntVar(HiveConf.ConfVars.HIVE_COMPACTOR_DELTA_NUM_THRESHOLD, 0);
- initiator.setHiveConf(conf);
+ initiator.setConf(conf);
AtomicBoolean stop = new AtomicBoolean();
stop.set(true);
initiator.init(stop, new AtomicBoolean());
@@ -303,7 +303,7 @@ public class TestCompactor {
initiator.setThreadId((int)initiator.getId());
// Set to 1 so insert doesn't set it off but update does
conf.setIntVar(HiveConf.ConfVars.HIVE_COMPACTOR_DELTA_NUM_THRESHOLD, 1);
- initiator.setHiveConf(conf);
+ initiator.setConf(conf);
AtomicBoolean stop = new AtomicBoolean();
stop.set(true);
initiator.init(stop, new AtomicBoolean());
@@ -455,7 +455,7 @@ public class TestCompactor {
txnHandler.compact(rqst);
Worker t = new Worker();
t.setThreadId((int) t.getId());
- t.setHiveConf(conf);
+ t.setConf(conf);
AtomicBoolean stop = new AtomicBoolean();
AtomicBoolean looped = new AtomicBoolean();
stop.set(true);
@@ -511,7 +511,7 @@ public class TestCompactor {
Initiator initiator = new Initiator();
initiator.setThreadId((int)initiator.getId());
conf.setIntVar(HiveConf.ConfVars.HIVE_COMPACTOR_DELTA_NUM_THRESHOLD, 0);
- initiator.setHiveConf(conf);
+ initiator.setConf(conf);
AtomicBoolean stop = new AtomicBoolean();
stop.set(true);
initiator.init(stop, new AtomicBoolean());
@@ -551,7 +551,7 @@ public class TestCompactor {
initiator.setThreadId((int)initiator.getId());
// Set to 1 so insert doesn't set it off but update does
conf.setIntVar(HiveConf.ConfVars.HIVE_COMPACTOR_DELTA_NUM_THRESHOLD, 1);
- initiator.setHiveConf(conf);
+ initiator.setConf(conf);
AtomicBoolean stop = new AtomicBoolean();
stop.set(true);
initiator.init(stop, new AtomicBoolean());
@@ -593,7 +593,7 @@ public class TestCompactor {
initiator.setThreadId((int)initiator.getId());
// Set to 2 so insert and update don't set it off but delete does
conf.setIntVar(HiveConf.ConfVars.HIVE_COMPACTOR_DELTA_NUM_THRESHOLD, 2);
- initiator.setHiveConf(conf);
+ initiator.setConf(conf);
AtomicBoolean stop = new AtomicBoolean();
stop.set(true);
initiator.init(stop, new AtomicBoolean());
@@ -643,7 +643,7 @@ public class TestCompactor {
txnHandler.compact(new CompactionRequest(dbName, tblName, CompactionType.MINOR));
Worker t = new Worker();
t.setThreadId((int) t.getId());
- t.setHiveConf(conf);
+ t.setConf(conf);
AtomicBoolean stop = new AtomicBoolean(true);
AtomicBoolean looped = new AtomicBoolean();
t.init(stop, looped);
@@ -705,7 +705,7 @@ public class TestCompactor {
txnHandler.compact(new CompactionRequest(dbName, tblName, CompactionType.MAJOR));
Worker t = new Worker();
t.setThreadId((int) t.getId());
- t.setHiveConf(conf);
+ t.setConf(conf);
AtomicBoolean stop = new AtomicBoolean(true);
AtomicBoolean looped = new AtomicBoolean();
t.init(stop, looped);
@@ -762,7 +762,7 @@ public class TestCompactor {
txnHandler.compact(new CompactionRequest(dbName, tblName, CompactionType.MINOR));
Worker t = new Worker();
t.setThreadId((int) t.getId());
- t.setHiveConf(conf);
+ t.setConf(conf);
AtomicBoolean stop = new AtomicBoolean(true);
AtomicBoolean looped = new AtomicBoolean();
t.init(stop, looped);
@@ -828,7 +828,7 @@ public class TestCompactor {
txnHandler.compact(new CompactionRequest(dbName, tblName, CompactionType.MAJOR));
Worker t = new Worker();
t.setThreadId((int) t.getId());
- t.setHiveConf(conf);
+ t.setConf(conf);
AtomicBoolean stop = new AtomicBoolean(true);
AtomicBoolean looped = new AtomicBoolean();
t.init(stop, looped);
@@ -886,7 +886,7 @@ public class TestCompactor {
txnHandler.compact(new CompactionRequest(dbName, tblName, CompactionType.MAJOR));
Worker t = new Worker();
t.setThreadId((int) t.getId());
- t.setHiveConf(conf);
+ t.setConf(conf);
AtomicBoolean stop = new AtomicBoolean(true);
AtomicBoolean looped = new AtomicBoolean();
t.init(stop, looped);
@@ -939,7 +939,7 @@ public class TestCompactor {
txnHandler.compact(new CompactionRequest(dbName, tblName, CompactionType.MINOR));
Worker t = new Worker();
t.setThreadId((int) t.getId());
- t.setHiveConf(conf);
+ t.setConf(conf);
AtomicBoolean stop = new AtomicBoolean(true);
AtomicBoolean looped = new AtomicBoolean();
t.init(stop, looped);
@@ -1016,7 +1016,7 @@ public class TestCompactor {
txnHandler.compact(new CompactionRequest(dbName, tblName, CompactionType.MINOR));
Worker t = new Worker();
t.setThreadId((int) t.getId());
- t.setHiveConf(conf);
+ t.setConf(conf);
AtomicBoolean stop = new AtomicBoolean(true);
AtomicBoolean looped = new AtomicBoolean();
t.init(stop, looped);
@@ -1095,7 +1095,7 @@ public class TestCompactor {
txnHandler.compact(new CompactionRequest(dbName, tblName, CompactionType.MINOR));
Worker t = new Worker();
t.setThreadId((int) t.getId());
- t.setHiveConf(conf);
+ t.setConf(conf);
AtomicBoolean stop = new AtomicBoolean(true);
AtomicBoolean looped = new AtomicBoolean();
t.init(stop, looped);
@@ -1203,7 +1203,7 @@ public class TestCompactor {
AtomicBoolean stop = new AtomicBoolean(true);
Worker t = new Worker();
t.setThreadId((int) t.getId());
- t.setHiveConf(conf);
+ t.setConf(conf);
AtomicBoolean looped = new AtomicBoolean();
t.init(stop, looped);
t.run();
@@ -1213,7 +1213,7 @@ public class TestCompactor {
stop = new AtomicBoolean(true);
t = new Worker();
t.setThreadId((int) t.getId());
- t.setHiveConf(conf);
+ t.setConf(conf);
looped = new AtomicBoolean();
t.init(stop, looped);
t.run();
@@ -1265,7 +1265,7 @@ public class TestCompactor {
stop = new AtomicBoolean(true);
t = new Worker();
t.setThreadId((int) t.getId());
- t.setHiveConf(conf);
+ t.setConf(conf);
looped = new AtomicBoolean();
t.init(stop, looped);
t.run();
@@ -1416,7 +1416,7 @@ public class TestCompactor {
AtomicBoolean stop = new AtomicBoolean(true);
Initiator t = new Initiator();
t.setThreadId((int) t.getId());
- t.setHiveConf(hiveConf);
+ t.setConf(hiveConf);
AtomicBoolean looped = new AtomicBoolean();
t.init(stop, looped);
t.run();
@@ -1426,7 +1426,7 @@ public class TestCompactor {
AtomicBoolean stop = new AtomicBoolean(true);
Worker t = new Worker();
t.setThreadId((int) t.getId());
- t.setHiveConf(hiveConf);
+ t.setConf(hiveConf);
AtomicBoolean looped = new AtomicBoolean();
t.init(stop, looped);
t.run();
@@ -1436,7 +1436,7 @@ public class TestCompactor {
AtomicBoolean stop = new AtomicBoolean(true);
Cleaner t = new Cleaner();
t.setThreadId((int) t.getId());
- t.setHiveConf(hiveConf);
+ t.setConf(hiveConf);
AtomicBoolean looped = new AtomicBoolean();
t.init(stop, looped);
t.run();
http://git-wip-us.apache.org/repos/asf/hive/blob/b0b6db73/metastore/pom.xml
----------------------------------------------------------------------
diff --git a/metastore/pom.xml b/metastore/pom.xml
index 62bddd1..8c35649 100644
--- a/metastore/pom.xml
+++ b/metastore/pom.xml
@@ -61,11 +61,6 @@
<version>${guava.version}</version>
</dependency>
<dependency>
- <groupId>com.google.protobuf</groupId>
- <artifactId>protobuf-java</artifactId>
- <version>${protobuf.version}</version>
- </dependency>
- <dependency>
<groupId>org.apache.hbase</groupId>
<artifactId>hbase-client</artifactId>
<version>${hbase.version}</version>
@@ -265,39 +260,6 @@
</dependencies>
<profiles>
- <profile>
- <id>protobuf</id>
- <build>
- <plugins>
- <plugin>
- <groupId>org.apache.maven.plugins</groupId>
- <artifactId>maven-antrun-plugin</artifactId>
- <executions>
- <execution>
- <id>generate-protobuf-sources</id>
- <phase>generate-sources</phase>
- <configuration>
- <target>
- <property name="protobuf.src.dir" location="${basedir}/src/protobuf"/>
- <property name="protobuf.build.dir" location="${basedir}/src/gen/protobuf/gen-java"/>
- <echo>Building HBase Metastore Protobuf</echo>
- <mkdir dir="${protobuf.build.dir}"/>
- <exec executable="protoc" failonerror="true">
- <arg value="--java_out=${protobuf.build.dir}"/>
- <arg value="-I=${protobuf.src.dir}/org/apache/hadoop/hive/metastore"/>
- <arg value="${protobuf.src.dir}/org/apache/hadoop/hive/metastore/metastore.proto"/>
- </exec>
- </target>
- </configuration>
- <goals>
- <goal>run</goal>
- </goals>
- </execution>
- </executions>
- </plugin>
- </plugins>
- </build>
- </profile>
</profiles>
<build>
@@ -343,7 +305,6 @@
<configuration>
<sources>
<source>src/model</source>
- <source>src/gen/protobuf/gen-java</source>
</sources>
</configuration>
</execution>
http://git-wip-us.apache.org/repos/asf/hive/blob/b0b6db73/metastore/src/gen/protobuf/gen-java/org/apache/hadoop/hive/metastore/Metastore.java
----------------------------------------------------------------------
diff --git a/metastore/src/gen/protobuf/gen-java/org/apache/hadoop/hive/metastore/Metastore.java b/metastore/src/gen/protobuf/gen-java/org/apache/hadoop/hive/metastore/Metastore.java
deleted file mode 100644
index ca928b9..0000000
--- a/metastore/src/gen/protobuf/gen-java/org/apache/hadoop/hive/metastore/Metastore.java
+++ /dev/null
@@ -1,1331 +0,0 @@
-// Generated by the protocol buffer compiler. DO NOT EDIT!
-// source: metastore.proto
-
-package org.apache.hadoop.hive.metastore;
-
-public final class Metastore {
- private Metastore() {}
- public static void registerAllExtensions(
- com.google.protobuf.ExtensionRegistry registry) {
- }
- public interface SplitInfoOrBuilder
- extends com.google.protobuf.MessageOrBuilder {
-
- // required int64 offset = 1;
- /**
- * <code>required int64 offset = 1;</code>
- */
- boolean hasOffset();
- /**
- * <code>required int64 offset = 1;</code>
- */
- long getOffset();
-
- // required int64 length = 2;
- /**
- * <code>required int64 length = 2;</code>
- */
- boolean hasLength();
- /**
- * <code>required int64 length = 2;</code>
- */
- long getLength();
-
- // required int32 index = 3;
- /**
- * <code>required int32 index = 3;</code>
- */
- boolean hasIndex();
- /**
- * <code>required int32 index = 3;</code>
- */
- int getIndex();
- }
- /**
- * Protobuf type {@code org.apache.hadoop.hive.metastore.SplitInfo}
- */
- public static final class SplitInfo extends
- com.google.protobuf.GeneratedMessage
- implements SplitInfoOrBuilder {
- // Use SplitInfo.newBuilder() to construct.
- private SplitInfo(com.google.protobuf.GeneratedMessage.Builder<?> builder) {
- super(builder);
- this.unknownFields = builder.getUnknownFields();
- }
- private SplitInfo(boolean noInit) { this.unknownFields = com.google.protobuf.UnknownFieldSet.getDefaultInstance(); }
-
- private static final SplitInfo defaultInstance;
- public static SplitInfo getDefaultInstance() {
- return defaultInstance;
- }
-
- public SplitInfo getDefaultInstanceForType() {
- return defaultInstance;
- }
-
- private final com.google.protobuf.UnknownFieldSet unknownFields;
- @java.lang.Override
- public final com.google.protobuf.UnknownFieldSet
- getUnknownFields() {
- return this.unknownFields;
- }
- private SplitInfo(
- com.google.protobuf.CodedInputStream input,
- com.google.protobuf.ExtensionRegistryLite extensionRegistry)
- throws com.google.protobuf.InvalidProtocolBufferException {
- initFields();
- int mutable_bitField0_ = 0;
- com.google.protobuf.UnknownFieldSet.Builder unknownFields =
- com.google.protobuf.UnknownFieldSet.newBuilder();
- try {
- boolean done = false;
- while (!done) {
- int tag = input.readTag();
- switch (tag) {
- case 0:
- done = true;
- break;
- default: {
- if (!parseUnknownField(input, unknownFields,
- extensionRegistry, tag)) {
- done = true;
- }
- break;
- }
- case 8: {
- bitField0_ |= 0x00000001;
- offset_ = input.readInt64();
- break;
- }
- case 16: {
- bitField0_ |= 0x00000002;
- length_ = input.readInt64();
- break;
- }
- case 24: {
- bitField0_ |= 0x00000004;
- index_ = input.readInt32();
- break;
- }
- }
- }
- } catch (com.google.protobuf.InvalidProtocolBufferException e) {
- throw e.setUnfinishedMessage(this);
- } catch (java.io.IOException e) {
- throw new com.google.protobuf.InvalidProtocolBufferException(
- e.getMessage()).setUnfinishedMessage(this);
- } finally {
- this.unknownFields = unknownFields.build();
- makeExtensionsImmutable();
- }
- }
- public static final com.google.protobuf.Descriptors.Descriptor
- getDescriptor() {
- return org.apache.hadoop.hive.metastore.Metastore.internal_static_org_apache_hadoop_hive_metastore_SplitInfo_descriptor;
- }
-
- protected com.google.protobuf.GeneratedMessage.FieldAccessorTable
- internalGetFieldAccessorTable() {
- return org.apache.hadoop.hive.metastore.Metastore.internal_static_org_apache_hadoop_hive_metastore_SplitInfo_fieldAccessorTable
- .ensureFieldAccessorsInitialized(
- org.apache.hadoop.hive.metastore.Metastore.SplitInfo.class, org.apache.hadoop.hive.metastore.Metastore.SplitInfo.Builder.class);
- }
-
- public static com.google.protobuf.Parser<SplitInfo> PARSER =
- new com.google.protobuf.AbstractParser<SplitInfo>() {
- public SplitInfo parsePartialFrom(
- com.google.protobuf.CodedInputStream input,
- com.google.protobuf.ExtensionRegistryLite extensionRegistry)
- throws com.google.protobuf.InvalidProtocolBufferException {
- return new SplitInfo(input, extensionRegistry);
- }
- };
-
- @java.lang.Override
- public com.google.protobuf.Parser<SplitInfo> getParserForType() {
- return PARSER;
- }
-
- private int bitField0_;
- // required int64 offset = 1;
- public static final int OFFSET_FIELD_NUMBER = 1;
- private long offset_;
- /**
- * <code>required int64 offset = 1;</code>
- */
- public boolean hasOffset() {
- return ((bitField0_ & 0x00000001) == 0x00000001);
- }
- /**
- * <code>required int64 offset = 1;</code>
- */
- public long getOffset() {
- return offset_;
- }
-
- // required int64 length = 2;
- public static final int LENGTH_FIELD_NUMBER = 2;
- private long length_;
- /**
- * <code>required int64 length = 2;</code>
- */
- public boolean hasLength() {
- return ((bitField0_ & 0x00000002) == 0x00000002);
- }
- /**
- * <code>required int64 length = 2;</code>
- */
- public long getLength() {
- return length_;
- }
-
- // required int32 index = 3;
- public static final int INDEX_FIELD_NUMBER = 3;
- private int index_;
- /**
- * <code>required int32 index = 3;</code>
- */
- public boolean hasIndex() {
- return ((bitField0_ & 0x00000004) == 0x00000004);
- }
- /**
- * <code>required int32 index = 3;</code>
- */
- public int getIndex() {
- return index_;
- }
-
- private void initFields() {
- offset_ = 0L;
- length_ = 0L;
- index_ = 0;
- }
- private byte memoizedIsInitialized = -1;
- public final boolean isInitialized() {
- byte isInitialized = memoizedIsInitialized;
- if (isInitialized != -1) return isInitialized == 1;
-
- if (!hasOffset()) {
- memoizedIsInitialized = 0;
- return false;
- }
- if (!hasLength()) {
- memoizedIsInitialized = 0;
- return false;
- }
- if (!hasIndex()) {
- memoizedIsInitialized = 0;
- return false;
- }
- memoizedIsInitialized = 1;
- return true;
- }
-
- public void writeTo(com.google.protobuf.CodedOutputStream output)
- throws java.io.IOException {
- getSerializedSize();
- if (((bitField0_ & 0x00000001) == 0x00000001)) {
- output.writeInt64(1, offset_);
- }
- if (((bitField0_ & 0x00000002) == 0x00000002)) {
- output.writeInt64(2, length_);
- }
- if (((bitField0_ & 0x00000004) == 0x00000004)) {
- output.writeInt32(3, index_);
- }
- getUnknownFields().writeTo(output);
- }
-
- private int memoizedSerializedSize = -1;
- public int getSerializedSize() {
- int size = memoizedSerializedSize;
- if (size != -1) return size;
-
- size = 0;
- if (((bitField0_ & 0x00000001) == 0x00000001)) {
- size += com.google.protobuf.CodedOutputStream
- .computeInt64Size(1, offset_);
- }
- if (((bitField0_ & 0x00000002) == 0x00000002)) {
- size += com.google.protobuf.CodedOutputStream
- .computeInt64Size(2, length_);
- }
- if (((bitField0_ & 0x00000004) == 0x00000004)) {
- size += com.google.protobuf.CodedOutputStream
- .computeInt32Size(3, index_);
- }
- size += getUnknownFields().getSerializedSize();
- memoizedSerializedSize = size;
- return size;
- }
-
- private static final long serialVersionUID = 0L;
- @java.lang.Override
- protected java.lang.Object writeReplace()
- throws java.io.ObjectStreamException {
- return super.writeReplace();
- }
-
- public static org.apache.hadoop.hive.metastore.Metastore.SplitInfo parseFrom(
- com.google.protobuf.ByteString data)
- throws com.google.protobuf.InvalidProtocolBufferException {
- return PARSER.parseFrom(data);
- }
- public static org.apache.hadoop.hive.metastore.Metastore.SplitInfo parseFrom(
- com.google.protobuf.ByteString data,
- com.google.protobuf.ExtensionRegistryLite extensionRegistry)
- throws com.google.protobuf.InvalidProtocolBufferException {
- return PARSER.parseFrom(data, extensionRegistry);
- }
- public static org.apache.hadoop.hive.metastore.Metastore.SplitInfo parseFrom(byte[] data)
- throws com.google.protobuf.InvalidProtocolBufferException {
- return PARSER.parseFrom(data);
- }
- public static org.apache.hadoop.hive.metastore.Metastore.SplitInfo parseFrom(
- byte[] data,
- com.google.protobuf.ExtensionRegistryLite extensionRegistry)
- throws com.google.protobuf.InvalidProtocolBufferException {
- return PARSER.parseFrom(data, extensionRegistry);
- }
- public static org.apache.hadoop.hive.metastore.Metastore.SplitInfo parseFrom(java.io.InputStream input)
- throws java.io.IOException {
- return PARSER.parseFrom(input);
- }
- public static org.apache.hadoop.hive.metastore.Metastore.SplitInfo parseFrom(
- java.io.InputStream input,
- com.google.protobuf.ExtensionRegistryLite extensionRegistry)
- throws java.io.IOException {
- return PARSER.parseFrom(input, extensionRegistry);
- }
- public static org.apache.hadoop.hive.metastore.Metastore.SplitInfo parseDelimitedFrom(java.io.InputStream input)
- throws java.io.IOException {
- return PARSER.parseDelimitedFrom(input);
- }
- public static org.apache.hadoop.hive.metastore.Metastore.SplitInfo parseDelimitedFrom(
- java.io.InputStream input,
- com.google.protobuf.ExtensionRegistryLite extensionRegistry)
- throws java.io.IOException {
- return PARSER.parseDelimitedFrom(input, extensionRegistry);
- }
- public static org.apache.hadoop.hive.metastore.Metastore.SplitInfo parseFrom(
- com.google.protobuf.CodedInputStream input)
- throws java.io.IOException {
- return PARSER.parseFrom(input);
- }
- public static org.apache.hadoop.hive.metastore.Metastore.SplitInfo parseFrom(
- com.google.protobuf.CodedInputStream input,
- com.google.protobuf.ExtensionRegistryLite extensionRegistry)
- throws java.io.IOException {
- return PARSER.parseFrom(input, extensionRegistry);
- }
-
- public static Builder newBuilder() { return Builder.create(); }
- public Builder newBuilderForType() { return newBuilder(); }
- public static Builder newBuilder(org.apache.hadoop.hive.metastore.Metastore.SplitInfo prototype) {
- return newBuilder().mergeFrom(prototype);
- }
- public Builder toBuilder() { return newBuilder(this); }
-
- @java.lang.Override
- protected Builder newBuilderForType(
- com.google.protobuf.GeneratedMessage.BuilderParent parent) {
- Builder builder = new Builder(parent);
- return builder;
- }
- /**
- * Protobuf type {@code org.apache.hadoop.hive.metastore.SplitInfo}
- */
- public static final class Builder extends
- com.google.protobuf.GeneratedMessage.Builder<Builder>
- implements org.apache.hadoop.hive.metastore.Metastore.SplitInfoOrBuilder {
- public static final com.google.protobuf.Descriptors.Descriptor
- getDescriptor() {
- return org.apache.hadoop.hive.metastore.Metastore.internal_static_org_apache_hadoop_hive_metastore_SplitInfo_descriptor;
- }
-
- protected com.google.protobuf.GeneratedMessage.FieldAccessorTable
- internalGetFieldAccessorTable() {
- return org.apache.hadoop.hive.metastore.Metastore.internal_static_org_apache_hadoop_hive_metastore_SplitInfo_fieldAccessorTable
- .ensureFieldAccessorsInitialized(
- org.apache.hadoop.hive.metastore.Metastore.SplitInfo.class, org.apache.hadoop.hive.metastore.Metastore.SplitInfo.Builder.class);
- }
-
- // Construct using org.apache.hadoop.hive.metastore.Metastore.SplitInfo.newBuilder()
- private Builder() {
- maybeForceBuilderInitialization();
- }
-
- private Builder(
- com.google.protobuf.GeneratedMessage.BuilderParent parent) {
- super(parent);
- maybeForceBuilderInitialization();
- }
- private void maybeForceBuilderInitialization() {
- if (com.google.protobuf.GeneratedMessage.alwaysUseFieldBuilders) {
- }
- }
- private static Builder create() {
- return new Builder();
- }
-
- public Builder clear() {
- super.clear();
- offset_ = 0L;
- bitField0_ = (bitField0_ & ~0x00000001);
- length_ = 0L;
- bitField0_ = (bitField0_ & ~0x00000002);
- index_ = 0;
- bitField0_ = (bitField0_ & ~0x00000004);
- return this;
- }
-
- public Builder clone() {
- return create().mergeFrom(buildPartial());
- }
-
- public com.google.protobuf.Descriptors.Descriptor
- getDescriptorForType() {
- return org.apache.hadoop.hive.metastore.Metastore.internal_static_org_apache_hadoop_hive_metastore_SplitInfo_descriptor;
- }
-
- public org.apache.hadoop.hive.metastore.Metastore.SplitInfo getDefaultInstanceForType() {
- return org.apache.hadoop.hive.metastore.Metastore.SplitInfo.getDefaultInstance();
- }
-
- public org.apache.hadoop.hive.metastore.Metastore.SplitInfo build() {
- org.apache.hadoop.hive.metastore.Metastore.SplitInfo result = buildPartial();
- if (!result.isInitialized()) {
- throw newUninitializedMessageException(result);
- }
- return result;
- }
-
- public org.apache.hadoop.hive.metastore.Metastore.SplitInfo buildPartial() {
- org.apache.hadoop.hive.metastore.Metastore.SplitInfo result = new org.apache.hadoop.hive.metastore.Metastore.SplitInfo(this);
- int from_bitField0_ = bitField0_;
- int to_bitField0_ = 0;
- if (((from_bitField0_ & 0x00000001) == 0x00000001)) {
- to_bitField0_ |= 0x00000001;
- }
- result.offset_ = offset_;
- if (((from_bitField0_ & 0x00000002) == 0x00000002)) {
- to_bitField0_ |= 0x00000002;
- }
- result.length_ = length_;
- if (((from_bitField0_ & 0x00000004) == 0x00000004)) {
- to_bitField0_ |= 0x00000004;
- }
- result.index_ = index_;
- result.bitField0_ = to_bitField0_;
- onBuilt();
- return result;
- }
-
- public Builder mergeFrom(com.google.protobuf.Message other) {
- if (other instanceof org.apache.hadoop.hive.metastore.Metastore.SplitInfo) {
- return mergeFrom((org.apache.hadoop.hive.metastore.Metastore.SplitInfo)other);
- } else {
- super.mergeFrom(other);
- return this;
- }
- }
-
- public Builder mergeFrom(org.apache.hadoop.hive.metastore.Metastore.SplitInfo other) {
- if (other == org.apache.hadoop.hive.metastore.Metastore.SplitInfo.getDefaultInstance()) return this;
- if (other.hasOffset()) {
- setOffset(other.getOffset());
- }
- if (other.hasLength()) {
- setLength(other.getLength());
- }
- if (other.hasIndex()) {
- setIndex(other.getIndex());
- }
- this.mergeUnknownFields(other.getUnknownFields());
- return this;
- }
-
- public final boolean isInitialized() {
- if (!hasOffset()) {
-
- return false;
- }
- if (!hasLength()) {
-
- return false;
- }
- if (!hasIndex()) {
-
- return false;
- }
- return true;
- }
-
- public Builder mergeFrom(
- com.google.protobuf.CodedInputStream input,
- com.google.protobuf.ExtensionRegistryLite extensionRegistry)
- throws java.io.IOException {
- org.apache.hadoop.hive.metastore.Metastore.SplitInfo parsedMessage = null;
- try {
- parsedMessage = PARSER.parsePartialFrom(input, extensionRegistry);
- } catch (com.google.protobuf.InvalidProtocolBufferException e) {
- parsedMessage = (org.apache.hadoop.hive.metastore.Metastore.SplitInfo) e.getUnfinishedMessage();
- throw e;
- } finally {
- if (parsedMessage != null) {
- mergeFrom(parsedMessage);
- }
- }
- return this;
- }
- private int bitField0_;
-
- // required int64 offset = 1;
- private long offset_ ;
- /**
- * <code>required int64 offset = 1;</code>
- */
- public boolean hasOffset() {
- return ((bitField0_ & 0x00000001) == 0x00000001);
- }
- /**
- * <code>required int64 offset = 1;</code>
- */
- public long getOffset() {
- return offset_;
- }
- /**
- * <code>required int64 offset = 1;</code>
- */
- public Builder setOffset(long value) {
- bitField0_ |= 0x00000001;
- offset_ = value;
- onChanged();
- return this;
- }
- /**
- * <code>required int64 offset = 1;</code>
- */
- public Builder clearOffset() {
- bitField0_ = (bitField0_ & ~0x00000001);
- offset_ = 0L;
- onChanged();
- return this;
- }
-
- // required int64 length = 2;
- private long length_ ;
- /**
- * <code>required int64 length = 2;</code>
- */
- public boolean hasLength() {
- return ((bitField0_ & 0x00000002) == 0x00000002);
- }
- /**
- * <code>required int64 length = 2;</code>
- */
- public long getLength() {
- return length_;
- }
- /**
- * <code>required int64 length = 2;</code>
- */
- public Builder setLength(long value) {
- bitField0_ |= 0x00000002;
- length_ = value;
- onChanged();
- return this;
- }
- /**
- * <code>required int64 length = 2;</code>
- */
- public Builder clearLength() {
- bitField0_ = (bitField0_ & ~0x00000002);
- length_ = 0L;
- onChanged();
- return this;
- }
-
- // required int32 index = 3;
- private int index_ ;
- /**
- * <code>required int32 index = 3;</code>
- */
- public boolean hasIndex() {
- return ((bitField0_ & 0x00000004) == 0x00000004);
- }
- /**
- * <code>required int32 index = 3;</code>
- */
- public int getIndex() {
- return index_;
- }
- /**
- * <code>required int32 index = 3;</code>
- */
- public Builder setIndex(int value) {
- bitField0_ |= 0x00000004;
- index_ = value;
- onChanged();
- return this;
- }
- /**
- * <code>required int32 index = 3;</code>
- */
- public Builder clearIndex() {
- bitField0_ = (bitField0_ & ~0x00000004);
- index_ = 0;
- onChanged();
- return this;
- }
-
- // @@protoc_insertion_point(builder_scope:org.apache.hadoop.hive.metastore.SplitInfo)
- }
-
- static {
- defaultInstance = new SplitInfo(true);
- defaultInstance.initFields();
- }
-
- // @@protoc_insertion_point(class_scope:org.apache.hadoop.hive.metastore.SplitInfo)
- }
-
- public interface SplitInfosOrBuilder
- extends com.google.protobuf.MessageOrBuilder {
-
- // repeated .org.apache.hadoop.hive.metastore.SplitInfo infos = 1;
- /**
- * <code>repeated .org.apache.hadoop.hive.metastore.SplitInfo infos = 1;</code>
- */
- java.util.List<org.apache.hadoop.hive.metastore.Metastore.SplitInfo>
- getInfosList();
- /**
- * <code>repeated .org.apache.hadoop.hive.metastore.SplitInfo infos = 1;</code>
- */
- org.apache.hadoop.hive.metastore.Metastore.SplitInfo getInfos(int index);
- /**
- * <code>repeated .org.apache.hadoop.hive.metastore.SplitInfo infos = 1;</code>
- */
- int getInfosCount();
- /**
- * <code>repeated .org.apache.hadoop.hive.metastore.SplitInfo infos = 1;</code>
- */
- java.util.List<? extends org.apache.hadoop.hive.metastore.Metastore.SplitInfoOrBuilder>
- getInfosOrBuilderList();
- /**
- * <code>repeated .org.apache.hadoop.hive.metastore.SplitInfo infos = 1;</code>
- */
- org.apache.hadoop.hive.metastore.Metastore.SplitInfoOrBuilder getInfosOrBuilder(
- int index);
- }
- /**
- * Protobuf type {@code org.apache.hadoop.hive.metastore.SplitInfos}
- */
- public static final class SplitInfos extends
- com.google.protobuf.GeneratedMessage
- implements SplitInfosOrBuilder {
- // Use SplitInfos.newBuilder() to construct.
- private SplitInfos(com.google.protobuf.GeneratedMessage.Builder<?> builder) {
- super(builder);
- this.unknownFields = builder.getUnknownFields();
- }
- private SplitInfos(boolean noInit) { this.unknownFields = com.google.protobuf.UnknownFieldSet.getDefaultInstance(); }
-
- private static final SplitInfos defaultInstance;
- public static SplitInfos getDefaultInstance() {
- return defaultInstance;
- }
-
- public SplitInfos getDefaultInstanceForType() {
- return defaultInstance;
- }
-
- private final com.google.protobuf.UnknownFieldSet unknownFields;
- @java.lang.Override
- public final com.google.protobuf.UnknownFieldSet
- getUnknownFields() {
- return this.unknownFields;
- }
- private SplitInfos(
- com.google.protobuf.CodedInputStream input,
- com.google.protobuf.ExtensionRegistryLite extensionRegistry)
- throws com.google.protobuf.InvalidProtocolBufferException {
- initFields();
- int mutable_bitField0_ = 0;
- com.google.protobuf.UnknownFieldSet.Builder unknownFields =
- com.google.protobuf.UnknownFieldSet.newBuilder();
- try {
- boolean done = false;
- while (!done) {
- int tag = input.readTag();
- switch (tag) {
- case 0:
- done = true;
- break;
- default: {
- if (!parseUnknownField(input, unknownFields,
- extensionRegistry, tag)) {
- done = true;
- }
- break;
- }
- case 10: {
- if (!((mutable_bitField0_ & 0x00000001) == 0x00000001)) {
- infos_ = new java.util.ArrayList<org.apache.hadoop.hive.metastore.Metastore.SplitInfo>();
- mutable_bitField0_ |= 0x00000001;
- }
- infos_.add(input.readMessage(org.apache.hadoop.hive.metastore.Metastore.SplitInfo.PARSER, extensionRegistry));
- break;
- }
- }
- }
- } catch (com.google.protobuf.InvalidProtocolBufferException e) {
- throw e.setUnfinishedMessage(this);
- } catch (java.io.IOException e) {
- throw new com.google.protobuf.InvalidProtocolBufferException(
- e.getMessage()).setUnfinishedMessage(this);
- } finally {
- if (((mutable_bitField0_ & 0x00000001) == 0x00000001)) {
- infos_ = java.util.Collections.unmodifiableList(infos_);
- }
- this.unknownFields = unknownFields.build();
- makeExtensionsImmutable();
- }
- }
- public static final com.google.protobuf.Descriptors.Descriptor
- getDescriptor() {
- return org.apache.hadoop.hive.metastore.Metastore.internal_static_org_apache_hadoop_hive_metastore_SplitInfos_descriptor;
- }
-
- protected com.google.protobuf.GeneratedMessage.FieldAccessorTable
- internalGetFieldAccessorTable() {
- return org.apache.hadoop.hive.metastore.Metastore.internal_static_org_apache_hadoop_hive_metastore_SplitInfos_fieldAccessorTable
- .ensureFieldAccessorsInitialized(
- org.apache.hadoop.hive.metastore.Metastore.SplitInfos.class, org.apache.hadoop.hive.metastore.Metastore.SplitInfos.Builder.class);
- }
-
- public static com.google.protobuf.Parser<SplitInfos> PARSER =
- new com.google.protobuf.AbstractParser<SplitInfos>() {
- public SplitInfos parsePartialFrom(
- com.google.protobuf.CodedInputStream input,
- com.google.protobuf.ExtensionRegistryLite extensionRegistry)
- throws com.google.protobuf.InvalidProtocolBufferException {
- return new SplitInfos(input, extensionRegistry);
- }
- };
-
- @java.lang.Override
- public com.google.protobuf.Parser<SplitInfos> getParserForType() {
- return PARSER;
- }
-
- // repeated .org.apache.hadoop.hive.metastore.SplitInfo infos = 1;
- public static final int INFOS_FIELD_NUMBER = 1;
- private java.util.List<org.apache.hadoop.hive.metastore.Metastore.SplitInfo> infos_;
- /**
- * <code>repeated .org.apache.hadoop.hive.metastore.SplitInfo infos = 1;</code>
- */
- public java.util.List<org.apache.hadoop.hive.metastore.Metastore.SplitInfo> getInfosList() {
- return infos_;
- }
- /**
- * <code>repeated .org.apache.hadoop.hive.metastore.SplitInfo infos = 1;</code>
- */
- public java.util.List<? extends org.apache.hadoop.hive.metastore.Metastore.SplitInfoOrBuilder>
- getInfosOrBuilderList() {
- return infos_;
- }
- /**
- * <code>repeated .org.apache.hadoop.hive.metastore.SplitInfo infos = 1;</code>
- */
- public int getInfosCount() {
- return infos_.size();
- }
- /**
- * <code>repeated .org.apache.hadoop.hive.metastore.SplitInfo infos = 1;</code>
- */
- public org.apache.hadoop.hive.metastore.Metastore.SplitInfo getInfos(int index) {
- return infos_.get(index);
- }
- /**
- * <code>repeated .org.apache.hadoop.hive.metastore.SplitInfo infos = 1;</code>
- */
- public org.apache.hadoop.hive.metastore.Metastore.SplitInfoOrBuilder getInfosOrBuilder(
- int index) {
- return infos_.get(index);
- }
-
- private void initFields() {
- infos_ = java.util.Collections.emptyList();
- }
- private byte memoizedIsInitialized = -1;
- public final boolean isInitialized() {
- byte isInitialized = memoizedIsInitialized;
- if (isInitialized != -1) return isInitialized == 1;
-
- for (int i = 0; i < getInfosCount(); i++) {
- if (!getInfos(i).isInitialized()) {
- memoizedIsInitialized = 0;
- return false;
- }
- }
- memoizedIsInitialized = 1;
- return true;
- }
-
- public void writeTo(com.google.protobuf.CodedOutputStream output)
- throws java.io.IOException {
- getSerializedSize();
- for (int i = 0; i < infos_.size(); i++) {
- output.writeMessage(1, infos_.get(i));
- }
- getUnknownFields().writeTo(output);
- }
-
- private int memoizedSerializedSize = -1;
- public int getSerializedSize() {
- int size = memoizedSerializedSize;
- if (size != -1) return size;
-
- size = 0;
- for (int i = 0; i < infos_.size(); i++) {
- size += com.google.protobuf.CodedOutputStream
- .computeMessageSize(1, infos_.get(i));
- }
- size += getUnknownFields().getSerializedSize();
- memoizedSerializedSize = size;
- return size;
- }
-
- private static final long serialVersionUID = 0L;
- @java.lang.Override
- protected java.lang.Object writeReplace()
- throws java.io.ObjectStreamException {
- return super.writeReplace();
- }
-
- public static org.apache.hadoop.hive.metastore.Metastore.SplitInfos parseFrom(
- com.google.protobuf.ByteString data)
- throws com.google.protobuf.InvalidProtocolBufferException {
- return PARSER.parseFrom(data);
- }
- public static org.apache.hadoop.hive.metastore.Metastore.SplitInfos parseFrom(
- com.google.protobuf.ByteString data,
- com.google.protobuf.ExtensionRegistryLite extensionRegistry)
- throws com.google.protobuf.InvalidProtocolBufferException {
- return PARSER.parseFrom(data, extensionRegistry);
- }
- public static org.apache.hadoop.hive.metastore.Metastore.SplitInfos parseFrom(byte[] data)
- throws com.google.protobuf.InvalidProtocolBufferException {
- return PARSER.parseFrom(data);
- }
- public static org.apache.hadoop.hive.metastore.Metastore.SplitInfos parseFrom(
- byte[] data,
- com.google.protobuf.ExtensionRegistryLite extensionRegistry)
- throws com.google.protobuf.InvalidProtocolBufferException {
- return PARSER.parseFrom(data, extensionRegistry);
- }
- public static org.apache.hadoop.hive.metastore.Metastore.SplitInfos parseFrom(java.io.InputStream input)
- throws java.io.IOException {
- return PARSER.parseFrom(input);
- }
- public static org.apache.hadoop.hive.metastore.Metastore.SplitInfos parseFrom(
- java.io.InputStream input,
- com.google.protobuf.ExtensionRegistryLite extensionRegistry)
- throws java.io.IOException {
- return PARSER.parseFrom(input, extensionRegistry);
- }
- public static org.apache.hadoop.hive.metastore.Metastore.SplitInfos parseDelimitedFrom(java.io.InputStream input)
- throws java.io.IOException {
- return PARSER.parseDelimitedFrom(input);
- }
- public static org.apache.hadoop.hive.metastore.Metastore.SplitInfos parseDelimitedFrom(
- java.io.InputStream input,
- com.google.protobuf.ExtensionRegistryLite extensionRegistry)
- throws java.io.IOException {
- return PARSER.parseDelimitedFrom(input, extensionRegistry);
- }
- public static org.apache.hadoop.hive.metastore.Metastore.SplitInfos parseFrom(
- com.google.protobuf.CodedInputStream input)
- throws java.io.IOException {
- return PARSER.parseFrom(input);
- }
- public static org.apache.hadoop.hive.metastore.Metastore.SplitInfos parseFrom(
- com.google.protobuf.CodedInputStream input,
- com.google.protobuf.ExtensionRegistryLite extensionRegistry)
- throws java.io.IOException {
- return PARSER.parseFrom(input, extensionRegistry);
- }
-
- public static Builder newBuilder() { return Builder.create(); }
- public Builder newBuilderForType() { return newBuilder(); }
- public static Builder newBuilder(org.apache.hadoop.hive.metastore.Metastore.SplitInfos prototype) {
- return newBuilder().mergeFrom(prototype);
- }
- public Builder toBuilder() { return newBuilder(this); }
-
- @java.lang.Override
- protected Builder newBuilderForType(
- com.google.protobuf.GeneratedMessage.BuilderParent parent) {
- Builder builder = new Builder(parent);
- return builder;
- }
- /**
- * Protobuf type {@code org.apache.hadoop.hive.metastore.SplitInfos}
- */
- public static final class Builder extends
- com.google.protobuf.GeneratedMessage.Builder<Builder>
- implements org.apache.hadoop.hive.metastore.Metastore.SplitInfosOrBuilder {
- public static final com.google.protobuf.Descriptors.Descriptor
- getDescriptor() {
- return org.apache.hadoop.hive.metastore.Metastore.internal_static_org_apache_hadoop_hive_metastore_SplitInfos_descriptor;
- }
-
- protected com.google.protobuf.GeneratedMessage.FieldAccessorTable
- internalGetFieldAccessorTable() {
- return org.apache.hadoop.hive.metastore.Metastore.internal_static_org_apache_hadoop_hive_metastore_SplitInfos_fieldAccessorTable
- .ensureFieldAccessorsInitialized(
- org.apache.hadoop.hive.metastore.Metastore.SplitInfos.class, org.apache.hadoop.hive.metastore.Metastore.SplitInfos.Builder.class);
- }
-
- // Construct using org.apache.hadoop.hive.metastore.Metastore.SplitInfos.newBuilder()
- private Builder() {
- maybeForceBuilderInitialization();
- }
-
- private Builder(
- com.google.protobuf.GeneratedMessage.BuilderParent parent) {
- super(parent);
- maybeForceBuilderInitialization();
- }
- private void maybeForceBuilderInitialization() {
- if (com.google.protobuf.GeneratedMessage.alwaysUseFieldBuilders) {
- getInfosFieldBuilder();
- }
- }
- private static Builder create() {
- return new Builder();
- }
-
- public Builder clear() {
- super.clear();
- if (infosBuilder_ == null) {
- infos_ = java.util.Collections.emptyList();
- bitField0_ = (bitField0_ & ~0x00000001);
- } else {
- infosBuilder_.clear();
- }
- return this;
- }
-
- public Builder clone() {
- return create().mergeFrom(buildPartial());
- }
-
- public com.google.protobuf.Descriptors.Descriptor
- getDescriptorForType() {
- return org.apache.hadoop.hive.metastore.Metastore.internal_static_org_apache_hadoop_hive_metastore_SplitInfos_descriptor;
- }
-
- public org.apache.hadoop.hive.metastore.Metastore.SplitInfos getDefaultInstanceForType() {
- return org.apache.hadoop.hive.metastore.Metastore.SplitInfos.getDefaultInstance();
- }
-
- public org.apache.hadoop.hive.metastore.Metastore.SplitInfos build() {
- org.apache.hadoop.hive.metastore.Metastore.SplitInfos result = buildPartial();
- if (!result.isInitialized()) {
- throw newUninitializedMessageException(result);
- }
- return result;
- }
-
- public org.apache.hadoop.hive.metastore.Metastore.SplitInfos buildPartial() {
- org.apache.hadoop.hive.metastore.Metastore.SplitInfos result = new org.apache.hadoop.hive.metastore.Metastore.SplitInfos(this);
- int from_bitField0_ = bitField0_;
- if (infosBuilder_ == null) {
- if (((bitField0_ & 0x00000001) == 0x00000001)) {
- infos_ = java.util.Collections.unmodifiableList(infos_);
- bitField0_ = (bitField0_ & ~0x00000001);
- }
- result.infos_ = infos_;
- } else {
- result.infos_ = infosBuilder_.build();
- }
- onBuilt();
- return result;
- }
-
- public Builder mergeFrom(com.google.protobuf.Message other) {
- if (other instanceof org.apache.hadoop.hive.metastore.Metastore.SplitInfos) {
- return mergeFrom((org.apache.hadoop.hive.metastore.Metastore.SplitInfos)other);
- } else {
- super.mergeFrom(other);
- return this;
- }
- }
-
- public Builder mergeFrom(org.apache.hadoop.hive.metastore.Metastore.SplitInfos other) {
- if (other == org.apache.hadoop.hive.metastore.Metastore.SplitInfos.getDefaultInstance()) return this;
- if (infosBuilder_ == null) {
- if (!other.infos_.isEmpty()) {
- if (infos_.isEmpty()) {
- infos_ = other.infos_;
- bitField0_ = (bitField0_ & ~0x00000001);
- } else {
- ensureInfosIsMutable();
- infos_.addAll(other.infos_);
- }
- onChanged();
- }
- } else {
- if (!other.infos_.isEmpty()) {
- if (infosBuilder_.isEmpty()) {
- infosBuilder_.dispose();
- infosBuilder_ = null;
- infos_ = other.infos_;
- bitField0_ = (bitField0_ & ~0x00000001);
- infosBuilder_ =
- com.google.protobuf.GeneratedMessage.alwaysUseFieldBuilders ?
- getInfosFieldBuilder() : null;
- } else {
- infosBuilder_.addAllMessages(other.infos_);
- }
- }
- }
- this.mergeUnknownFields(other.getUnknownFields());
- return this;
- }
-
- public final boolean isInitialized() {
- for (int i = 0; i < getInfosCount(); i++) {
- if (!getInfos(i).isInitialized()) {
-
- return false;
- }
- }
- return true;
- }
-
- public Builder mergeFrom(
- com.google.protobuf.CodedInputStream input,
- com.google.protobuf.ExtensionRegistryLite extensionRegistry)
- throws java.io.IOException {
- org.apache.hadoop.hive.metastore.Metastore.SplitInfos parsedMessage = null;
- try {
- parsedMessage = PARSER.parsePartialFrom(input, extensionRegistry);
- } catch (com.google.protobuf.InvalidProtocolBufferException e) {
- parsedMessage = (org.apache.hadoop.hive.metastore.Metastore.SplitInfos) e.getUnfinishedMessage();
- throw e;
- } finally {
- if (parsedMessage != null) {
- mergeFrom(parsedMessage);
- }
- }
- return this;
- }
- private int bitField0_;
-
- // repeated .org.apache.hadoop.hive.metastore.SplitInfo infos = 1;
- private java.util.List<org.apache.hadoop.hive.metastore.Metastore.SplitInfo> infos_ =
- java.util.Collections.emptyList();
- private void ensureInfosIsMutable() {
- if (!((bitField0_ & 0x00000001) == 0x00000001)) {
- infos_ = new java.util.ArrayList<org.apache.hadoop.hive.metastore.Metastore.SplitInfo>(infos_);
- bitField0_ |= 0x00000001;
- }
- }
-
- private com.google.protobuf.RepeatedFieldBuilder<
- org.apache.hadoop.hive.metastore.Metastore.SplitInfo, org.apache.hadoop.hive.metastore.Metastore.SplitInfo.Builder, org.apache.hadoop.hive.metastore.Metastore.SplitInfoOrBuilder> infosBuilder_;
-
- /**
- * <code>repeated .org.apache.hadoop.hive.metastore.SplitInfo infos = 1;</code>
- */
- public java.util.List<org.apache.hadoop.hive.metastore.Metastore.SplitInfo> getInfosList() {
- if (infosBuilder_ == null) {
- return java.util.Collections.unmodifiableList(infos_);
- } else {
- return infosBuilder_.getMessageList();
- }
- }
- /**
- * <code>repeated .org.apache.hadoop.hive.metastore.SplitInfo infos = 1;</code>
- */
- public int getInfosCount() {
- if (infosBuilder_ == null) {
- return infos_.size();
- } else {
- return infosBuilder_.getCount();
- }
- }
- /**
- * <code>repeated .org.apache.hadoop.hive.metastore.SplitInfo infos = 1;</code>
- */
- public org.apache.hadoop.hive.metastore.Metastore.SplitInfo getInfos(int index) {
- if (infosBuilder_ == null) {
- return infos_.get(index);
- } else {
- return infosBuilder_.getMessage(index);
- }
- }
- /**
- * <code>repeated .org.apache.hadoop.hive.metastore.SplitInfo infos = 1;</code>
- */
- public Builder setInfos(
- int index, org.apache.hadoop.hive.metastore.Metastore.SplitInfo value) {
- if (infosBuilder_ == null) {
- if (value == null) {
- throw new NullPointerException();
- }
- ensureInfosIsMutable();
- infos_.set(index, value);
- onChanged();
- } else {
- infosBuilder_.setMessage(index, value);
- }
- return this;
- }
- /**
- * <code>repeated .org.apache.hadoop.hive.metastore.SplitInfo infos = 1;</code>
- */
- public Builder setInfos(
- int index, org.apache.hadoop.hive.metastore.Metastore.SplitInfo.Builder builderForValue) {
- if (infosBuilder_ == null) {
- ensureInfosIsMutable();
- infos_.set(index, builderForValue.build());
- onChanged();
- } else {
- infosBuilder_.setMessage(index, builderForValue.build());
- }
- return this;
- }
- /**
- * <code>repeated .org.apache.hadoop.hive.metastore.SplitInfo infos = 1;</code>
- */
- public Builder addInfos(org.apache.hadoop.hive.metastore.Metastore.SplitInfo value) {
- if (infosBuilder_ == null) {
- if (value == null) {
- throw new NullPointerException();
- }
- ensureInfosIsMutable();
- infos_.add(value);
- onChanged();
- } else {
- infosBuilder_.addMessage(value);
- }
- return this;
- }
- /**
- * <code>repeated .org.apache.hadoop.hive.metastore.SplitInfo infos = 1;</code>
- */
- public Builder addInfos(
- int index, org.apache.hadoop.hive.metastore.Metastore.SplitInfo value) {
- if (infosBuilder_ == null) {
- if (value == null) {
- throw new NullPointerException();
- }
- ensureInfosIsMutable();
- infos_.add(index, value);
- onChanged();
- } else {
- infosBuilder_.addMessage(index, value);
- }
- return this;
- }
- /**
- * <code>repeated .org.apache.hadoop.hive.metastore.SplitInfo infos = 1;</code>
- */
- public Builder addInfos(
- org.apache.hadoop.hive.metastore.Metastore.SplitInfo.Builder builderForValue) {
- if (infosBuilder_ == null) {
- ensureInfosIsMutable();
- infos_.add(builderForValue.build());
- onChanged();
- } else {
- infosBuilder_.addMessage(builderForValue.build());
- }
- return this;
- }
- /**
- * <code>repeated .org.apache.hadoop.hive.metastore.SplitInfo infos = 1;</code>
- */
- public Builder addInfos(
- int index, org.apache.hadoop.hive.metastore.Metastore.SplitInfo.Builder builderForValue) {
- if (infosBuilder_ == null) {
- ensureInfosIsMutable();
- infos_.add(index, builderForValue.build());
- onChanged();
- } else {
- infosBuilder_.addMessage(index, builderForValue.build());
- }
- return this;
- }
- /**
- * <code>repeated .org.apache.hadoop.hive.metastore.SplitInfo infos = 1;</code>
- */
- public Builder addAllInfos(
- java.lang.Iterable<? extends org.apache.hadoop.hive.metastore.Metastore.SplitInfo> values) {
- if (infosBuilder_ == null) {
- ensureInfosIsMutable();
- super.addAll(values, infos_);
- onChanged();
- } else {
- infosBuilder_.addAllMessages(values);
- }
- return this;
- }
- /**
- * <code>repeated .org.apache.hadoop.hive.metastore.SplitInfo infos = 1;</code>
- */
- public Builder clearInfos() {
- if (infosBuilder_ == null) {
- infos_ = java.util.Collections.emptyList();
- bitField0_ = (bitField0_ & ~0x00000001);
- onChanged();
- } else {
- infosBuilder_.clear();
- }
- return this;
- }
- /**
- * <code>repeated .org.apache.hadoop.hive.metastore.SplitInfo infos = 1;</code>
- */
- public Builder removeInfos(int index) {
- if (infosBuilder_ == null) {
- ensureInfosIsMutable();
- infos_.remove(index);
- onChanged();
- } else {
- infosBuilder_.remove(index);
- }
- return this;
- }
- /**
- * <code>repeated .org.apache.hadoop.hive.metastore.SplitInfo infos = 1;</code>
- */
- public org.apache.hadoop.hive.metastore.Metastore.SplitInfo.Builder getInfosBuilder(
- int index) {
- return getInfosFieldBuilder().getBuilder(index);
- }
- /**
- * <code>repeated .org.apache.hadoop.hive.metastore.SplitInfo infos = 1;</code>
- */
- public org.apache.hadoop.hive.metastore.Metastore.SplitInfoOrBuilder getInfosOrBuilder(
- int index) {
- if (infosBuilder_ == null) {
- return infos_.get(index); } else {
- return infosBuilder_.getMessageOrBuilder(index);
- }
- }
- /**
- * <code>repeated .org.apache.hadoop.hive.metastore.SplitInfo infos = 1;</code>
- */
- public java.util.List<? extends org.apache.hadoop.hive.metastore.Metastore.SplitInfoOrBuilder>
- getInfosOrBuilderList() {
- if (infosBuilder_ != null) {
- return infosBuilder_.getMessageOrBuilderList();
- } else {
- return java.util.Collections.unmodifiableList(infos_);
- }
- }
- /**
- * <code>repeated .org.apache.hadoop.hive.metastore.SplitInfo infos = 1;</code>
- */
- public org.apache.hadoop.hive.metastore.Metastore.SplitInfo.Builder addInfosBuilder() {
- return getInfosFieldBuilder().addBuilder(
- org.apache.hadoop.hive.metastore.Metastore.SplitInfo.getDefaultInstance());
- }
- /**
- * <code>repeated .org.apache.hadoop.hive.metastore.SplitInfo infos = 1;</code>
- */
- public org.apache.hadoop.hive.metastore.Metastore.SplitInfo.Builder addInfosBuilder(
- int index) {
- return getInfosFieldBuilder().addBuilder(
- index, org.apache.hadoop.hive.metastore.Metastore.SplitInfo.getDefaultInstance());
- }
- /**
- * <code>repeated .org.apache.hadoop.hive.metastore.SplitInfo infos = 1;</code>
- */
- public java.util.List<org.apache.hadoop.hive.metastore.Metastore.SplitInfo.Builder>
- getInfosBuilderList() {
- return getInfosFieldBuilder().getBuilderList();
- }
- private com.google.protobuf.RepeatedFieldBuilder<
- org.apache.hadoop.hive.metastore.Metastore.SplitInfo, org.apache.hadoop.hive.metastore.Metastore.SplitInfo.Builder, org.apache.hadoop.hive.metastore.Metastore.SplitInfoOrBuilder>
- getInfosFieldBuilder() {
- if (infosBuilder_ == null) {
- infosBuilder_ = new com.google.protobuf.RepeatedFieldBuilder<
- org.apache.hadoop.hive.metastore.Metastore.SplitInfo, org.apache.hadoop.hive.metastore.Metastore.SplitInfo.Builder, org.apache.hadoop.hive.metastore.Metastore.SplitInfoOrBuilder>(
- infos_,
- ((bitField0_ & 0x00000001) == 0x00000001),
- getParentForChildren(),
- isClean());
- infos_ = null;
- }
- return infosBuilder_;
- }
-
- // @@protoc_insertion_point(builder_scope:org.apache.hadoop.hive.metastore.SplitInfos)
- }
-
- static {
- defaultInstance = new SplitInfos(true);
- defaultInstance.initFields();
- }
-
- // @@protoc_insertion_point(class_scope:org.apache.hadoop.hive.metastore.SplitInfos)
- }
-
- private static com.google.protobuf.Descriptors.Descriptor
- internal_static_org_apache_hadoop_hive_metastore_SplitInfo_descriptor;
- private static
- com.google.protobuf.GeneratedMessage.FieldAccessorTable
- internal_static_org_apache_hadoop_hive_metastore_SplitInfo_fieldAccessorTable;
- private static com.google.protobuf.Descriptors.Descriptor
- internal_static_org_apache_hadoop_hive_metastore_SplitInfos_descriptor;
- private static
- com.google.protobuf.GeneratedMessage.FieldAccessorTable
- internal_static_org_apache_hadoop_hive_metastore_SplitInfos_fieldAccessorTable;
-
- public static com.google.protobuf.Descriptors.FileDescriptor
- getDescriptor() {
- return descriptor;
- }
- private static com.google.protobuf.Descriptors.FileDescriptor
- descriptor;
- static {
- java.lang.String[] descriptorData = {
- "\n\017metastore.proto\022 org.apache.hadoop.hiv" +
- "e.metastore\":\n\tSplitInfo\022\016\n\006offset\030\001 \002(\003" +
- "\022\016\n\006length\030\002 \002(\003\022\r\n\005index\030\003 \002(\005\"H\n\nSplit" +
- "Infos\022:\n\005infos\030\001 \003(\0132+.org.apache.hadoop" +
- ".hive.metastore.SplitInfo"
- };
- com.google.protobuf.Descriptors.FileDescriptor.InternalDescriptorAssigner assigner =
- new com.google.protobuf.Descriptors.FileDescriptor.InternalDescriptorAssigner() {
- public com.google.protobuf.ExtensionRegistry assignDescriptors(
- com.google.protobuf.Descriptors.FileDescriptor root) {
- descriptor = root;
- internal_static_org_apache_hadoop_hive_metastore_SplitInfo_descriptor =
- getDescriptor().getMessageTypes().get(0);
- internal_static_org_apache_hadoop_hive_metastore_SplitInfo_fieldAccessorTable = new
- com.google.protobuf.GeneratedMessage.FieldAccessorTable(
- internal_static_org_apache_hadoop_hive_metastore_SplitInfo_descriptor,
- new java.lang.String[] { "Offset", "Length", "Index", });
- internal_static_org_apache_hadoop_hive_metastore_SplitInfos_descriptor =
- getDescriptor().getMessageTypes().get(1);
- internal_static_org_apache_hadoop_hive_metastore_SplitInfos_fieldAccessorTable = new
- com.google.protobuf.GeneratedMessage.FieldAccessorTable(
- internal_static_org_apache_hadoop_hive_metastore_SplitInfos_descriptor,
- new java.lang.String[] { "Infos", });
- return null;
- }
- };
- com.google.protobuf.Descriptors.FileDescriptor
- .internalBuildGeneratedFileFrom(descriptorData,
- new com.google.protobuf.Descriptors.FileDescriptor[] {
- }, assigner);
- }
-
- // @@protoc_insertion_point(outer_class_scope)
-}
[5/7] hive git commit: HIVE-17488 Move first set of classes to
standalone metastore. This closes #244. (Alan Gates,
reviewed by Owen O'Malley)
Posted by ga...@apache.org.
http://git-wip-us.apache.org/repos/asf/hive/blob/b0b6db73/metastore/src/java/org/apache/hadoop/hive/metastore/MetaStoreInit.java
----------------------------------------------------------------------
diff --git a/metastore/src/java/org/apache/hadoop/hive/metastore/MetaStoreInit.java b/metastore/src/java/org/apache/hadoop/hive/metastore/MetaStoreInit.java
deleted file mode 100644
index 6123a1e..0000000
--- a/metastore/src/java/org/apache/hadoop/hive/metastore/MetaStoreInit.java
+++ /dev/null
@@ -1,112 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.hive.metastore;
-
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.hive.common.JavaUtils;
-import org.apache.hadoop.hive.conf.HiveConf;
-import org.apache.hadoop.hive.metastore.api.MetaException;
-import org.apache.hadoop.hive.metastore.hooks.JDOConnectionURLHook;
-import org.apache.hadoop.util.ReflectionUtils;
-
-/**
- * MetaStoreInit defines functions to init/update MetaStore connection url.
- *
- */
-public class MetaStoreInit {
-
- private static final Logger LOG = LoggerFactory.getLogger(MetaStoreInit.class);
-
- static class MetaStoreInitData {
- JDOConnectionURLHook urlHook = null;
- String urlHookClassName = "";
- }
-
- /**
- * Updates the connection URL in hiveConf using the hook (if a hook has been
- * set using hive.metastore.ds.connection.url.hook property)
- * @param originalConf - original configuration used to look up hook settings
- * @param activeConf - the configuration file in use for looking up db url
- * @param badUrl
- * @param updateData - hook information
- * @return true if a new connection URL was loaded into the thread local
- * configuration
- * @throws MetaException
- */
- static boolean updateConnectionURL(HiveConf originalConf, Configuration activeConf,
- String badUrl, MetaStoreInitData updateData)
- throws MetaException {
- String connectUrl = null;
- String currentUrl = MetaStoreInit.getConnectionURL(activeConf);
- try {
- // We always call init because the hook name in the configuration could
- // have changed.
- MetaStoreInit.initConnectionUrlHook(originalConf, updateData);
- if (updateData.urlHook != null) {
- if (badUrl != null) {
- updateData.urlHook.notifyBadConnectionUrl(badUrl);
- }
- connectUrl = updateData.urlHook.getJdoConnectionUrl(originalConf);
- }
- } catch (Exception e) {
- LOG.error("Exception while getting connection URL from the hook: " +
- e);
- }
-
- if (connectUrl != null && !connectUrl.equals(currentUrl)) {
- LOG.error(
- String.format("Overriding %s with %s",
- HiveConf.ConfVars.METASTORECONNECTURLKEY.toString(),
- connectUrl));
- activeConf.set(HiveConf.ConfVars.METASTORECONNECTURLKEY.toString(),
- connectUrl);
- return true;
- }
- return false;
- }
-
- static String getConnectionURL(Configuration conf) {
- return conf.get(
- HiveConf.ConfVars.METASTORECONNECTURLKEY.toString(), "");
- }
-
- // Multiple threads could try to initialize at the same time.
- synchronized private static void initConnectionUrlHook(HiveConf hiveConf,
- MetaStoreInitData updateData) throws ClassNotFoundException {
-
- String className =
- hiveConf.get(HiveConf.ConfVars.METASTORECONNECTURLHOOK.toString(), "").trim();
- if (className.equals("")) {
- updateData.urlHookClassName = "";
- updateData.urlHook = null;
- return;
- }
- boolean urlHookChanged = !updateData.urlHookClassName.equals(className);
- if (updateData.urlHook == null || urlHookChanged) {
- updateData.urlHookClassName = className.trim();
-
- Class<?> urlHookClass = Class.forName(updateData.urlHookClassName, true,
- JavaUtils.getClassLoader());
- updateData.urlHook = (JDOConnectionURLHook) ReflectionUtils.newInstance(urlHookClass, null);
- }
- return;
- }
-}
http://git-wip-us.apache.org/repos/asf/hive/blob/b0b6db73/metastore/src/java/org/apache/hadoop/hive/metastore/MetaStoreSchemaInfo.java
----------------------------------------------------------------------
diff --git a/metastore/src/java/org/apache/hadoop/hive/metastore/MetaStoreSchemaInfo.java b/metastore/src/java/org/apache/hadoop/hive/metastore/MetaStoreSchemaInfo.java
deleted file mode 100644
index 8117ca1..0000000
--- a/metastore/src/java/org/apache/hadoop/hive/metastore/MetaStoreSchemaInfo.java
+++ /dev/null
@@ -1,234 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.hive.metastore;
-
-import java.io.BufferedReader;
-import java.io.File;
-import java.io.FileNotFoundException;
-import java.io.FileReader;
-import java.io.IOException;
-import java.sql.Connection;
-import java.sql.ResultSet;
-import java.sql.SQLException;
-import java.sql.Statement;
-import java.util.ArrayList;
-import java.util.List;
-import java.util.Map;
-
-import org.apache.hadoop.hive.metastore.tools.HiveSchemaHelper;
-import org.apache.hadoop.hive.metastore.tools.HiveSchemaHelper.MetaStoreConnectionInfo;
-import org.apache.hive.common.util.HiveVersionInfo;
-
-import com.google.common.collect.ImmutableMap;
-
-
-public class MetaStoreSchemaInfo implements IMetaStoreSchemaInfo {
- protected static final String UPGRADE_FILE_PREFIX = "upgrade-";
- private static final String INIT_FILE_PREFIX = "hive-schema-";
- private static final String VERSION_UPGRADE_LIST = "upgrade.order";
- private static final String PRE_UPGRADE_PREFIX = "pre-";
- protected final String dbType;
- private String[] hiveSchemaVersions;
- private final String hiveHome;
-
- // Some version upgrades often don't change schema. So they are equivalent to
- // a version
- // that has a corresponding schema. eg "0.13.1" is equivalent to "0.13.0"
- private static final Map<String, String> EQUIVALENT_VERSIONS =
- ImmutableMap.of("0.13.1", "0.13.0",
- "1.0.0", "0.14.0",
- "1.0.1", "1.0.0",
- "1.1.1", "1.1.0",
- "1.2.1", "1.2.0"
- );
-
- public MetaStoreSchemaInfo(String hiveHome, String dbType) throws HiveMetaException {
- this.hiveHome = hiveHome;
- this.dbType = dbType;
- }
-
- private void loadAllUpgradeScripts(String dbType) throws HiveMetaException {
- // load upgrade order for the given dbType
- List<String> upgradeOrderList = new ArrayList<String>();
- String upgradeListFile = getMetaStoreScriptDir() + File.separator +
- VERSION_UPGRADE_LIST + "." + dbType;
- try (FileReader fr = new FileReader(upgradeListFile);
- BufferedReader bfReader = new BufferedReader(fr)) {
- String currSchemaVersion;
- while ((currSchemaVersion = bfReader.readLine()) != null) {
- upgradeOrderList.add(currSchemaVersion.trim());
- }
- } catch (FileNotFoundException e) {
- throw new HiveMetaException("File " + upgradeListFile + "not found ", e);
- } catch (IOException e) {
- throw new HiveMetaException("Error reading " + upgradeListFile, e);
- }
- hiveSchemaVersions = upgradeOrderList.toArray(new String[0]);
- }
-
- /***
- * Get the list of sql scripts required to upgrade from the give version to current
- * @param fromVersion
- * @return
- * @throws HiveMetaException
- */
- @Override
- public List<String> getUpgradeScripts(String fromVersion)
- throws HiveMetaException {
- List <String> upgradeScriptList = new ArrayList<String>();
-
- // check if we are already at current schema level
- if (getHiveSchemaVersion().equals(fromVersion)) {
- return upgradeScriptList;
- }
- loadAllUpgradeScripts(dbType);
- // Find the list of scripts to execute for this upgrade
- int firstScript = hiveSchemaVersions.length;
- for (int i=0; i < hiveSchemaVersions.length; i++) {
- if (hiveSchemaVersions[i].startsWith(fromVersion)) {
- firstScript = i;
- }
- }
- if (firstScript == hiveSchemaVersions.length) {
- throw new HiveMetaException("Unknown version specified for upgrade " +
- fromVersion + " Metastore schema may be too old or newer");
- }
-
- for (int i=firstScript; i < hiveSchemaVersions.length; i++) {
- String scriptFile = generateUpgradeFileName(hiveSchemaVersions[i]);
- upgradeScriptList.add(scriptFile);
- }
- return upgradeScriptList;
- }
-
- /***
- * Get the name of the script to initialize the schema for given version
- * @param toVersion Target version. If it's null, then the current server version is used
- * @return
- * @throws HiveMetaException
- */
- @Override
- public String generateInitFileName(String toVersion) throws HiveMetaException {
- if (toVersion == null) {
- toVersion = getHiveSchemaVersion();
- }
- String initScriptName = INIT_FILE_PREFIX + toVersion + "." +
- dbType + SQL_FILE_EXTENSION;
- // check if the file exists
- if (!(new File(getMetaStoreScriptDir() + File.separatorChar +
- initScriptName).exists())) {
- throw new HiveMetaException("Unknown version specified for initialization: " + toVersion);
- }
- return initScriptName;
- }
-
- /**
- * Find the directory of metastore scripts
- * @return
- */
- @Override
- public String getMetaStoreScriptDir() {
- return hiveHome + File.separatorChar +
- "scripts" + File.separatorChar + "metastore" +
- File.separatorChar + "upgrade" + File.separatorChar + dbType;
- }
-
- // format the upgrade script name eg upgrade-x-y-dbType.sql
- private String generateUpgradeFileName(String fileVersion) {
- return UPGRADE_FILE_PREFIX + fileVersion + "." + dbType + SQL_FILE_EXTENSION;
- }
-
- @Override
- public String getPreUpgradeScriptName(int index, String upgradeScriptName) {
- return PRE_UPGRADE_PREFIX + index + "-" + upgradeScriptName;
- }
-
- @Override
- public String getHiveSchemaVersion() {
- String hiveVersion = HiveVersionInfo.getShortVersion();
- return getEquivalentVersion(hiveVersion);
- }
-
- private static String getEquivalentVersion(String hiveVersion) {
- // if there is an equivalent version, return that, else return this version
- String equivalentVersion = EQUIVALENT_VERSIONS.get(hiveVersion);
- if (equivalentVersion != null) {
- return equivalentVersion;
- } else {
- return hiveVersion;
- }
- }
-
- @Override
- public boolean isVersionCompatible(String hiveVersion, String dbVersion) {
- hiveVersion = getEquivalentVersion(hiveVersion);
- dbVersion = getEquivalentVersion(dbVersion);
- if (hiveVersion.equals(dbVersion)) {
- return true;
- }
- String[] hiveVerParts = hiveVersion.split("\\.");
- String[] dbVerParts = dbVersion.split("\\.");
- if (hiveVerParts.length != 3 || dbVerParts.length != 3) {
- // these are non standard version numbers. can't perform the
- // comparison on these, so assume that they are incompatible
- return false;
- }
-
- for (int i = 0; i < dbVerParts.length; i++) {
- int dbVerPart = Integer.parseInt(dbVerParts[i]);
- int hiveVerPart = Integer.parseInt(hiveVerParts[i]);
- if (dbVerPart > hiveVerPart) {
- return true;
- } else if (dbVerPart < hiveVerPart) {
- return false;
- } else {
- continue; // compare next part
- }
- }
-
- return true;
- }
-
- @Override
- public String getMetaStoreSchemaVersion(MetaStoreConnectionInfo connectionInfo)
- throws HiveMetaException {
- String versionQuery;
- boolean needsQuotedIdentifier =
- HiveSchemaHelper.getDbCommandParser(connectionInfo.getDbType()).needsQuotedIdentifier();
- if (needsQuotedIdentifier) {
- versionQuery = "select t.\"SCHEMA_VERSION\" from \"VERSION\" t";
- } else {
- versionQuery = "select t.SCHEMA_VERSION from VERSION t";
- }
- try (Connection metastoreDbConnection =
- HiveSchemaHelper.getConnectionToMetastore(connectionInfo); Statement stmt =
- metastoreDbConnection.createStatement()) {
- ResultSet res = stmt.executeQuery(versionQuery);
- if (!res.next()) {
- throw new HiveMetaException("Could not find version info in metastore VERSION table.");
- }
- String currentSchemaVersion = res.getString(1);
- if (res.next()) {
- throw new HiveMetaException("Multiple versions were found in metastore.");
- }
- return currentSchemaVersion;
- } catch (SQLException e) {
- throw new HiveMetaException("Failed to get schema version, Cause:" + e.getMessage());
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/hive/blob/b0b6db73/metastore/src/java/org/apache/hadoop/hive/metastore/MetaStoreSchemaInfoFactory.java
----------------------------------------------------------------------
diff --git a/metastore/src/java/org/apache/hadoop/hive/metastore/MetaStoreSchemaInfoFactory.java b/metastore/src/java/org/apache/hadoop/hive/metastore/MetaStoreSchemaInfoFactory.java
deleted file mode 100644
index 1133cf2..0000000
--- a/metastore/src/java/org/apache/hadoop/hive/metastore/MetaStoreSchemaInfoFactory.java
+++ /dev/null
@@ -1,65 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.hive.metastore;
-
-import java.lang.reflect.Constructor;
-import java.lang.reflect.InvocationTargetException;
-
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.hive.conf.HiveConf;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-/**
- * Factory class implementation to create instances of IMetaStoreSchemaInfo
- * based on the provided configuration
- */
-public class MetaStoreSchemaInfoFactory {
- public static final Logger LOG = LoggerFactory.getLogger(MetaStoreSchemaInfoFactory.class);
-
- public static IMetaStoreSchemaInfo get(Configuration conf) {
- String hiveHome = System.getenv("HIVE_HOME");
- if (hiveHome == null) {
- LOG.debug("HIVE_HOME is not set. Using current directory instead");
- hiveHome = ".";
- }
- return get(conf, hiveHome, null);
- }
-
- public static IMetaStoreSchemaInfo get(Configuration conf, String hiveHome, String dbType) {
- String className = conf.get(HiveConf.ConfVars.METASTORE_SCHEMA_INFO_CLASS.varname,
- HiveConf.ConfVars.METASTORE_SCHEMA_INFO_CLASS.defaultStrVal);
- Class<?> clasz = null;
- try {
- clasz = conf.getClassByName(className);
- } catch (ClassNotFoundException e) {
- LOG.error("Unable to load class " + className, e);
- throw new IllegalArgumentException(e);
- }
- Constructor<?> constructor = null;
- try {
- constructor = clasz.getConstructor(String.class, String.class);
- constructor.setAccessible(true);
- return (IMetaStoreSchemaInfo) constructor.newInstance(hiveHome, dbType);
- } catch (NoSuchMethodException | InstantiationException | IllegalAccessException
- | IllegalArgumentException | InvocationTargetException e) {
- LOG.error("Unable to create instance of class " + className, e);
- throw new IllegalArgumentException(e);
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/hive/blob/b0b6db73/metastore/src/java/org/apache/hadoop/hive/metastore/MetaStoreThread.java
----------------------------------------------------------------------
diff --git a/metastore/src/java/org/apache/hadoop/hive/metastore/MetaStoreThread.java b/metastore/src/java/org/apache/hadoop/hive/metastore/MetaStoreThread.java
deleted file mode 100644
index b62c45f..0000000
--- a/metastore/src/java/org/apache/hadoop/hive/metastore/MetaStoreThread.java
+++ /dev/null
@@ -1,62 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.hive.metastore;
-
-import org.apache.hadoop.hive.conf.HiveConf;
-import org.apache.hadoop.hive.metastore.api.MetaException;
-
-import java.util.concurrent.atomic.AtomicBoolean;
-
-/**
- * A thread that runs in the metastore, separate from the threads in the thrift service.
- */
-public interface MetaStoreThread {
-
- /**
- * Set the Hive configuration for this thread.
- * @param conf
- */
- void setHiveConf(HiveConf conf);
-
- /**
- * Set the id for this thread.
- * @param threadId
- */
- void setThreadId(int threadId);
-
- /**
- * Initialize the thread. This must not be called until after
- * {@link #setHiveConf(org.apache.hadoop.hive.conf.HiveConf)} and {@link #setThreadId(int)}
- * have been called.
- * @param stop a flag to watch for when to stop. If this value is set to true,
- * the thread will terminate the next time through its main loop.
- * @param looped a flag that is set to true everytime a thread goes through it's main loop.
- * This is purely for testing so that tests can assure themselves that the thread
- * has run through it's loop once. The test can set this value to false. The
- * thread should then assure that the loop has been gone completely through at
- * least once.
- */
- void init(AtomicBoolean stop, AtomicBoolean looped) throws MetaException;
-
- /**
- * Run the thread in the background. This must not be called until
- * {@link MetaStoreThread#init(java.util.concurrent.atomic.AtomicBoolean,java.util.concurrent.atomic.AtomicBoolean)} has
- * been called.
- */
- void start();
-}
http://git-wip-us.apache.org/repos/asf/hive/blob/b0b6db73/metastore/src/java/org/apache/hadoop/hive/metastore/PartitionDropOptions.java
----------------------------------------------------------------------
diff --git a/metastore/src/java/org/apache/hadoop/hive/metastore/PartitionDropOptions.java b/metastore/src/java/org/apache/hadoop/hive/metastore/PartitionDropOptions.java
deleted file mode 100644
index e8ffbd5..0000000
--- a/metastore/src/java/org/apache/hadoop/hive/metastore/PartitionDropOptions.java
+++ /dev/null
@@ -1,54 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.hive.metastore;
-
-/**
- * Class to generalize the switches for dropPartitions().
- */
-public class PartitionDropOptions {
-
- public boolean deleteData = true;
- public boolean ifExists = false;
- public boolean returnResults = true;
- public boolean purgeData = false;
-
- public static PartitionDropOptions instance() { return new PartitionDropOptions(); }
-
- public PartitionDropOptions deleteData(boolean deleteData) {
- this.deleteData = deleteData;
- return this;
- }
-
- public PartitionDropOptions ifExists(boolean ifExists) {
- this.ifExists = ifExists;
- return this;
- }
-
- public PartitionDropOptions returnResults(boolean returnResults) {
- this.returnResults = returnResults;
- return this;
- }
-
- public PartitionDropOptions purgeData(boolean purgeData) {
- this.purgeData = purgeData;
- return this;
- }
-
-} // class PartitionDropSwitches;
-
http://git-wip-us.apache.org/repos/asf/hive/blob/b0b6db73/metastore/src/java/org/apache/hadoop/hive/metastore/PartitionExpressionProxy.java
----------------------------------------------------------------------
diff --git a/metastore/src/java/org/apache/hadoop/hive/metastore/PartitionExpressionProxy.java b/metastore/src/java/org/apache/hadoop/hive/metastore/PartitionExpressionProxy.java
deleted file mode 100644
index af0a6bd..0000000
--- a/metastore/src/java/org/apache/hadoop/hive/metastore/PartitionExpressionProxy.java
+++ /dev/null
@@ -1,73 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.hive.metastore;
-
-import java.util.List;
-
-import org.apache.hadoop.hive.metastore.api.FieldSchema;
-import org.apache.hadoop.hive.metastore.api.FileMetadataExprType;
-import org.apache.hadoop.hive.metastore.api.MetaException;
-import org.apache.hadoop.hive.ql.io.sarg.SearchArgument;
-
-/**
- * The proxy interface that metastore uses for variety of QL operations (metastore can't depend
- * on QL because QL depends on metastore; creating metastore-client module would be a proper way
- * to solve this problem).
- */
-public interface PartitionExpressionProxy {
-
- /**
- * Converts serialized Hive expression into filter in the format suitable for Filter.g.
- * @param expr Serialized expression.
- * @return The filter string.
- */
- public String convertExprToFilter(byte[] expr) throws MetaException;
-
- /**
- * Filters the partition names via serialized Hive expression.
- * @param partColumns Partition columns in the underlying table.
- * @param expr Serialized expression.
- * @param defaultPartitionName Default partition name from job or server configuration.
- * @param partitionNames Partition names; the list is modified in place.
- * @return Whether there were any unknown partitions preserved in the name list.
- */
- boolean filterPartitionsByExpr(List<FieldSchema> partColumns,
- byte[] expr, String defaultPartitionName, List<String> partitionNames) throws MetaException;
-
- /**
- * Determines the file metadata type from input format of the source table or partition.
- * @param inputFormat Input format name.
- * @return The file metadata type.
- */
- FileMetadataExprType getMetadataType(String inputFormat);
-
- /**
- * Gets a separate proxy that can be used to call file-format-specific methods.
- * @param type The file metadata type.
- * @return The proxy.
- */
- FileFormatProxy getFileFormatProxy(FileMetadataExprType type);
-
- /**
- * Creates SARG from serialized representation.
- * @param expr SARG, serialized as Kryo.
- * @return SARG.
- */
- SearchArgument createSarg(byte[] expr);
-}
http://git-wip-us.apache.org/repos/asf/hive/blob/b0b6db73/metastore/src/java/org/apache/hadoop/hive/metastore/TServerSocketKeepAlive.java
----------------------------------------------------------------------
diff --git a/metastore/src/java/org/apache/hadoop/hive/metastore/TServerSocketKeepAlive.java b/metastore/src/java/org/apache/hadoop/hive/metastore/TServerSocketKeepAlive.java
deleted file mode 100644
index 95bd76e..0000000
--- a/metastore/src/java/org/apache/hadoop/hive/metastore/TServerSocketKeepAlive.java
+++ /dev/null
@@ -1,47 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.hive.metastore;
-
-import java.net.SocketException;
-
-import org.apache.thrift.transport.TServerSocket;
-import org.apache.thrift.transport.TSocket;
-import org.apache.thrift.transport.TTransportException;
-
-/**
- * TServerSocketKeepAlive - like TServerSocket, but will enable keepalive for
- * accepted sockets.
- *
- */
-public class TServerSocketKeepAlive extends TServerSocket {
- public TServerSocketKeepAlive(TServerSocket serverSocket) throws TTransportException {
- super(serverSocket.getServerSocket());
- }
-
- @Override
- protected TSocket acceptImpl() throws TTransportException {
- TSocket ts = super.acceptImpl();
- try {
- ts.getSocket().setKeepAlive(true);
- } catch (SocketException e) {
- throw new TTransportException(e);
- }
- return ts;
- }
-}
http://git-wip-us.apache.org/repos/asf/hive/blob/b0b6db73/metastore/src/java/org/apache/hadoop/hive/metastore/TableType.java
----------------------------------------------------------------------
diff --git a/metastore/src/java/org/apache/hadoop/hive/metastore/TableType.java b/metastore/src/java/org/apache/hadoop/hive/metastore/TableType.java
deleted file mode 100644
index e9e16d7..0000000
--- a/metastore/src/java/org/apache/hadoop/hive/metastore/TableType.java
+++ /dev/null
@@ -1,26 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.hive.metastore;
-
-/**
- * Typesafe enum for types of tables described by the metastore.
- */
-public enum TableType {
- MANAGED_TABLE, EXTERNAL_TABLE, VIRTUAL_VIEW, INDEX_TABLE, MATERIALIZED_VIEW
-}
http://git-wip-us.apache.org/repos/asf/hive/blob/b0b6db73/metastore/src/java/org/apache/hadoop/hive/metastore/annotation/NoReconnect.java
----------------------------------------------------------------------
diff --git a/metastore/src/java/org/apache/hadoop/hive/metastore/annotation/NoReconnect.java b/metastore/src/java/org/apache/hadoop/hive/metastore/annotation/NoReconnect.java
deleted file mode 100644
index edf0831..0000000
--- a/metastore/src/java/org/apache/hadoop/hive/metastore/annotation/NoReconnect.java
+++ /dev/null
@@ -1,29 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.hadoop.hive.metastore.annotation;
-
-import java.lang.annotation.ElementType;
-import java.lang.annotation.Retention;
-import java.lang.annotation.RetentionPolicy;
-import java.lang.annotation.Target;
-
-@Target({ElementType.METHOD})
-@Retention(RetentionPolicy.RUNTIME)
-public @interface NoReconnect {
-}
http://git-wip-us.apache.org/repos/asf/hive/blob/b0b6db73/metastore/src/java/org/apache/hadoop/hive/metastore/hooks/JDOConnectionURLHook.java
----------------------------------------------------------------------
diff --git a/metastore/src/java/org/apache/hadoop/hive/metastore/hooks/JDOConnectionURLHook.java b/metastore/src/java/org/apache/hadoop/hive/metastore/hooks/JDOConnectionURLHook.java
deleted file mode 100644
index a1a2fb9..0000000
--- a/metastore/src/java/org/apache/hadoop/hive/metastore/hooks/JDOConnectionURLHook.java
+++ /dev/null
@@ -1,53 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.hive.metastore.hooks;
-
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.hive.common.classification.InterfaceAudience;
-import org.apache.hadoop.hive.common.classification.InterfaceStability;
-
-/**
- * JDOConnectURLHook is used to get the URL that JDO uses to connect to the
- * database that stores the metastore data. Classes implementing this must be
- * thread-safe (for Thrift server).
- */
-@InterfaceAudience.Public
-@InterfaceStability.Stable
-public interface JDOConnectionURLHook {
-
- /**
- * Gets the connection URL to supply to JDO. In addition to initialization,
- * this method will be called after a connection failure for each reconnect
- * attempt.
- *
- * @param conf The configuration used to initialize this instance of the HMS
- * @return the connection URL
- * @throws Exception
- */
- public String getJdoConnectionUrl(Configuration conf)
- throws Exception;
-
- /**
- * Alerts this that the connection URL was bad. Can be used to collect stats,
- * etc.
- *
- * @param url
- */
- public void notifyBadConnectionUrl(String url);
-}
http://git-wip-us.apache.org/repos/asf/hive/blob/b0b6db73/metastore/src/java/org/apache/hadoop/hive/metastore/partition/spec/CompositePartitionSpecProxy.java
----------------------------------------------------------------------
diff --git a/metastore/src/java/org/apache/hadoop/hive/metastore/partition/spec/CompositePartitionSpecProxy.java b/metastore/src/java/org/apache/hadoop/hive/metastore/partition/spec/CompositePartitionSpecProxy.java
deleted file mode 100644
index 7e94e34..0000000
--- a/metastore/src/java/org/apache/hadoop/hive/metastore/partition/spec/CompositePartitionSpecProxy.java
+++ /dev/null
@@ -1,228 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.hive.metastore.partition.spec;
-
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-import org.apache.hadoop.hive.metastore.api.MetaException;
-import org.apache.hadoop.hive.metastore.api.Partition;
-import org.apache.hadoop.hive.metastore.api.PartitionSpec;
-
-import java.util.ArrayList;
-import java.util.List;
-import java.util.Map;
-
-/**
- * Implementation of PartitionSpecProxy that composes a list of PartitionSpecProxy.
- */
-public class CompositePartitionSpecProxy extends PartitionSpecProxy {
-
- private String dbName;
- private String tableName;
- private List<PartitionSpec> partitionSpecs;
- private List<PartitionSpecProxy> partitionSpecProxies;
- private int size = 0;
-
- protected CompositePartitionSpecProxy(List<PartitionSpec> partitionSpecs) {
- this.partitionSpecs = partitionSpecs;
- if (partitionSpecs.isEmpty()) {
- dbName = null;
- tableName = null;
- }
- else {
- dbName = partitionSpecs.get(0).getDbName();
- tableName = partitionSpecs.get(0).getTableName();
- this.partitionSpecProxies = new ArrayList<PartitionSpecProxy>(partitionSpecs.size());
- for (PartitionSpec partitionSpec : partitionSpecs) {
- PartitionSpecProxy partitionSpecProxy = Factory.get(partitionSpec);
- this.partitionSpecProxies.add(partitionSpecProxy);
- size += partitionSpecProxy.size();
- }
- }
- // Assert class-invariant.
- assert isValid() : "Invalid CompositePartitionSpecProxy!";
- }
-
- protected CompositePartitionSpecProxy(String dbName, String tableName, List<PartitionSpec> partitionSpecs) {
- this.dbName = dbName;
- this.tableName = tableName;
- this.partitionSpecs = partitionSpecs;
- this.partitionSpecProxies = new ArrayList<PartitionSpecProxy>(partitionSpecs.size());
- for (PartitionSpec partitionSpec : partitionSpecs) {
- this.partitionSpecProxies.add(PartitionSpecProxy.Factory.get(partitionSpec));
- }
- // Assert class-invariant.
- assert isValid() : "Invalid CompositePartitionSpecProxy!";
- }
-
- private boolean isValid() {
- for (PartitionSpecProxy partitionSpecProxy : partitionSpecProxies) {
- if (partitionSpecProxy instanceof CompositePartitionSpecProxy) {
- return false;
- }
- }
-
- return true;
- }
-
- @Override
- public int size() {
- return size;
- }
-
- /**
- * Iterator to iterate over all Partitions, across all PartitionSpecProxy instances within the Composite.
- */
- public static class Iterator implements PartitionIterator {
-
- private CompositePartitionSpecProxy composite;
- private List<PartitionSpecProxy> partitionSpecProxies;
- private int index = -1; // Index into partitionSpecs.
- private PartitionIterator iterator = null;
-
- public Iterator(CompositePartitionSpecProxy composite) {
- this.composite = composite;
- this.partitionSpecProxies = composite.partitionSpecProxies;
-
- if (this.partitionSpecProxies != null && !this.partitionSpecProxies.isEmpty()) {
- this.index = 0;
- this.iterator = this.partitionSpecProxies.get(this.index).getPartitionIterator();
- }
- }
-
- @Override
- public boolean hasNext() {
-
- if (iterator == null) {
- return false;
- }
-
- if (iterator.hasNext()) {
- return true;
- }
-
- while ( ++index < partitionSpecProxies.size()
- && !(iterator = partitionSpecProxies.get(index).getPartitionIterator()).hasNext());
-
- return index < partitionSpecProxies.size() && iterator.hasNext();
-
- }
-
- @Override
- public Partition next() {
-
- if (iterator.hasNext())
- return iterator.next();
-
- while (++index < partitionSpecProxies.size()
- && !(iterator = partitionSpecProxies.get(index).getPartitionIterator()).hasNext());
-
- return index == partitionSpecProxies.size()? null : iterator.next();
-
- }
-
- @Override
- public void remove() {
- iterator.remove();
- }
-
- @Override
- public Partition getCurrent() {
- return iterator.getCurrent();
- }
-
- @Override
- public String getDbName() {
- return composite.dbName;
- }
-
- @Override
- public String getTableName() {
- return composite.tableName;
- }
-
- @Override
- public Map<String, String> getParameters() {
- return iterator.getParameters();
- }
-
- @Override
- public void setParameters(Map<String, String> parameters) {
- iterator.setParameters(parameters);
- }
-
- @Override
- public String getLocation() {
- return iterator.getLocation();
- }
-
- @Override
- public void putToParameters(String key, String value) {
- iterator.putToParameters(key, value);
- }
-
- @Override
- public void setCreateTime(long time) {
- iterator.setCreateTime(time);
- }
- }
-
- @Override
- public void setDbName(String dbName) {
- this.dbName = dbName;
- for (PartitionSpecProxy partSpecProxy : partitionSpecProxies) {
- partSpecProxy.setDbName(dbName);
- }
- }
-
- @Override
- public void setTableName(String tableName) {
- this.tableName = tableName;
- for (PartitionSpecProxy partSpecProxy : partitionSpecProxies) {
- partSpecProxy.setTableName(tableName);
- }
- }
-
- @Override
- public String getDbName() {
- return dbName;
- }
-
- @Override
- public String getTableName() {
- return tableName;
- }
-
- @Override
- public PartitionIterator getPartitionIterator() {
- return new Iterator(this);
- }
-
- @Override
- public List<PartitionSpec> toPartitionSpec() {
- return partitionSpecs;
- }
-
- @Override
- public void setRootLocation(String rootLocation) throws MetaException {
- for (PartitionSpecProxy partSpecProxy : partitionSpecProxies) {
- partSpecProxy.setRootLocation(rootLocation);
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/hive/blob/b0b6db73/metastore/src/java/org/apache/hadoop/hive/metastore/partition/spec/PartitionListComposingSpecProxy.java
----------------------------------------------------------------------
diff --git a/metastore/src/java/org/apache/hadoop/hive/metastore/partition/spec/PartitionListComposingSpecProxy.java b/metastore/src/java/org/apache/hadoop/hive/metastore/partition/spec/PartitionListComposingSpecProxy.java
deleted file mode 100644
index 154011e..0000000
--- a/metastore/src/java/org/apache/hadoop/hive/metastore/partition/spec/PartitionListComposingSpecProxy.java
+++ /dev/null
@@ -1,171 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.hive.metastore.partition.spec;
-
-import org.apache.hadoop.hive.metastore.api.MetaException;
-import org.apache.hadoop.hive.metastore.api.Partition;
-import org.apache.hadoop.hive.metastore.api.PartitionSpec;
-
-import java.util.Arrays;
-import java.util.List;
-import java.util.Map;
-
-/**
- * PartitionSpecProxy implementation that composes a List of Partitions.
- */
-public class PartitionListComposingSpecProxy extends PartitionSpecProxy {
-
- private PartitionSpec partitionSpec;
-
- protected PartitionListComposingSpecProxy(PartitionSpec partitionSpec) {
- assert partitionSpec.isSetPartitionList()
- : "Partition-list should have been set.";
- this.partitionSpec = partitionSpec;
- }
-
- @Override
- public String getDbName() {
- return partitionSpec.getDbName();
- }
-
- @Override
- public String getTableName() {
- return partitionSpec.getTableName();
- }
-
- @Override
- public PartitionIterator getPartitionIterator() {
- return new Iterator(this);
- }
-
- @Override
- public List<PartitionSpec> toPartitionSpec() {
- return Arrays.asList(partitionSpec);
- }
-
- @Override
- public int size() {
- return partitionSpec.getPartitionList().getPartitionsSize();
- }
-
- @Override
- public void setDbName(String dbName) {
- partitionSpec.setDbName(dbName);
- for (Partition partition : partitionSpec.getPartitionList().getPartitions()) {
- partition.setDbName(dbName);
- }
- }
-
- @Override
- public void setTableName(String tableName) {
- partitionSpec.setTableName(tableName);
- for (Partition partition : partitionSpec.getPartitionList().getPartitions()) {
- partition.setTableName(tableName);
- }
- }
-
- @Override
- public void setRootLocation(String newRootPath) throws MetaException {
-
- String oldRootPath = partitionSpec.getRootPath();
-
- if (oldRootPath == null) {
- throw new MetaException("No common root-path. Can't replace root-path!");
- }
-
- for (Partition partition : partitionSpec.getPartitionList().getPartitions()) {
- String location = partition.getSd().getLocation();
- if (location.startsWith(oldRootPath)) {
- partition.getSd().setLocation(location.replace(oldRootPath, newRootPath));
- }
- else {
- throw new MetaException("Common root-path not found. Can't replace root-path!");
- }
- }
- }
-
- public static class Iterator implements PartitionIterator {
-
- PartitionListComposingSpecProxy partitionSpecProxy;
- List<Partition> partitionList;
- int index;
-
- public Iterator(PartitionListComposingSpecProxy partitionSpecProxy) {
- this.partitionSpecProxy = partitionSpecProxy;
- this.partitionList = partitionSpecProxy.partitionSpec.getPartitionList().getPartitions();
- this.index = 0;
- }
-
- @Override
- public Partition getCurrent() {
- return partitionList.get(index);
- }
-
- @Override
- public String getDbName() {
- return partitionSpecProxy.getDbName();
- }
-
- @Override
- public String getTableName() {
- return partitionSpecProxy.getTableName();
- }
-
- @Override
- public Map<String, String> getParameters() {
- return partitionList.get(index).getParameters();
- }
-
- @Override
- public void setParameters(Map<String, String> parameters) {
- partitionList.get(index).setParameters(parameters);
- }
-
- @Override
- public String getLocation() {
- return partitionList.get(index).getSd().getLocation();
- }
-
- @Override
- public void putToParameters(String key, String value) {
- partitionList.get(index).putToParameters(key, value);
- }
-
- @Override
- public void setCreateTime(long time) {
- partitionList.get(index).setCreateTime((int)time);
- }
-
- @Override
- public boolean hasNext() {
- return index < partitionList.size();
- }
-
- @Override
- public Partition next() {
- return partitionList.get(index++);
- }
-
- @Override
- public void remove() {
- partitionList.remove(index);
- }
- } // class Iterator;
-
-} // class PartitionListComposingSpecProxy;
http://git-wip-us.apache.org/repos/asf/hive/blob/b0b6db73/metastore/src/java/org/apache/hadoop/hive/metastore/partition/spec/PartitionSpecProxy.java
----------------------------------------------------------------------
diff --git a/metastore/src/java/org/apache/hadoop/hive/metastore/partition/spec/PartitionSpecProxy.java b/metastore/src/java/org/apache/hadoop/hive/metastore/partition/spec/PartitionSpecProxy.java
deleted file mode 100644
index fdb0867..0000000
--- a/metastore/src/java/org/apache/hadoop/hive/metastore/partition/spec/PartitionSpecProxy.java
+++ /dev/null
@@ -1,199 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.hive.metastore.partition.spec;
-
-import org.apache.hadoop.hive.metastore.api.MetaException;
-import org.apache.hadoop.hive.metastore.api.Partition;
-import org.apache.hadoop.hive.metastore.api.PartitionSpec;
-
-import java.util.List;
-import java.util.Map;
-
-/**
- * Polymorphic proxy class, equivalent to org.apache.hadoop.hive.metastore.api.PartitionSpec.
- */
-public abstract class PartitionSpecProxy {
-
- /**
- * The number of Partition instances represented by the PartitionSpec.
- * @return Number of partitions.
- */
- public abstract int size();
-
- /**
- * Setter for name of the DB.
- * @param dbName The name of the DB.
- */
- public abstract void setDbName(String dbName);
-
- /**
- * Setter for name of the table.
- * @param tableName The name of the table.
- */
- public abstract void setTableName(String tableName);
-
- /**
- * Getter for name of the DB.
- * @return The name of the DB.
- */
- public abstract String getDbName();
-
- /**
- * Getter for name of the table.
- * @return The name of the table.
- */
- public abstract String getTableName();
-
- /**
- * Iterator to the (virtual) sequence of Partitions represented by the PartitionSpec.
- * @return A PartitionIterator to the beginning of the Partition sequence.
- */
- public abstract PartitionIterator getPartitionIterator();
-
- /**
- * Conversion to a org.apache.hadoop.hive.metastore.api.PartitionSpec sequence.
- * @return A list of org.apache.hadoop.hive.metastore.api.PartitionSpec instances.
- */
- public abstract List<PartitionSpec> toPartitionSpec();
-
- /**
- * Setter for the common root-location for all partitions in the PartitionSet.
- * @param rootLocation The new common root-location.
- * @throws MetaException
- */
- public abstract void setRootLocation(String rootLocation) throws MetaException;
-
- /**
- * Factory to construct PartitionSetProxy instances, from PartitionSets.
- */
- public static class Factory {
-
- /**
- * Factory method. Construct PartitionSpecProxy from raw PartitionSpec.
- * @param partSpec Raw PartitionSpec from the Thrift API.
- * @return PartitionSpecProxy instance.
- */
- public static PartitionSpecProxy get(PartitionSpec partSpec) {
-
- if (partSpec == null) {
- return null;
- }
- else
- if (partSpec.isSetPartitionList()) {
- return new PartitionListComposingSpecProxy(partSpec);
- }
- else
- if (partSpec.isSetSharedSDPartitionSpec()) {
- return new PartitionSpecWithSharedSDProxy(partSpec);
- }
-
- assert false : "Unsupported type of PartitionSpec!";
- return null;
- }
-
- /**
- * Factory method to construct CompositePartitionSpecProxy.
- * @param partitionSpecs List of raw PartitionSpecs.
- * @return A CompositePartitionSpecProxy instance.
- */
- public static PartitionSpecProxy get(List<PartitionSpec> partitionSpecs) {
- return new CompositePartitionSpecProxy(partitionSpecs);
- }
-
- } // class Factory;
-
- /**
- * Iterator to iterate over Partitions corresponding to a PartitionSpec.
- */
- public static interface PartitionIterator extends java.util.Iterator<Partition> {
-
- /**
- * Getter for the Partition "pointed to" by the iterator.
- * Like next(), but without advancing the iterator.
- * @return The "current" partition object.
- */
- public Partition getCurrent();
-
- /**
- * Getter for the name of the DB.
- * @return Name of the DB.
- */
- public String getDbName();
-
- /**
- * Getter for the name of the table.
- * @return Name of the table.
- */
- public String getTableName();
-
- /**
- * Getter for the Partition parameters.
- * @return Key-value map for Partition-level parameters.
- */
- public Map<String, String> getParameters();
-
- /**
- * Setter for Partition parameters.
- * @param parameters Key-value map fo Partition-level parameters.
- */
- public void setParameters(Map<String, String> parameters);
-
- /**
- * Insert an individual parameter to a Partition's parameter-set.
- * @param key
- * @param value
- */
- public void putToParameters(String key, String value);
-
- /**
- * Getter for Partition-location.
- * @return Partition's location.
- */
- public String getLocation();
-
- /**
- * Setter for creation-time of a Partition.
- * @param time Timestamp indicating the time of creation of the Partition.
- */
- public void setCreateTime(long time);
-
- } // class PartitionIterator;
-
- /**
- * Simple wrapper class for pre-constructed Partitions, to expose a PartitionIterator interface,
- * where the iterator-sequence consists of just one Partition.
- */
- public static class SimplePartitionWrapperIterator implements PartitionIterator {
- private Partition partition;
- public SimplePartitionWrapperIterator(Partition partition) {this.partition = partition;}
-
- @Override public Partition getCurrent() { return partition; }
- @Override public String getDbName() { return partition.getDbName(); }
- @Override public String getTableName() { return partition.getTableName(); }
- @Override public Map<String, String> getParameters() { return partition.getParameters(); }
- @Override public void setParameters(Map<String, String> parameters) { partition.setParameters(parameters); }
- @Override public void putToParameters(String key, String value) { partition.putToParameters(key, value);}
- @Override public String getLocation() { return partition.getSd().getLocation(); }
- @Override public void setCreateTime(long time) { partition.setCreateTime((int)time);}
- @Override public boolean hasNext() { return false; } // No next partition.
- @Override public Partition next() { return null; } // No next partition.
- @Override public void remove() {} // Do nothing.
- } // P
-
-} // class PartitionSpecProxy;
http://git-wip-us.apache.org/repos/asf/hive/blob/b0b6db73/metastore/src/java/org/apache/hadoop/hive/metastore/partition/spec/PartitionSpecWithSharedSDProxy.java
----------------------------------------------------------------------
diff --git a/metastore/src/java/org/apache/hadoop/hive/metastore/partition/spec/PartitionSpecWithSharedSDProxy.java b/metastore/src/java/org/apache/hadoop/hive/metastore/partition/spec/PartitionSpecWithSharedSDProxy.java
deleted file mode 100644
index 6a3e147..0000000
--- a/metastore/src/java/org/apache/hadoop/hive/metastore/partition/spec/PartitionSpecWithSharedSDProxy.java
+++ /dev/null
@@ -1,172 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.hive.metastore.partition.spec;
-
-import org.apache.hadoop.hive.metastore.api.MetaException;
-import org.apache.hadoop.hive.metastore.api.Partition;
-import org.apache.hadoop.hive.metastore.api.PartitionSpec;
-import org.apache.hadoop.hive.metastore.api.PartitionSpecWithSharedSD;
-import org.apache.hadoop.hive.metastore.api.PartitionWithoutSD;
-import org.apache.hadoop.hive.metastore.api.StorageDescriptor;
-
-import java.util.Arrays;
-import java.util.List;
-import java.util.Map;
-
-/**
- * Subclass of PartitionSpecProxy that pulls out commonality of
- * StorageDescriptor properties within a Partition-list into a common
- * StorageDescriptor instance.
- */
-public class PartitionSpecWithSharedSDProxy extends PartitionSpecProxy {
-
- private PartitionSpec partitionSpec;
-
- public PartitionSpecWithSharedSDProxy(PartitionSpec partitionSpec) {
- assert partitionSpec.isSetSharedSDPartitionSpec();
- this.partitionSpec = partitionSpec;
- }
-
- @Override
- public int size() {
- return partitionSpec.getSharedSDPartitionSpec().getPartitionsSize();
- }
-
- @Override
- public void setDbName(String dbName) {
- partitionSpec.setDbName(dbName);
- }
-
- @Override
- public void setTableName(String tableName) {
- partitionSpec.setTableName(tableName);
- }
-
- @Override
- public String getDbName() {
- return partitionSpec.getDbName();
- }
-
- @Override
- public String getTableName() {
- return partitionSpec.getTableName();
- }
-
- public PartitionIterator getPartitionIterator() {
- return new Iterator(this);
- }
-
- @Override
- public List<PartitionSpec> toPartitionSpec() {
- return Arrays.asList(partitionSpec);
- }
-
- @Override
- public void setRootLocation(String rootLocation) throws MetaException {
- partitionSpec.setRootPath(rootLocation);
- partitionSpec.getSharedSDPartitionSpec().getSd().setLocation(rootLocation);
- }
-
- /**
- * Iterator implementation to iterate over all Partitions within the PartitionSpecWithSharedSDProxy.
- */
- public static class Iterator implements PartitionIterator {
-
- private PartitionSpecWithSharedSDProxy partitionSpecWithSharedSDProxy;
- private PartitionSpecWithSharedSD pSpec;
- private int index;
-
- Iterator(PartitionSpecWithSharedSDProxy partitionSpecWithSharedSDProxy) {
- this.partitionSpecWithSharedSDProxy = partitionSpecWithSharedSDProxy;
- this.pSpec = this.partitionSpecWithSharedSDProxy.partitionSpec.getSharedSDPartitionSpec();
- this.index = 0;
- }
-
- @Override
- public boolean hasNext() {
- return index < pSpec.getPartitions().size();
- }
-
- @Override
- public Partition next() {
- Partition partition = getCurrent();
- ++index;
- return partition;
- }
-
- @Override
- public void remove() {
- pSpec.getPartitions().remove(index);
- }
-
- @Override
- public Partition getCurrent() {
- PartitionWithoutSD partWithoutSD = pSpec.getPartitions().get(index);
- StorageDescriptor partSD = new StorageDescriptor(pSpec.getSd());
- partSD.setLocation(partSD.getLocation() + partWithoutSD.getRelativePath());
-
- return new Partition(
- partWithoutSD.getValues(),
- partitionSpecWithSharedSDProxy.partitionSpec.getDbName(),
- partitionSpecWithSharedSDProxy.partitionSpec.getTableName(),
- partWithoutSD.getCreateTime(),
- partWithoutSD.getLastAccessTime(),
- partSD,
- partWithoutSD.getParameters()
- );
- }
-
- @Override
- public String getDbName() {
- return partitionSpecWithSharedSDProxy.partitionSpec.getDbName();
- }
-
- @Override
- public String getTableName() {
- return partitionSpecWithSharedSDProxy.partitionSpec.getTableName();
- }
-
- @Override
- public Map<String, String> getParameters() {
- return pSpec.getPartitions().get(index).getParameters();
- }
-
- @Override
- public void setParameters(Map<String, String> parameters) {
- pSpec.getPartitions().get(index).setParameters(parameters);
- }
-
- @Override
- public String getLocation() {
- return pSpec.getSd().getLocation() + pSpec.getPartitions().get(index).getRelativePath();
- }
-
- @Override
- public void putToParameters(String key, String value) {
- pSpec.getPartitions().get(index).putToParameters(key, value);
- }
-
- @Override
- public void setCreateTime(long time) {
- pSpec.getPartitions().get(index).setCreateTime((int)time);
- }
-
- } // static class Iterator;
-
-}
http://git-wip-us.apache.org/repos/asf/hive/blob/b0b6db73/metastore/src/java/org/apache/hadoop/hive/metastore/tools/HiveSchemaHelper.java
----------------------------------------------------------------------
diff --git a/metastore/src/java/org/apache/hadoop/hive/metastore/tools/HiveSchemaHelper.java b/metastore/src/java/org/apache/hadoop/hive/metastore/tools/HiveSchemaHelper.java
deleted file mode 100644
index 620ea5f..0000000
--- a/metastore/src/java/org/apache/hadoop/hive/metastore/tools/HiveSchemaHelper.java
+++ /dev/null
@@ -1,640 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.hive.metastore.tools;
-
-import com.google.common.annotations.VisibleForTesting;
-import com.google.common.collect.Lists;
-import org.apache.hadoop.hive.conf.HiveConf;
-import org.apache.hadoop.hive.metastore.HiveMetaException;
-
-import java.io.BufferedReader;
-import java.io.File;
-import java.io.FileReader;
-import java.io.IOException;
-import java.sql.Connection;
-import java.sql.DriverManager;
-import java.sql.SQLException;
-import java.util.IllegalFormatException;
-import java.util.List;
-
-public class HiveSchemaHelper {
- public static final String DB_DERBY = "derby";
- public static final String DB_HIVE = "hive";
- public static final String DB_MSSQL = "mssql";
- public static final String DB_MYSQL = "mysql";
- public static final String DB_POSTGRACE = "postgres";
- public static final String DB_ORACLE = "oracle";
-
- /***
- * Get JDBC connection to metastore db
- *
- * @param userName metastore connection username
- * @param password metastore connection password
- * @param printInfo print connection parameters
- * @param hiveConf hive config object
- * @return metastore connection object
- * @throws org.apache.hadoop.hive.metastore.HiveMetaException
- */
- public static Connection getConnectionToMetastore(String userName,
- String password, String url, String driver, boolean printInfo,
- HiveConf hiveConf)
- throws HiveMetaException {
- try {
- url = url == null ? getValidConfVar(
- HiveConf.ConfVars.METASTORECONNECTURLKEY, hiveConf) : url;
- driver = driver == null ? getValidConfVar(
- HiveConf.ConfVars.METASTORE_CONNECTION_DRIVER, hiveConf) : driver;
- if (printInfo) {
- System.out.println("Metastore connection URL:\t " + url);
- System.out.println("Metastore Connection Driver :\t " + driver);
- System.out.println("Metastore connection User:\t " + userName);
- }
- if ((userName == null) || userName.isEmpty()) {
- throw new HiveMetaException("UserName empty ");
- }
-
- // load required JDBC driver
- Class.forName(driver);
-
- // Connect using the JDBC URL and user/pass from conf
- return DriverManager.getConnection(url, userName, password);
- } catch (IOException e) {
- throw new HiveMetaException("Failed to get schema version.", e);
- } catch (SQLException e) {
- throw new HiveMetaException("Failed to get schema version.", e);
- } catch (ClassNotFoundException e) {
- throw new HiveMetaException("Failed to load driver", e);
- }
- }
-
- public static Connection getConnectionToMetastore(MetaStoreConnectionInfo info) throws HiveMetaException {
- return getConnectionToMetastore(info.getUsername(), info.getPassword(), info.getUrl(),
- info.getDriver(), info.getPrintInfo(), info.getHiveConf());
- }
-
- public static String getValidConfVar(HiveConf.ConfVars confVar, HiveConf hiveConf)
- throws IOException {
- String confVarStr = hiveConf.get(confVar.varname);
- if (confVarStr == null || confVarStr.isEmpty()) {
- throw new IOException("Empty " + confVar.varname);
- }
- return confVarStr.trim();
- }
-
- public interface NestedScriptParser {
-
- public enum CommandType {
- PARTIAL_STATEMENT,
- TERMINATED_STATEMENT,
- COMMENT
- }
-
- static final String DEFAULT_DELIMITER = ";";
- static final String DEFAULT_QUOTE = "\"";
-
- /**
- * Find the type of given command
- *
- * @param dbCommand
- * @return
- */
- public boolean isPartialCommand(String dbCommand) throws IllegalArgumentException;
-
- /**
- * Parse the DB specific nesting format and extract the inner script name if any
- *
- * @param dbCommand command from parent script
- * @return
- * @throws IllegalFormatException
- */
- public String getScriptName(String dbCommand) throws IllegalArgumentException;
-
- /**
- * Find if the given command is a nested script execution
- *
- * @param dbCommand
- * @return
- */
- public boolean isNestedScript(String dbCommand);
-
- /**
- * Find if the given command should not be passed to DB
- *
- * @param dbCommand
- * @return
- */
- public boolean isNonExecCommand(String dbCommand);
-
- /**
- * Get the SQL statement delimiter
- *
- * @return
- */
- public String getDelimiter();
-
- /**
- * Get the SQL indentifier quotation character
- *
- * @return
- */
- public String getQuoteCharacter();
-
- /**
- * Clear any client specific tags
- *
- * @return
- */
- public String cleanseCommand(String dbCommand);
-
- /**
- * Does the DB required table/column names quoted
- *
- * @return
- */
- public boolean needsQuotedIdentifier();
-
- /**
- * Flatten the nested upgrade script into a buffer
- *
- * @param scriptDir upgrade script directory
- * @param scriptFile upgrade script file
- * @return string of sql commands
- */
- public String buildCommand(String scriptDir, String scriptFile)
- throws IllegalFormatException, IOException;
-
- /**
- * Flatten the nested upgrade script into a buffer
- *
- * @param scriptDir upgrade script directory
- * @param scriptFile upgrade script file
- * @param fixQuotes whether to replace quote characters
- * @return string of sql commands
- */
- public String buildCommand(String scriptDir, String scriptFile, boolean fixQuotes)
- throws IllegalFormatException, IOException;
- }
-
- /***
- * Base implementation of NestedScriptParser
- * abstractCommandParser.
- *
- */
- private static abstract class AbstractCommandParser implements NestedScriptParser {
- private List<String> dbOpts;
- private String msUsername;
- private String msPassword;
- private HiveConf hiveConf;
-
- public AbstractCommandParser(String dbOpts, String msUsername, String msPassword,
- HiveConf hiveConf) {
- setDbOpts(dbOpts);
- this.msUsername = msUsername;
- this.msPassword = msPassword;
- this.hiveConf = hiveConf;
- }
-
- @Override
- public boolean isPartialCommand(String dbCommand) throws IllegalArgumentException{
- if (dbCommand == null || dbCommand.isEmpty()) {
- throw new IllegalArgumentException("invalid command line " + dbCommand);
- }
- dbCommand = dbCommand.trim();
- if (dbCommand.endsWith(getDelimiter()) || isNonExecCommand(dbCommand)) {
- return false;
- } else {
- return true;
- }
- }
-
- @Override
- public boolean isNonExecCommand(String dbCommand) {
- return (dbCommand.startsWith("--") || dbCommand.startsWith("#"));
- }
-
- @Override
- public String getDelimiter() {
- return DEFAULT_DELIMITER;
- }
-
- @Override
- public String getQuoteCharacter() {
- return DEFAULT_QUOTE;
- }
-
-
- @Override
- public String cleanseCommand(String dbCommand) {
- // strip off the delimiter
- if (dbCommand.endsWith(getDelimiter())) {
- dbCommand = dbCommand.substring(0,
- dbCommand.length() - getDelimiter().length());
- }
- return dbCommand;
- }
-
- @Override
- public boolean needsQuotedIdentifier() {
- return false;
- }
-
- @Override
- public String buildCommand(
- String scriptDir, String scriptFile) throws IllegalFormatException, IOException {
- return buildCommand(scriptDir, scriptFile, false);
- }
-
- @Override
- public String buildCommand(
- String scriptDir, String scriptFile, boolean fixQuotes) throws IllegalFormatException, IOException {
- BufferedReader bfReader =
- new BufferedReader(new FileReader(scriptDir + File.separatorChar + scriptFile));
- String currLine;
- StringBuilder sb = new StringBuilder();
- String currentCommand = null;
- while ((currLine = bfReader.readLine()) != null) {
- currLine = currLine.trim();
-
- if (fixQuotes && !getQuoteCharacter().equals(DEFAULT_QUOTE)) {
- currLine = currLine.replace("\\\"", getQuoteCharacter());
- }
-
- if (currLine.isEmpty()) {
- continue; // skip empty lines
- }
-
- if (currentCommand == null) {
- currentCommand = currLine;
- } else {
- currentCommand = currentCommand + " " + currLine;
- }
- if (isPartialCommand(currLine)) {
- // if its a partial line, continue collecting the pieces
- continue;
- }
-
- // if this is a valid executable command then add it to the buffer
- if (!isNonExecCommand(currentCommand)) {
- currentCommand = cleanseCommand(currentCommand);
- if (isNestedScript(currentCommand)) {
- // if this is a nested sql script then flatten it
- String currScript = getScriptName(currentCommand);
- sb.append(buildCommand(scriptDir, currScript));
- } else {
- // Now we have a complete statement, process it
- // write the line to buffer
- sb.append(currentCommand);
- sb.append(System.getProperty("line.separator"));
- }
- }
- currentCommand = null;
- }
- bfReader.close();
- return sb.toString();
- }
-
- private void setDbOpts(String dbOpts) {
- if (dbOpts != null) {
- this.dbOpts = Lists.newArrayList(dbOpts.split(","));
- } else {
- this.dbOpts = Lists.newArrayList();
- }
- }
-
- protected List<String> getDbOpts() {
- return dbOpts;
- }
-
- protected String getMsUsername() {
- return msUsername;
- }
-
- protected String getMsPassword() {
- return msPassword;
- }
-
- protected HiveConf getHiveConf() {
- return hiveConf;
- }
- }
-
- // Derby commandline parser
- public static class DerbyCommandParser extends AbstractCommandParser {
- private static final String DERBY_NESTING_TOKEN = "RUN";
-
- public DerbyCommandParser(String dbOpts, String msUsername, String msPassword,
- HiveConf hiveConf) {
- super(dbOpts, msUsername, msPassword, hiveConf);
- }
-
- @Override
- public String getScriptName(String dbCommand) throws IllegalArgumentException {
-
- if (!isNestedScript(dbCommand)) {
- throw new IllegalArgumentException("Not a script format " + dbCommand);
- }
- String[] tokens = dbCommand.split(" ");
- if (tokens.length != 2) {
- throw new IllegalArgumentException("Couldn't parse line " + dbCommand);
- }
- return tokens[1].replace(";", "").replaceAll("'", "");
- }
-
- @Override
- public boolean isNestedScript(String dbCommand) {
- // Derby script format is RUN '<file>'
- return dbCommand.startsWith(DERBY_NESTING_TOKEN);
- }
- }
-
- // Derby commandline parser
- public static class HiveCommandParser extends AbstractCommandParser {
- private static String HIVE_NESTING_TOKEN = "SOURCE";
- private final NestedScriptParser nestedDbCommandParser;
-
- public HiveCommandParser(String dbOpts, String msUsername, String msPassword,
- HiveConf hiveConf, String metaDbType) {
- super(dbOpts, msUsername, msPassword, hiveConf);
- nestedDbCommandParser = getDbCommandParser(metaDbType);
- }
-
- @Override
- public String getQuoteCharacter() {
- return nestedDbCommandParser.getQuoteCharacter();
- }
-
- @Override
- public String getScriptName(String dbCommand) throws IllegalArgumentException {
-
- if (!isNestedScript(dbCommand)) {
- throw new IllegalArgumentException("Not a script format " + dbCommand);
- }
- String[] tokens = dbCommand.split(" ");
- if (tokens.length != 2) {
- throw new IllegalArgumentException("Couldn't parse line " + dbCommand);
- }
- return tokens[1].replace(";", "");
- }
-
- @Override
- public boolean isNestedScript(String dbCommand) {
- return dbCommand.startsWith(HIVE_NESTING_TOKEN);
- }
- }
-
- // MySQL parser
- public static class MySqlCommandParser extends AbstractCommandParser {
- private static final String MYSQL_NESTING_TOKEN = "SOURCE";
- private static final String DELIMITER_TOKEN = "DELIMITER";
- private String delimiter = DEFAULT_DELIMITER;
-
- public MySqlCommandParser(String dbOpts, String msUsername, String msPassword,
- HiveConf hiveConf) {
- super(dbOpts, msUsername, msPassword, hiveConf);
- }
-
- @Override
- public boolean isPartialCommand(String dbCommand) throws IllegalArgumentException{
- boolean isPartial = super.isPartialCommand(dbCommand);
- // if this is a delimiter directive, reset our delimiter
- if (dbCommand.startsWith(DELIMITER_TOKEN)) {
- String[] tokens = dbCommand.split(" ");
- if (tokens.length != 2) {
- throw new IllegalArgumentException("Couldn't parse line " + dbCommand);
- }
- delimiter = tokens[1];
- }
- return isPartial;
- }
-
- @Override
- public String getScriptName(String dbCommand) throws IllegalArgumentException {
- String[] tokens = dbCommand.split(" ");
- if (tokens.length != 2) {
- throw new IllegalArgumentException("Couldn't parse line " + dbCommand);
- }
- // remove ending ';'
- return tokens[1].replace(";", "");
- }
-
- @Override
- public boolean isNestedScript(String dbCommand) {
- return dbCommand.startsWith(MYSQL_NESTING_TOKEN);
- }
-
- @Override
- public String getDelimiter() {
- return delimiter;
- }
-
- @Override
- public String getQuoteCharacter() {
- return "`";
- }
-
- @Override
- public boolean isNonExecCommand(String dbCommand) {
- return super.isNonExecCommand(dbCommand) ||
- (dbCommand.startsWith("/*") && dbCommand.endsWith("*/")) ||
- dbCommand.startsWith(DELIMITER_TOKEN);
- }
-
- @Override
- public String cleanseCommand(String dbCommand) {
- return super.cleanseCommand(dbCommand).replaceAll("/\\*.*?\\*/[^;]", "");
- }
-
- }
-
- // Postgres specific parser
- public static class PostgresCommandParser extends AbstractCommandParser {
- private static final String POSTGRES_NESTING_TOKEN = "\\i";
- @VisibleForTesting
- public static final String POSTGRES_STANDARD_STRINGS_OPT = "SET standard_conforming_strings";
- @VisibleForTesting
- public static final String POSTGRES_SKIP_STANDARD_STRINGS_DBOPT = "postgres.filter.81";
-
- public PostgresCommandParser(String dbOpts, String msUsername, String msPassword,
- HiveConf hiveConf) {
- super(dbOpts, msUsername, msPassword, hiveConf);
- }
-
- @Override
- public String getScriptName(String dbCommand) throws IllegalArgumentException {
- String[] tokens = dbCommand.split(" ");
- if (tokens.length != 2) {
- throw new IllegalArgumentException("Couldn't parse line " + dbCommand);
- }
- // remove ending ';'
- return tokens[1].replace(";", "");
- }
-
- @Override
- public boolean isNestedScript(String dbCommand) {
- return dbCommand.startsWith(POSTGRES_NESTING_TOKEN);
- }
-
- @Override
- public boolean needsQuotedIdentifier() {
- return true;
- }
-
- @Override
- public boolean isNonExecCommand(String dbCommand) {
- // Skip "standard_conforming_strings" command which is read-only in older
- // Postgres versions like 8.1
- // See: http://www.postgresql.org/docs/8.2/static/release-8-1.html
- if (getDbOpts().contains(POSTGRES_SKIP_STANDARD_STRINGS_DBOPT)) {
- if (dbCommand.startsWith(POSTGRES_STANDARD_STRINGS_OPT)) {
- return true;
- }
- }
- return super.isNonExecCommand(dbCommand);
- }
- }
-
- //Oracle specific parser
- public static class OracleCommandParser extends AbstractCommandParser {
- private static final String ORACLE_NESTING_TOKEN = "@";
-
- public OracleCommandParser(String dbOpts, String msUsername, String msPassword,
- HiveConf hiveConf) {
- super(dbOpts, msUsername, msPassword, hiveConf);
- }
-
- @Override
- public String getScriptName(String dbCommand) throws IllegalArgumentException {
- if (!isNestedScript(dbCommand)) {
- throw new IllegalArgumentException("Not a nested script format " + dbCommand);
- }
- // remove ending ';' and starting '@'
- return dbCommand.replace(";", "").replace(ORACLE_NESTING_TOKEN, "");
- }
-
- @Override
- public boolean isNestedScript(String dbCommand) {
- return dbCommand.startsWith(ORACLE_NESTING_TOKEN);
- }
- }
-
- //MSSQL specific parser
- public static class MSSQLCommandParser extends AbstractCommandParser {
- private static final String MSSQL_NESTING_TOKEN = ":r";
-
- public MSSQLCommandParser(String dbOpts, String msUsername, String msPassword,
- HiveConf hiveConf) {
- super(dbOpts, msUsername, msPassword, hiveConf);
- }
-
- @Override
- public String getScriptName(String dbCommand) throws IllegalArgumentException {
- String[] tokens = dbCommand.split(" ");
- if (tokens.length != 2) {
- throw new IllegalArgumentException("Couldn't parse line " + dbCommand);
- }
- return tokens[1];
- }
-
- @Override
- public boolean isNestedScript(String dbCommand) {
- return dbCommand.startsWith(MSSQL_NESTING_TOKEN);
- }
- }
-
- public static NestedScriptParser getDbCommandParser(String dbName) {
- return getDbCommandParser(dbName, null);
- }
-
- public static NestedScriptParser getDbCommandParser(String dbName, String metaDbName) {
- return getDbCommandParser(dbName, null, null, null, null, metaDbName);
- }
-
- public static NestedScriptParser getDbCommandParser(String dbName,
- String dbOpts, String msUsername, String msPassword,
- HiveConf hiveConf, String metaDbType) {
- if (dbName.equalsIgnoreCase(DB_DERBY)) {
- return new DerbyCommandParser(dbOpts, msUsername, msPassword, hiveConf);
- } else if (dbName.equalsIgnoreCase(DB_HIVE)) {
- return new HiveCommandParser(dbOpts, msUsername, msPassword, hiveConf, metaDbType);
- } else if (dbName.equalsIgnoreCase(DB_MSSQL)) {
- return new MSSQLCommandParser(dbOpts, msUsername, msPassword, hiveConf);
- } else if (dbName.equalsIgnoreCase(DB_MYSQL)) {
- return new MySqlCommandParser(dbOpts, msUsername, msPassword, hiveConf);
- } else if (dbName.equalsIgnoreCase(DB_POSTGRACE)) {
- return new PostgresCommandParser(dbOpts, msUsername, msPassword, hiveConf);
- } else if (dbName.equalsIgnoreCase(DB_ORACLE)) {
- return new OracleCommandParser(dbOpts, msUsername, msPassword, hiveConf);
- } else {
- throw new IllegalArgumentException("Unknown dbType " + dbName);
- }
- }
-
- public static class MetaStoreConnectionInfo {
- private final String userName;
- private final String password;
- private final String url;
- private final String driver;
- private final boolean printInfo;
- private final HiveConf hiveConf;
- private final String dbType;
-
- public MetaStoreConnectionInfo(String userName, String password, String url, String driver,
- boolean printInfo, HiveConf hiveConf, String dbType) {
- super();
- this.userName = userName;
- this.password = password;
- this.url = url;
- this.driver = driver;
- this.printInfo = printInfo;
- this.hiveConf = hiveConf;
- this.dbType = dbType;
- }
-
- public String getPassword() {
- return password;
- }
-
- public String getUrl() {
- return url;
- }
-
- public String getDriver() {
- return driver;
- }
-
- public boolean isPrintInfo() {
- return printInfo;
- }
-
- public HiveConf getHiveConf() {
- return hiveConf;
- }
-
- public String getUsername() {
- return userName;
- }
-
- public boolean getPrintInfo() {
- return printInfo;
- }
-
- public String getDbType() {
- return dbType;
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/hive/blob/b0b6db73/metastore/src/protobuf/org/apache/hadoop/hive/metastore/metastore.proto
----------------------------------------------------------------------
diff --git a/metastore/src/protobuf/org/apache/hadoop/hive/metastore/metastore.proto b/metastore/src/protobuf/org/apache/hadoop/hive/metastore/metastore.proto
deleted file mode 100644
index 29b99b4..0000000
--- a/metastore/src/protobuf/org/apache/hadoop/hive/metastore/metastore.proto
+++ /dev/null
@@ -1,29 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.hive.metastore;
-
-
-message SplitInfo {
- required int64 offset = 1;
- required int64 length = 2;
- required int32 index = 3;
-}
-
-message SplitInfos {
- repeated SplitInfo infos = 1;
-}
[2/7] hive git commit: HIVE-17488 Move first set of classes to
standalone metastore. This closes #244. (Alan Gates,
reviewed by Owen O'Malley)
Posted by ga...@apache.org.
http://git-wip-us.apache.org/repos/asf/hive/blob/b0b6db73/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/partition/spec/CompositePartitionSpecProxy.java
----------------------------------------------------------------------
diff --git a/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/partition/spec/CompositePartitionSpecProxy.java b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/partition/spec/CompositePartitionSpecProxy.java
new file mode 100644
index 0000000..e34335d
--- /dev/null
+++ b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/partition/spec/CompositePartitionSpecProxy.java
@@ -0,0 +1,226 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hive.metastore.partition.spec;
+
+import org.apache.hadoop.hive.metastore.api.MetaException;
+import org.apache.hadoop.hive.metastore.api.Partition;
+import org.apache.hadoop.hive.metastore.api.PartitionSpec;
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Map;
+
+/**
+ * Implementation of PartitionSpecProxy that composes a list of PartitionSpecProxy.
+ */
+public class CompositePartitionSpecProxy extends PartitionSpecProxy {
+
+ private String dbName;
+ private String tableName;
+ private List<PartitionSpec> partitionSpecs;
+ private List<PartitionSpecProxy> partitionSpecProxies;
+ private int size = 0;
+
+ protected CompositePartitionSpecProxy(List<PartitionSpec> partitionSpecs) {
+ this.partitionSpecs = partitionSpecs;
+ if (partitionSpecs.isEmpty()) {
+ dbName = null;
+ tableName = null;
+ }
+ else {
+ dbName = partitionSpecs.get(0).getDbName();
+ tableName = partitionSpecs.get(0).getTableName();
+ this.partitionSpecProxies = new ArrayList<>(partitionSpecs.size());
+ for (PartitionSpec partitionSpec : partitionSpecs) {
+ PartitionSpecProxy partitionSpecProxy = Factory.get(partitionSpec);
+ this.partitionSpecProxies.add(partitionSpecProxy);
+ size += partitionSpecProxy.size();
+ }
+ }
+ // Assert class-invariant.
+ assert isValid() : "Invalid CompositePartitionSpecProxy!";
+ }
+
+ protected CompositePartitionSpecProxy(String dbName, String tableName, List<PartitionSpec> partitionSpecs) {
+ this.dbName = dbName;
+ this.tableName = tableName;
+ this.partitionSpecs = partitionSpecs;
+ this.partitionSpecProxies = new ArrayList<>(partitionSpecs.size());
+ for (PartitionSpec partitionSpec : partitionSpecs) {
+ this.partitionSpecProxies.add(PartitionSpecProxy.Factory.get(partitionSpec));
+ }
+ // Assert class-invariant.
+ assert isValid() : "Invalid CompositePartitionSpecProxy!";
+ }
+
+ private boolean isValid() {
+ for (PartitionSpecProxy partitionSpecProxy : partitionSpecProxies) {
+ if (partitionSpecProxy instanceof CompositePartitionSpecProxy) {
+ return false;
+ }
+ }
+
+ return true;
+ }
+
+ @Override
+ public int size() {
+ return size;
+ }
+
+ /**
+ * Iterator to iterate over all Partitions, across all PartitionSpecProxy instances within the Composite.
+ */
+ public static class Iterator implements PartitionIterator {
+
+ private CompositePartitionSpecProxy composite;
+ private List<PartitionSpecProxy> partitionSpecProxies;
+ private int index = -1; // Index into partitionSpecs.
+ private PartitionIterator iterator = null;
+
+ public Iterator(CompositePartitionSpecProxy composite) {
+ this.composite = composite;
+ this.partitionSpecProxies = composite.partitionSpecProxies;
+
+ if (this.partitionSpecProxies != null && !this.partitionSpecProxies.isEmpty()) {
+ this.index = 0;
+ this.iterator = this.partitionSpecProxies.get(this.index).getPartitionIterator();
+ }
+ }
+
+ @Override
+ public boolean hasNext() {
+
+ if (iterator == null) {
+ return false;
+ }
+
+ if (iterator.hasNext()) {
+ return true;
+ }
+
+ while ( ++index < partitionSpecProxies.size()
+ && !(iterator = partitionSpecProxies.get(index).getPartitionIterator()).hasNext());
+
+ return index < partitionSpecProxies.size() && iterator.hasNext();
+
+ }
+
+ @Override
+ public Partition next() {
+
+ if (iterator.hasNext())
+ return iterator.next();
+
+ while (++index < partitionSpecProxies.size()
+ && !(iterator = partitionSpecProxies.get(index).getPartitionIterator()).hasNext());
+
+ return index == partitionSpecProxies.size()? null : iterator.next();
+
+ }
+
+ @Override
+ public void remove() {
+ iterator.remove();
+ }
+
+ @Override
+ public Partition getCurrent() {
+ return iterator.getCurrent();
+ }
+
+ @Override
+ public String getDbName() {
+ return composite.dbName;
+ }
+
+ @Override
+ public String getTableName() {
+ return composite.tableName;
+ }
+
+ @Override
+ public Map<String, String> getParameters() {
+ return iterator.getParameters();
+ }
+
+ @Override
+ public void setParameters(Map<String, String> parameters) {
+ iterator.setParameters(parameters);
+ }
+
+ @Override
+ public String getLocation() {
+ return iterator.getLocation();
+ }
+
+ @Override
+ public void putToParameters(String key, String value) {
+ iterator.putToParameters(key, value);
+ }
+
+ @Override
+ public void setCreateTime(long time) {
+ iterator.setCreateTime(time);
+ }
+ }
+
+ @Override
+ public void setDbName(String dbName) {
+ this.dbName = dbName;
+ for (PartitionSpecProxy partSpecProxy : partitionSpecProxies) {
+ partSpecProxy.setDbName(dbName);
+ }
+ }
+
+ @Override
+ public void setTableName(String tableName) {
+ this.tableName = tableName;
+ for (PartitionSpecProxy partSpecProxy : partitionSpecProxies) {
+ partSpecProxy.setTableName(tableName);
+ }
+ }
+
+ @Override
+ public String getDbName() {
+ return dbName;
+ }
+
+ @Override
+ public String getTableName() {
+ return tableName;
+ }
+
+ @Override
+ public PartitionIterator getPartitionIterator() {
+ return new Iterator(this);
+ }
+
+ @Override
+ public List<PartitionSpec> toPartitionSpec() {
+ return partitionSpecs;
+ }
+
+ @Override
+ public void setRootLocation(String rootLocation) throws MetaException {
+ for (PartitionSpecProxy partSpecProxy : partitionSpecProxies) {
+ partSpecProxy.setRootLocation(rootLocation);
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/hive/blob/b0b6db73/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/partition/spec/PartitionListComposingSpecProxy.java
----------------------------------------------------------------------
diff --git a/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/partition/spec/PartitionListComposingSpecProxy.java b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/partition/spec/PartitionListComposingSpecProxy.java
new file mode 100644
index 0000000..7b0550b
--- /dev/null
+++ b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/partition/spec/PartitionListComposingSpecProxy.java
@@ -0,0 +1,171 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hive.metastore.partition.spec;
+
+import org.apache.hadoop.hive.metastore.api.MetaException;
+import org.apache.hadoop.hive.metastore.api.Partition;
+import org.apache.hadoop.hive.metastore.api.PartitionSpec;
+
+import java.util.Arrays;
+import java.util.List;
+import java.util.Map;
+
+/**
+ * PartitionSpecProxy implementation that composes a List of Partitions.
+ */
+public class PartitionListComposingSpecProxy extends PartitionSpecProxy {
+
+ private PartitionSpec partitionSpec;
+
+ protected PartitionListComposingSpecProxy(PartitionSpec partitionSpec) {
+ assert partitionSpec.isSetPartitionList()
+ : "Partition-list should have been set.";
+ this.partitionSpec = partitionSpec;
+ }
+
+ @Override
+ public String getDbName() {
+ return partitionSpec.getDbName();
+ }
+
+ @Override
+ public String getTableName() {
+ return partitionSpec.getTableName();
+ }
+
+ @Override
+ public PartitionIterator getPartitionIterator() {
+ return new Iterator(this);
+ }
+
+ @Override
+ public List<PartitionSpec> toPartitionSpec() {
+ return Arrays.asList(partitionSpec);
+ }
+
+ @Override
+ public int size() {
+ return partitionSpec.getPartitionList().getPartitionsSize();
+ }
+
+ @Override
+ public void setDbName(String dbName) {
+ partitionSpec.setDbName(dbName);
+ for (Partition partition : partitionSpec.getPartitionList().getPartitions()) {
+ partition.setDbName(dbName);
+ }
+ }
+
+ @Override
+ public void setTableName(String tableName) {
+ partitionSpec.setTableName(tableName);
+ for (Partition partition : partitionSpec.getPartitionList().getPartitions()) {
+ partition.setTableName(tableName);
+ }
+ }
+
+ @Override
+ public void setRootLocation(String newRootPath) throws MetaException {
+
+ String oldRootPath = partitionSpec.getRootPath();
+
+ if (oldRootPath == null) {
+ throw new MetaException("No common root-path. Can't replace root-path!");
+ }
+
+ for (Partition partition : partitionSpec.getPartitionList().getPartitions()) {
+ String location = partition.getSd().getLocation();
+ if (location.startsWith(oldRootPath)) {
+ partition.getSd().setLocation(location.replace(oldRootPath, newRootPath));
+ }
+ else {
+ throw new MetaException("Common root-path not found. Can't replace root-path!");
+ }
+ }
+ }
+
+ public static class Iterator implements PartitionIterator {
+
+ PartitionListComposingSpecProxy partitionSpecProxy;
+ List<Partition> partitionList;
+ int index;
+
+ public Iterator(PartitionListComposingSpecProxy partitionSpecProxy) {
+ this.partitionSpecProxy = partitionSpecProxy;
+ this.partitionList = partitionSpecProxy.partitionSpec.getPartitionList().getPartitions();
+ this.index = 0;
+ }
+
+ @Override
+ public Partition getCurrent() {
+ return partitionList.get(index);
+ }
+
+ @Override
+ public String getDbName() {
+ return partitionSpecProxy.getDbName();
+ }
+
+ @Override
+ public String getTableName() {
+ return partitionSpecProxy.getTableName();
+ }
+
+ @Override
+ public Map<String, String> getParameters() {
+ return partitionList.get(index).getParameters();
+ }
+
+ @Override
+ public void setParameters(Map<String, String> parameters) {
+ partitionList.get(index).setParameters(parameters);
+ }
+
+ @Override
+ public String getLocation() {
+ return partitionList.get(index).getSd().getLocation();
+ }
+
+ @Override
+ public void putToParameters(String key, String value) {
+ partitionList.get(index).putToParameters(key, value);
+ }
+
+ @Override
+ public void setCreateTime(long time) {
+ partitionList.get(index).setCreateTime((int)time);
+ }
+
+ @Override
+ public boolean hasNext() {
+ return index < partitionList.size();
+ }
+
+ @Override
+ public Partition next() {
+ return partitionList.get(index++);
+ }
+
+ @Override
+ public void remove() {
+ partitionList.remove(index);
+ }
+ } // class Iterator;
+
+} // class PartitionListComposingSpecProxy;
http://git-wip-us.apache.org/repos/asf/hive/blob/b0b6db73/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/partition/spec/PartitionSpecProxy.java
----------------------------------------------------------------------
diff --git a/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/partition/spec/PartitionSpecProxy.java b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/partition/spec/PartitionSpecProxy.java
new file mode 100644
index 0000000..2640a24
--- /dev/null
+++ b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/partition/spec/PartitionSpecProxy.java
@@ -0,0 +1,199 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hive.metastore.partition.spec;
+
+import org.apache.hadoop.hive.metastore.api.MetaException;
+import org.apache.hadoop.hive.metastore.api.Partition;
+import org.apache.hadoop.hive.metastore.api.PartitionSpec;
+
+import java.util.List;
+import java.util.Map;
+
+/**
+ * Polymorphic proxy class, equivalent to org.apache.hadoop.hive.metastore.api.PartitionSpec.
+ */
+public abstract class PartitionSpecProxy {
+
+ /**
+ * The number of Partition instances represented by the PartitionSpec.
+ * @return Number of partitions.
+ */
+ public abstract int size();
+
+ /**
+ * Setter for name of the DB.
+ * @param dbName The name of the DB.
+ */
+ public abstract void setDbName(String dbName);
+
+ /**
+ * Setter for name of the table.
+ * @param tableName The name of the table.
+ */
+ public abstract void setTableName(String tableName);
+
+ /**
+ * Getter for name of the DB.
+ * @return The name of the DB.
+ */
+ public abstract String getDbName();
+
+ /**
+ * Getter for name of the table.
+ * @return The name of the table.
+ */
+ public abstract String getTableName();
+
+ /**
+ * Iterator to the (virtual) sequence of Partitions represented by the PartitionSpec.
+ * @return A PartitionIterator to the beginning of the Partition sequence.
+ */
+ public abstract PartitionIterator getPartitionIterator();
+
+ /**
+ * Conversion to a org.apache.hadoop.hive.metastore.api.PartitionSpec sequence.
+ * @return A list of org.apache.hadoop.hive.metastore.api.PartitionSpec instances.
+ */
+ public abstract List<PartitionSpec> toPartitionSpec();
+
+ /**
+ * Setter for the common root-location for all partitions in the PartitionSet.
+ * @param rootLocation The new common root-location.
+ * @throws MetaException
+ */
+ public abstract void setRootLocation(String rootLocation) throws MetaException;
+
+ /**
+ * Factory to construct PartitionSetProxy instances, from PartitionSets.
+ */
+ public static class Factory {
+
+ /**
+ * Factory method. Construct PartitionSpecProxy from raw PartitionSpec.
+ * @param partSpec Raw PartitionSpec from the Thrift API.
+ * @return PartitionSpecProxy instance.
+ */
+ public static PartitionSpecProxy get(PartitionSpec partSpec) {
+
+ if (partSpec == null) {
+ return null;
+ }
+ else
+ if (partSpec.isSetPartitionList()) {
+ return new PartitionListComposingSpecProxy(partSpec);
+ }
+ else
+ if (partSpec.isSetSharedSDPartitionSpec()) {
+ return new PartitionSpecWithSharedSDProxy(partSpec);
+ }
+
+ assert false : "Unsupported type of PartitionSpec!";
+ return null;
+ }
+
+ /**
+ * Factory method to construct CompositePartitionSpecProxy.
+ * @param partitionSpecs List of raw PartitionSpecs.
+ * @return A CompositePartitionSpecProxy instance.
+ */
+ public static PartitionSpecProxy get(List<PartitionSpec> partitionSpecs) {
+ return new CompositePartitionSpecProxy(partitionSpecs);
+ }
+
+ } // class Factory;
+
+ /**
+ * Iterator to iterate over Partitions corresponding to a PartitionSpec.
+ */
+ public interface PartitionIterator extends java.util.Iterator<Partition> {
+
+ /**
+ * Getter for the Partition "pointed to" by the iterator.
+ * Like next(), but without advancing the iterator.
+ * @return The "current" partition object.
+ */
+ Partition getCurrent();
+
+ /**
+ * Getter for the name of the DB.
+ * @return Name of the DB.
+ */
+ String getDbName();
+
+ /**
+ * Getter for the name of the table.
+ * @return Name of the table.
+ */
+ String getTableName();
+
+ /**
+ * Getter for the Partition parameters.
+ * @return Key-value map for Partition-level parameters.
+ */
+ Map<String, String> getParameters();
+
+ /**
+ * Setter for Partition parameters.
+ * @param parameters Key-value map fo Partition-level parameters.
+ */
+ void setParameters(Map<String, String> parameters);
+
+ /**
+ * Insert an individual parameter to a Partition's parameter-set.
+ * @param key
+ * @param value
+ */
+ void putToParameters(String key, String value);
+
+ /**
+ * Getter for Partition-location.
+ * @return Partition's location.
+ */
+ String getLocation();
+
+ /**
+ * Setter for creation-time of a Partition.
+ * @param time Timestamp indicating the time of creation of the Partition.
+ */
+ void setCreateTime(long time);
+
+ } // class PartitionIterator;
+
+ /**
+ * Simple wrapper class for pre-constructed Partitions, to expose a PartitionIterator interface,
+ * where the iterator-sequence consists of just one Partition.
+ */
+ public static class SimplePartitionWrapperIterator implements PartitionIterator {
+ private Partition partition;
+ public SimplePartitionWrapperIterator(Partition partition) {this.partition = partition;}
+
+ @Override public Partition getCurrent() { return partition; }
+ @Override public String getDbName() { return partition.getDbName(); }
+ @Override public String getTableName() { return partition.getTableName(); }
+ @Override public Map<String, String> getParameters() { return partition.getParameters(); }
+ @Override public void setParameters(Map<String, String> parameters) { partition.setParameters(parameters); }
+ @Override public void putToParameters(String key, String value) { partition.putToParameters(key, value);}
+ @Override public String getLocation() { return partition.getSd().getLocation(); }
+ @Override public void setCreateTime(long time) { partition.setCreateTime((int)time);}
+ @Override public boolean hasNext() { return false; } // No next partition.
+ @Override public Partition next() { return null; } // No next partition.
+ @Override public void remove() {} // Do nothing.
+ } // P
+
+} // class PartitionSpecProxy;
http://git-wip-us.apache.org/repos/asf/hive/blob/b0b6db73/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/partition/spec/PartitionSpecWithSharedSDProxy.java
----------------------------------------------------------------------
diff --git a/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/partition/spec/PartitionSpecWithSharedSDProxy.java b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/partition/spec/PartitionSpecWithSharedSDProxy.java
new file mode 100644
index 0000000..36b05f7
--- /dev/null
+++ b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/partition/spec/PartitionSpecWithSharedSDProxy.java
@@ -0,0 +1,172 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hive.metastore.partition.spec;
+
+import org.apache.hadoop.hive.metastore.api.MetaException;
+import org.apache.hadoop.hive.metastore.api.Partition;
+import org.apache.hadoop.hive.metastore.api.PartitionSpec;
+import org.apache.hadoop.hive.metastore.api.PartitionSpecWithSharedSD;
+import org.apache.hadoop.hive.metastore.api.PartitionWithoutSD;
+import org.apache.hadoop.hive.metastore.api.StorageDescriptor;
+
+import java.util.Arrays;
+import java.util.List;
+import java.util.Map;
+
+/**
+ * Subclass of PartitionSpecProxy that pulls out commonality of
+ * StorageDescriptor properties within a Partition-list into a common
+ * StorageDescriptor instance.
+ */
+public class PartitionSpecWithSharedSDProxy extends PartitionSpecProxy {
+
+ private PartitionSpec partitionSpec;
+
+ public PartitionSpecWithSharedSDProxy(PartitionSpec partitionSpec) {
+ assert partitionSpec.isSetSharedSDPartitionSpec();
+ this.partitionSpec = partitionSpec;
+ }
+
+ @Override
+ public int size() {
+ return partitionSpec.getSharedSDPartitionSpec().getPartitionsSize();
+ }
+
+ @Override
+ public void setDbName(String dbName) {
+ partitionSpec.setDbName(dbName);
+ }
+
+ @Override
+ public void setTableName(String tableName) {
+ partitionSpec.setTableName(tableName);
+ }
+
+ @Override
+ public String getDbName() {
+ return partitionSpec.getDbName();
+ }
+
+ @Override
+ public String getTableName() {
+ return partitionSpec.getTableName();
+ }
+
+ public PartitionIterator getPartitionIterator() {
+ return new Iterator(this);
+ }
+
+ @Override
+ public List<PartitionSpec> toPartitionSpec() {
+ return Arrays.asList(partitionSpec);
+ }
+
+ @Override
+ public void setRootLocation(String rootLocation) throws MetaException {
+ partitionSpec.setRootPath(rootLocation);
+ partitionSpec.getSharedSDPartitionSpec().getSd().setLocation(rootLocation);
+ }
+
+ /**
+ * Iterator implementation to iterate over all Partitions within the PartitionSpecWithSharedSDProxy.
+ */
+ public static class Iterator implements PartitionIterator {
+
+ private PartitionSpecWithSharedSDProxy partitionSpecWithSharedSDProxy;
+ private PartitionSpecWithSharedSD pSpec;
+ private int index;
+
+ Iterator(PartitionSpecWithSharedSDProxy partitionSpecWithSharedSDProxy) {
+ this.partitionSpecWithSharedSDProxy = partitionSpecWithSharedSDProxy;
+ this.pSpec = this.partitionSpecWithSharedSDProxy.partitionSpec.getSharedSDPartitionSpec();
+ this.index = 0;
+ }
+
+ @Override
+ public boolean hasNext() {
+ return index < pSpec.getPartitions().size();
+ }
+
+ @Override
+ public Partition next() {
+ Partition partition = getCurrent();
+ ++index;
+ return partition;
+ }
+
+ @Override
+ public void remove() {
+ pSpec.getPartitions().remove(index);
+ }
+
+ @Override
+ public Partition getCurrent() {
+ PartitionWithoutSD partWithoutSD = pSpec.getPartitions().get(index);
+ StorageDescriptor partSD = new StorageDescriptor(pSpec.getSd());
+ partSD.setLocation(partSD.getLocation() + partWithoutSD.getRelativePath());
+
+ return new Partition(
+ partWithoutSD.getValues(),
+ partitionSpecWithSharedSDProxy.partitionSpec.getDbName(),
+ partitionSpecWithSharedSDProxy.partitionSpec.getTableName(),
+ partWithoutSD.getCreateTime(),
+ partWithoutSD.getLastAccessTime(),
+ partSD,
+ partWithoutSD.getParameters()
+ );
+ }
+
+ @Override
+ public String getDbName() {
+ return partitionSpecWithSharedSDProxy.partitionSpec.getDbName();
+ }
+
+ @Override
+ public String getTableName() {
+ return partitionSpecWithSharedSDProxy.partitionSpec.getTableName();
+ }
+
+ @Override
+ public Map<String, String> getParameters() {
+ return pSpec.getPartitions().get(index).getParameters();
+ }
+
+ @Override
+ public void setParameters(Map<String, String> parameters) {
+ pSpec.getPartitions().get(index).setParameters(parameters);
+ }
+
+ @Override
+ public String getLocation() {
+ return pSpec.getSd().getLocation() + pSpec.getPartitions().get(index).getRelativePath();
+ }
+
+ @Override
+ public void putToParameters(String key, String value) {
+ pSpec.getPartitions().get(index).putToParameters(key, value);
+ }
+
+ @Override
+ public void setCreateTime(long time) {
+ pSpec.getPartitions().get(index).setCreateTime((int)time);
+ }
+
+ } // static class Iterator;
+
+}
http://git-wip-us.apache.org/repos/asf/hive/blob/b0b6db73/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/tools/HiveSchemaHelper.java
----------------------------------------------------------------------
diff --git a/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/tools/HiveSchemaHelper.java b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/tools/HiveSchemaHelper.java
new file mode 100644
index 0000000..4ba11b8
--- /dev/null
+++ b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/tools/HiveSchemaHelper.java
@@ -0,0 +1,641 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hive.metastore.tools;
+
+import com.google.common.annotations.VisibleForTesting;
+import com.google.common.collect.Lists;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hive.metastore.HiveMetaException;
+import org.apache.hadoop.hive.metastore.conf.MetastoreConf;
+
+import java.io.BufferedReader;
+import java.io.File;
+import java.io.FileReader;
+import java.io.IOException;
+import java.sql.Connection;
+import java.sql.DriverManager;
+import java.sql.SQLException;
+import java.util.IllegalFormatException;
+import java.util.List;
+
+public class HiveSchemaHelper {
+ public static final String DB_DERBY = "derby";
+ public static final String DB_HIVE = "hive";
+ public static final String DB_MSSQL = "mssql";
+ public static final String DB_MYSQL = "mysql";
+ public static final String DB_POSTGRACE = "postgres";
+ public static final String DB_ORACLE = "oracle";
+
+ /***
+ * Get JDBC connection to metastore db
+ *
+ * @param userName metastore connection username
+ * @param password metastore connection password
+ * @param printInfo print connection parameters
+ * @param conf hive config object
+ * @return metastore connection object
+ * @throws org.apache.hadoop.hive.metastore.HiveMetaException
+ */
+ public static Connection getConnectionToMetastore(String userName,
+ String password, String url, String driver, boolean printInfo,
+ Configuration conf)
+ throws HiveMetaException {
+ try {
+ url = url == null ? getValidConfVar(
+ MetastoreConf.ConfVars.CONNECTURLKEY, conf) : url;
+ driver = driver == null ? getValidConfVar(
+ MetastoreConf.ConfVars.CONNECTION_DRIVER, conf) : driver;
+ if (printInfo) {
+ System.out.println("Metastore connection URL:\t " + url);
+ System.out.println("Metastore Connection Driver :\t " + driver);
+ System.out.println("Metastore connection User:\t " + userName);
+ }
+ if ((userName == null) || userName.isEmpty()) {
+ throw new HiveMetaException("UserName empty ");
+ }
+
+ // load required JDBC driver
+ Class.forName(driver);
+
+ // Connect using the JDBC URL and user/pass from conf
+ return DriverManager.getConnection(url, userName, password);
+ } catch (IOException e) {
+ throw new HiveMetaException("Failed to get schema version.", e);
+ } catch (SQLException e) {
+ throw new HiveMetaException("Failed to get schema version.", e);
+ } catch (ClassNotFoundException e) {
+ throw new HiveMetaException("Failed to load driver", e);
+ }
+ }
+
+ public static Connection getConnectionToMetastore(MetaStoreConnectionInfo info) throws HiveMetaException {
+ return getConnectionToMetastore(info.getUsername(), info.getPassword(), info.getUrl(),
+ info.getDriver(), info.getPrintInfo(), info.getConf());
+ }
+
+ public static String getValidConfVar(MetastoreConf.ConfVars confVar, Configuration conf)
+ throws IOException {
+ String confVarStr = conf.get(confVar.varname);
+ if (confVarStr == null || confVarStr.isEmpty()) {
+ throw new IOException("Empty " + confVar.varname);
+ }
+ return confVarStr.trim();
+ }
+
+ public interface NestedScriptParser {
+
+ enum CommandType {
+ PARTIAL_STATEMENT,
+ TERMINATED_STATEMENT,
+ COMMENT
+ }
+
+ String DEFAULT_DELIMITER = ";";
+ String DEFAULT_QUOTE = "\"";
+
+ /**
+ * Find the type of given command
+ *
+ * @param dbCommand
+ * @return
+ */
+ boolean isPartialCommand(String dbCommand) throws IllegalArgumentException;
+
+ /**
+ * Parse the DB specific nesting format and extract the inner script name if any
+ *
+ * @param dbCommand command from parent script
+ * @return
+ * @throws IllegalFormatException
+ */
+ String getScriptName(String dbCommand) throws IllegalArgumentException;
+
+ /**
+ * Find if the given command is a nested script execution
+ *
+ * @param dbCommand
+ * @return
+ */
+ boolean isNestedScript(String dbCommand);
+
+ /**
+ * Find if the given command should not be passed to DB
+ *
+ * @param dbCommand
+ * @return
+ */
+ boolean isNonExecCommand(String dbCommand);
+
+ /**
+ * Get the SQL statement delimiter
+ *
+ * @return
+ */
+ String getDelimiter();
+
+ /**
+ * Get the SQL indentifier quotation character
+ *
+ * @return
+ */
+ String getQuoteCharacter();
+
+ /**
+ * Clear any client specific tags
+ *
+ * @return
+ */
+ String cleanseCommand(String dbCommand);
+
+ /**
+ * Does the DB required table/column names quoted
+ *
+ * @return
+ */
+ boolean needsQuotedIdentifier();
+
+ /**
+ * Flatten the nested upgrade script into a buffer
+ *
+ * @param scriptDir upgrade script directory
+ * @param scriptFile upgrade script file
+ * @return string of sql commands
+ */
+ String buildCommand(String scriptDir, String scriptFile)
+ throws IllegalFormatException, IOException;
+
+ /**
+ * Flatten the nested upgrade script into a buffer
+ *
+ * @param scriptDir upgrade script directory
+ * @param scriptFile upgrade script file
+ * @param fixQuotes whether to replace quote characters
+ * @return string of sql commands
+ */
+ String buildCommand(String scriptDir, String scriptFile, boolean fixQuotes)
+ throws IllegalFormatException, IOException;
+ }
+
+ /***
+ * Base implementation of NestedScriptParser
+ * abstractCommandParser.
+ *
+ */
+ private static abstract class AbstractCommandParser implements NestedScriptParser {
+ private List<String> dbOpts;
+ private String msUsername;
+ private String msPassword;
+ private Configuration conf;
+
+ public AbstractCommandParser(String dbOpts, String msUsername, String msPassword,
+ Configuration conf) {
+ setDbOpts(dbOpts);
+ this.msUsername = msUsername;
+ this.msPassword = msPassword;
+ this.conf = conf;
+ }
+
+ @Override
+ public boolean isPartialCommand(String dbCommand) throws IllegalArgumentException{
+ if (dbCommand == null || dbCommand.isEmpty()) {
+ throw new IllegalArgumentException("invalid command line " + dbCommand);
+ }
+ dbCommand = dbCommand.trim();
+ if (dbCommand.endsWith(getDelimiter()) || isNonExecCommand(dbCommand)) {
+ return false;
+ } else {
+ return true;
+ }
+ }
+
+ @Override
+ public boolean isNonExecCommand(String dbCommand) {
+ return (dbCommand.startsWith("--") || dbCommand.startsWith("#"));
+ }
+
+ @Override
+ public String getDelimiter() {
+ return DEFAULT_DELIMITER;
+ }
+
+ @Override
+ public String getQuoteCharacter() {
+ return DEFAULT_QUOTE;
+ }
+
+
+ @Override
+ public String cleanseCommand(String dbCommand) {
+ // strip off the delimiter
+ if (dbCommand.endsWith(getDelimiter())) {
+ dbCommand = dbCommand.substring(0,
+ dbCommand.length() - getDelimiter().length());
+ }
+ return dbCommand;
+ }
+
+ @Override
+ public boolean needsQuotedIdentifier() {
+ return false;
+ }
+
+ @Override
+ public String buildCommand(
+ String scriptDir, String scriptFile) throws IllegalFormatException, IOException {
+ return buildCommand(scriptDir, scriptFile, false);
+ }
+
+ @Override
+ public String buildCommand(
+ String scriptDir, String scriptFile, boolean fixQuotes) throws IllegalFormatException, IOException {
+ BufferedReader bfReader =
+ new BufferedReader(new FileReader(scriptDir + File.separatorChar + scriptFile));
+ String currLine;
+ StringBuilder sb = new StringBuilder();
+ String currentCommand = null;
+ while ((currLine = bfReader.readLine()) != null) {
+ currLine = currLine.trim();
+
+ if (fixQuotes && !getQuoteCharacter().equals(DEFAULT_QUOTE)) {
+ currLine = currLine.replace("\\\"", getQuoteCharacter());
+ }
+
+ if (currLine.isEmpty()) {
+ continue; // skip empty lines
+ }
+
+ if (currentCommand == null) {
+ currentCommand = currLine;
+ } else {
+ currentCommand = currentCommand + " " + currLine;
+ }
+ if (isPartialCommand(currLine)) {
+ // if its a partial line, continue collecting the pieces
+ continue;
+ }
+
+ // if this is a valid executable command then add it to the buffer
+ if (!isNonExecCommand(currentCommand)) {
+ currentCommand = cleanseCommand(currentCommand);
+ if (isNestedScript(currentCommand)) {
+ // if this is a nested sql script then flatten it
+ String currScript = getScriptName(currentCommand);
+ sb.append(buildCommand(scriptDir, currScript));
+ } else {
+ // Now we have a complete statement, process it
+ // write the line to buffer
+ sb.append(currentCommand);
+ sb.append(System.getProperty("line.separator"));
+ }
+ }
+ currentCommand = null;
+ }
+ bfReader.close();
+ return sb.toString();
+ }
+
+ private void setDbOpts(String dbOpts) {
+ if (dbOpts != null) {
+ this.dbOpts = Lists.newArrayList(dbOpts.split(","));
+ } else {
+ this.dbOpts = Lists.newArrayList();
+ }
+ }
+
+ protected List<String> getDbOpts() {
+ return dbOpts;
+ }
+
+ protected String getMsUsername() {
+ return msUsername;
+ }
+
+ protected String getMsPassword() {
+ return msPassword;
+ }
+
+ protected Configuration getConf() {
+ return conf;
+ }
+ }
+
+ // Derby commandline parser
+ public static class DerbyCommandParser extends AbstractCommandParser {
+ private static final String DERBY_NESTING_TOKEN = "RUN";
+
+ public DerbyCommandParser(String dbOpts, String msUsername, String msPassword,
+ Configuration conf) {
+ super(dbOpts, msUsername, msPassword, conf);
+ }
+
+ @Override
+ public String getScriptName(String dbCommand) throws IllegalArgumentException {
+
+ if (!isNestedScript(dbCommand)) {
+ throw new IllegalArgumentException("Not a script format " + dbCommand);
+ }
+ String[] tokens = dbCommand.split(" ");
+ if (tokens.length != 2) {
+ throw new IllegalArgumentException("Couldn't parse line " + dbCommand);
+ }
+ return tokens[1].replace(";", "").replaceAll("'", "");
+ }
+
+ @Override
+ public boolean isNestedScript(String dbCommand) {
+ // Derby script format is RUN '<file>'
+ return dbCommand.startsWith(DERBY_NESTING_TOKEN);
+ }
+ }
+
+ // Derby commandline parser
+ public static class HiveCommandParser extends AbstractCommandParser {
+ private static String HIVE_NESTING_TOKEN = "SOURCE";
+ private final NestedScriptParser nestedDbCommandParser;
+
+ public HiveCommandParser(String dbOpts, String msUsername, String msPassword,
+ Configuration conf, String metaDbType) {
+ super(dbOpts, msUsername, msPassword, conf);
+ nestedDbCommandParser = getDbCommandParser(metaDbType);
+ }
+
+ @Override
+ public String getQuoteCharacter() {
+ return nestedDbCommandParser.getQuoteCharacter();
+ }
+
+ @Override
+ public String getScriptName(String dbCommand) throws IllegalArgumentException {
+
+ if (!isNestedScript(dbCommand)) {
+ throw new IllegalArgumentException("Not a script format " + dbCommand);
+ }
+ String[] tokens = dbCommand.split(" ");
+ if (tokens.length != 2) {
+ throw new IllegalArgumentException("Couldn't parse line " + dbCommand);
+ }
+ return tokens[1].replace(";", "");
+ }
+
+ @Override
+ public boolean isNestedScript(String dbCommand) {
+ return dbCommand.startsWith(HIVE_NESTING_TOKEN);
+ }
+ }
+
+ // MySQL parser
+ public static class MySqlCommandParser extends AbstractCommandParser {
+ private static final String MYSQL_NESTING_TOKEN = "SOURCE";
+ private static final String DELIMITER_TOKEN = "DELIMITER";
+ private String delimiter = DEFAULT_DELIMITER;
+
+ public MySqlCommandParser(String dbOpts, String msUsername, String msPassword,
+ Configuration conf) {
+ super(dbOpts, msUsername, msPassword, conf);
+ }
+
+ @Override
+ public boolean isPartialCommand(String dbCommand) throws IllegalArgumentException{
+ boolean isPartial = super.isPartialCommand(dbCommand);
+ // if this is a delimiter directive, reset our delimiter
+ if (dbCommand.startsWith(DELIMITER_TOKEN)) {
+ String[] tokens = dbCommand.split(" ");
+ if (tokens.length != 2) {
+ throw new IllegalArgumentException("Couldn't parse line " + dbCommand);
+ }
+ delimiter = tokens[1];
+ }
+ return isPartial;
+ }
+
+ @Override
+ public String getScriptName(String dbCommand) throws IllegalArgumentException {
+ String[] tokens = dbCommand.split(" ");
+ if (tokens.length != 2) {
+ throw new IllegalArgumentException("Couldn't parse line " + dbCommand);
+ }
+ // remove ending ';'
+ return tokens[1].replace(";", "");
+ }
+
+ @Override
+ public boolean isNestedScript(String dbCommand) {
+ return dbCommand.startsWith(MYSQL_NESTING_TOKEN);
+ }
+
+ @Override
+ public String getDelimiter() {
+ return delimiter;
+ }
+
+ @Override
+ public String getQuoteCharacter() {
+ return "`";
+ }
+
+ @Override
+ public boolean isNonExecCommand(String dbCommand) {
+ return super.isNonExecCommand(dbCommand) ||
+ (dbCommand.startsWith("/*") && dbCommand.endsWith("*/")) ||
+ dbCommand.startsWith(DELIMITER_TOKEN);
+ }
+
+ @Override
+ public String cleanseCommand(String dbCommand) {
+ return super.cleanseCommand(dbCommand).replaceAll("/\\*.*?\\*/[^;]", "");
+ }
+
+ }
+
+ // Postgres specific parser
+ public static class PostgresCommandParser extends AbstractCommandParser {
+ private static final String POSTGRES_NESTING_TOKEN = "\\i";
+ @VisibleForTesting
+ public static final String POSTGRES_STANDARD_STRINGS_OPT = "SET standard_conforming_strings";
+ @VisibleForTesting
+ public static final String POSTGRES_SKIP_STANDARD_STRINGS_DBOPT = "postgres.filter.81";
+
+ public PostgresCommandParser(String dbOpts, String msUsername, String msPassword,
+ Configuration conf) {
+ super(dbOpts, msUsername, msPassword, conf);
+ }
+
+ @Override
+ public String getScriptName(String dbCommand) throws IllegalArgumentException {
+ String[] tokens = dbCommand.split(" ");
+ if (tokens.length != 2) {
+ throw new IllegalArgumentException("Couldn't parse line " + dbCommand);
+ }
+ // remove ending ';'
+ return tokens[1].replace(";", "");
+ }
+
+ @Override
+ public boolean isNestedScript(String dbCommand) {
+ return dbCommand.startsWith(POSTGRES_NESTING_TOKEN);
+ }
+
+ @Override
+ public boolean needsQuotedIdentifier() {
+ return true;
+ }
+
+ @Override
+ public boolean isNonExecCommand(String dbCommand) {
+ // Skip "standard_conforming_strings" command which is read-only in older
+ // Postgres versions like 8.1
+ // See: http://www.postgresql.org/docs/8.2/static/release-8-1.html
+ if (getDbOpts().contains(POSTGRES_SKIP_STANDARD_STRINGS_DBOPT)) {
+ if (dbCommand.startsWith(POSTGRES_STANDARD_STRINGS_OPT)) {
+ return true;
+ }
+ }
+ return super.isNonExecCommand(dbCommand);
+ }
+ }
+
+ //Oracle specific parser
+ public static class OracleCommandParser extends AbstractCommandParser {
+ private static final String ORACLE_NESTING_TOKEN = "@";
+
+ public OracleCommandParser(String dbOpts, String msUsername, String msPassword,
+ Configuration conf) {
+ super(dbOpts, msUsername, msPassword, conf);
+ }
+
+ @Override
+ public String getScriptName(String dbCommand) throws IllegalArgumentException {
+ if (!isNestedScript(dbCommand)) {
+ throw new IllegalArgumentException("Not a nested script format " + dbCommand);
+ }
+ // remove ending ';' and starting '@'
+ return dbCommand.replace(";", "").replace(ORACLE_NESTING_TOKEN, "");
+ }
+
+ @Override
+ public boolean isNestedScript(String dbCommand) {
+ return dbCommand.startsWith(ORACLE_NESTING_TOKEN);
+ }
+ }
+
+ //MSSQL specific parser
+ public static class MSSQLCommandParser extends AbstractCommandParser {
+ private static final String MSSQL_NESTING_TOKEN = ":r";
+
+ public MSSQLCommandParser(String dbOpts, String msUsername, String msPassword,
+ Configuration conf) {
+ super(dbOpts, msUsername, msPassword, conf);
+ }
+
+ @Override
+ public String getScriptName(String dbCommand) throws IllegalArgumentException {
+ String[] tokens = dbCommand.split(" ");
+ if (tokens.length != 2) {
+ throw new IllegalArgumentException("Couldn't parse line " + dbCommand);
+ }
+ return tokens[1];
+ }
+
+ @Override
+ public boolean isNestedScript(String dbCommand) {
+ return dbCommand.startsWith(MSSQL_NESTING_TOKEN);
+ }
+ }
+
+ public static NestedScriptParser getDbCommandParser(String dbName) {
+ return getDbCommandParser(dbName, null);
+ }
+
+ public static NestedScriptParser getDbCommandParser(String dbName, String metaDbName) {
+ return getDbCommandParser(dbName, null, null, null, null, metaDbName);
+ }
+
+ public static NestedScriptParser getDbCommandParser(String dbName,
+ String dbOpts, String msUsername, String msPassword,
+ Configuration conf, String metaDbType) {
+ if (dbName.equalsIgnoreCase(DB_DERBY)) {
+ return new DerbyCommandParser(dbOpts, msUsername, msPassword, conf);
+ } else if (dbName.equalsIgnoreCase(DB_HIVE)) {
+ return new HiveCommandParser(dbOpts, msUsername, msPassword, conf, metaDbType);
+ } else if (dbName.equalsIgnoreCase(DB_MSSQL)) {
+ return new MSSQLCommandParser(dbOpts, msUsername, msPassword, conf);
+ } else if (dbName.equalsIgnoreCase(DB_MYSQL)) {
+ return new MySqlCommandParser(dbOpts, msUsername, msPassword, conf);
+ } else if (dbName.equalsIgnoreCase(DB_POSTGRACE)) {
+ return new PostgresCommandParser(dbOpts, msUsername, msPassword, conf);
+ } else if (dbName.equalsIgnoreCase(DB_ORACLE)) {
+ return new OracleCommandParser(dbOpts, msUsername, msPassword, conf);
+ } else {
+ throw new IllegalArgumentException("Unknown dbType " + dbName);
+ }
+ }
+
+ public static class MetaStoreConnectionInfo {
+ private final String userName;
+ private final String password;
+ private final String url;
+ private final String driver;
+ private final boolean printInfo;
+ private final Configuration conf;
+ private final String dbType;
+
+ public MetaStoreConnectionInfo(String userName, String password, String url, String driver,
+ boolean printInfo, Configuration conf, String dbType) {
+ super();
+ this.userName = userName;
+ this.password = password;
+ this.url = url;
+ this.driver = driver;
+ this.printInfo = printInfo;
+ this.conf = conf;
+ this.dbType = dbType;
+ }
+
+ public String getPassword() {
+ return password;
+ }
+
+ public String getUrl() {
+ return url;
+ }
+
+ public String getDriver() {
+ return driver;
+ }
+
+ public boolean isPrintInfo() {
+ return printInfo;
+ }
+
+ public Configuration getConf() {
+ return conf;
+ }
+
+ public String getUsername() {
+ return userName;
+ }
+
+ public boolean getPrintInfo() {
+ return printInfo;
+ }
+
+ public String getDbType() {
+ return dbType;
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/hive/blob/b0b6db73/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/utils/FileUtils.java
----------------------------------------------------------------------
diff --git a/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/utils/FileUtils.java b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/utils/FileUtils.java
new file mode 100644
index 0000000..2310df6
--- /dev/null
+++ b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/utils/FileUtils.java
@@ -0,0 +1,66 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hive.metastore.utils;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.Trash;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.IOException;
+
+public class FileUtils {
+ private static final Logger LOG = LoggerFactory.getLogger(FileUtils.class);
+
+ /**
+ * Move a particular file or directory to the trash.
+ * @param fs FileSystem to use
+ * @param f path of file or directory to move to trash.
+ * @param conf
+ * @return true if move successful
+ * @throws IOException
+ */
+ public static boolean moveToTrash(FileSystem fs, Path f, Configuration conf, boolean purge)
+ throws IOException {
+ LOG.debug("deleting " + f);
+ boolean result = false;
+ try {
+ if(purge) {
+ LOG.debug("purge is set to true. Not moving to Trash " + f);
+ } else {
+ result = Trash.moveToAppropriateTrash(fs, f, conf);
+ if (result) {
+ LOG.trace("Moved to trash: " + f);
+ return true;
+ }
+ }
+ } catch (IOException ioe) {
+ // for whatever failure reason including that trash has lower encryption zone
+ // retry with force delete
+ LOG.warn(ioe.getMessage() + "; Force to delete it.");
+ }
+
+ result = fs.delete(f, true);
+ if (!result) {
+ LOG.error("Failed to delete " + f);
+ }
+ return result;
+ }
+}
http://git-wip-us.apache.org/repos/asf/hive/blob/b0b6db73/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/utils/JavaUtils.java
----------------------------------------------------------------------
diff --git a/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/utils/JavaUtils.java b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/utils/JavaUtils.java
new file mode 100644
index 0000000..81f8a85
--- /dev/null
+++ b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/utils/JavaUtils.java
@@ -0,0 +1,37 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hive.metastore.utils;
+
+public class JavaUtils {
+ /**
+ * Standard way of getting classloader in Hive code (outside of Hadoop).
+ *
+ * Uses the context loader to get access to classpaths to auxiliary and jars
+ * added with 'add jar' command. Falls back to current classloader.
+ *
+ * In Hadoop-related code, we use Configuration.getClassLoader().
+ * @return the class loader
+ */
+ public static ClassLoader getClassLoader() {
+ ClassLoader classLoader = Thread.currentThread().getContextClassLoader();
+ if (classLoader == null) {
+ classLoader = JavaUtils.class.getClassLoader();
+ }
+ return classLoader;
+ }
+}
http://git-wip-us.apache.org/repos/asf/hive/blob/b0b6db73/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/utils/MetaStoreUtils.java
----------------------------------------------------------------------
diff --git a/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/utils/MetaStoreUtils.java b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/utils/MetaStoreUtils.java
new file mode 100644
index 0000000..3ef7e51
--- /dev/null
+++ b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/utils/MetaStoreUtils.java
@@ -0,0 +1,41 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hive.metastore.utils;
+
+import org.apache.hadoop.hive.metastore.api.MetaException;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+public class MetaStoreUtils {
+ private static final Logger LOG = LoggerFactory.getLogger(MetaStoreUtils.class);
+
+ /**
+ * Catches exceptions that can't be handled and bundles them to MetaException
+ *
+ * @param e exception to wrap.
+ * @throws MetaException wrapper for the exception
+ */
+ public static void logAndThrowMetaException(Exception e) throws MetaException {
+ String exInfo = "Got exception: " + e.getClass().getName() + " "
+ + e.getMessage();
+ LOG.error(exInfo, e);
+ LOG.error("Converting exception to MetaException");
+ throw new MetaException(exInfo);
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/hive/blob/b0b6db73/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/utils/MetastoreVersionInfo.java
----------------------------------------------------------------------
diff --git a/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/utils/MetastoreVersionInfo.java b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/utils/MetastoreVersionInfo.java
new file mode 100644
index 0000000..de54ff3
--- /dev/null
+++ b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/utils/MetastoreVersionInfo.java
@@ -0,0 +1,133 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hive.metastore.utils;
+
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
+import org.apache.hadoop.hive.metastore.annotation.MetastoreVersionAnnotation;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * Class that uses package information to figure out which version of the metastore this program is.
+ */
+@InterfaceAudience.Private
+@InterfaceStability.Unstable
+public class MetastoreVersionInfo {
+ private static final Logger LOG = LoggerFactory.getLogger(MetastoreVersionInfo.class);
+
+ private static Package myPackage;
+ private static MetastoreVersionAnnotation version;
+
+ static {
+ myPackage = MetastoreVersionAnnotation.class.getPackage();
+ version = myPackage.getAnnotation(MetastoreVersionAnnotation.class);
+ }
+
+ /**
+ * Get the meta-data for the Hive package.
+ * @return
+ */
+ static Package getPackage() {
+ return myPackage;
+ }
+
+ /**
+ * Get the Hive version.
+ * @return the Hive version string, eg. "0.6.3-dev"
+ */
+ public static String getVersion() {
+ return version != null ? version.version() : "Unknown";
+ }
+
+ /**
+ * Get the Hive short version, with major/minor/change version numbers.
+ * @return short version string, eg. "0.6.3"
+ */
+ public static String getShortVersion() {
+ return version != null ? version.shortVersion() : "Unknown";
+ }
+
+ /**
+ * Get the git revision number for the root directory
+ * @return the revision number, eg. "451451"
+ */
+ public static String getRevision() {
+ return version != null ? version.revision() : "Unknown";
+ }
+
+ /**
+ * Get the branch on which this originated.
+ * @return The branch name, e.g. "trunk" or "branches/branch-0.20"
+ */
+ public static String getBranch() {
+ return version != null ? version.branch() : "Unknown";
+ }
+
+ /**
+ * The date that Hive was compiled.
+ * @return the compilation date in unix date format
+ */
+ public static String getDate() {
+ return version != null ? version.date() : "Unknown";
+ }
+
+ /**
+ * The user that compiled Hive.
+ * @return the username of the user
+ */
+ public static String getUser() {
+ return version != null ? version.user() : "Unknown";
+ }
+
+ /**
+ * Get the git URL for the root Hive directory.
+ */
+ public static String getUrl() {
+ return version != null ? version.url() : "Unknown";
+ }
+
+ /**
+ * Get the checksum of the source files from which Hive was
+ * built.
+ **/
+ public static String getSrcChecksum() {
+ return version != null ? version.srcChecksum() : "Unknown";
+ }
+
+ /**
+ * Returns the buildVersion which includes version,
+ * revision, user and date.
+ */
+ public static String getBuildVersion(){
+ return MetastoreVersionInfo.getVersion() +
+ " from " + MetastoreVersionInfo.getRevision() +
+ " by " + MetastoreVersionInfo.getUser() +
+ " source checksum " + MetastoreVersionInfo.getSrcChecksum();
+ }
+
+ public static void main(String[] args) {
+ LOG.debug("version: "+ version);
+ System.out.println("Hive " + getVersion());
+ System.out.println("Git " + getUrl() + " -r " + getRevision());
+ System.out.println("Compiled by " + getUser() + " on " + getDate());
+ System.out.println("From source with checksum " + getSrcChecksum());
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/hive/blob/b0b6db73/standalone-metastore/src/main/protobuf/org/apache/hadoop/hive/metastore/metastore.proto
----------------------------------------------------------------------
diff --git a/standalone-metastore/src/main/protobuf/org/apache/hadoop/hive/metastore/metastore.proto b/standalone-metastore/src/main/protobuf/org/apache/hadoop/hive/metastore/metastore.proto
new file mode 100644
index 0000000..29b99b4
--- /dev/null
+++ b/standalone-metastore/src/main/protobuf/org/apache/hadoop/hive/metastore/metastore.proto
@@ -0,0 +1,29 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hive.metastore;
+
+
+message SplitInfo {
+ required int64 offset = 1;
+ required int64 length = 2;
+ required int32 index = 3;
+}
+
+message SplitInfos {
+ repeated SplitInfo infos = 1;
+}
http://git-wip-us.apache.org/repos/asf/hive/blob/b0b6db73/standalone-metastore/src/main/resources/saveVersion.sh
----------------------------------------------------------------------
diff --git a/standalone-metastore/src/main/resources/saveVersion.sh b/standalone-metastore/src/main/resources/saveVersion.sh
new file mode 100755
index 0000000..c9bde68
--- /dev/null
+++ b/standalone-metastore/src/main/resources/saveVersion.sh
@@ -0,0 +1,91 @@
+#!/usr/bin/env bash
+
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+
+# This file is used to generate the package-info.java class that
+# records the version, revision, branch, user, timestamp, and url
+unset LANG
+unset LC_CTYPE
+unset LC_TIME
+version=$1
+shortversion=$2
+src_dir=$3
+revision=$4
+branch=$5
+url=$6
+user=`whoami`
+date=`date`
+dir=`pwd`
+cwd=`dirname $dir`
+if [ "$revision" = "" ]; then
+ if git rev-parse HEAD 2>/dev/null > /dev/null ; then
+ revision=`git log -1 --pretty=format:"%H"`
+ hostname=`hostname`
+ branch=`git branch | sed -n -e 's/^* //p'`
+ url="git://${hostname}${cwd}"
+ elif [ -d .svn ]; then
+ revision=`svn info ../ | sed -n -e 's/Last Changed Rev: \(.*\)/\1/p'`
+ url=`svn info ../ | sed -n -e 's/^URL: \(.*\)/\1/p'`
+ # Get canonical branch (branches/X, tags/X, or trunk)
+ branch=`echo $url | sed -n -e 's,.*\(branches/.*\)$,\1,p' \
+ -e 's,.*\(tags/.*\)$,\1,p' \
+ -e 's,.*trunk$,trunk,p'`
+ else
+ revision="Unknown"
+ branch="Unknown"
+ url="file://$cwd"
+ fi
+fi
+if [ "$branch" = "" ]; then
+ branch="Unknown"
+fi
+if [ "$url" = "" ]; then
+ url="file://$cwd"
+fi
+
+if [ -x /sbin/md5 ]; then
+ md5="/sbin/md5"
+else
+ md5="md5sum"
+fi
+
+srcChecksum=`find ../ -name '*.java' | grep -v generated-sources | LC_ALL=C sort | xargs $md5 | $md5 | cut -d ' ' -f 1`
+
+mkdir -p $src_dir/gen/org/apache/hadoop/hive/metastore/annotation
+
+# In Windows, all the following string ends with \r, need to get rid of them
+branch=`echo $branch | tr -d '\r'`
+user=`echo $user | tr -d '\r'`
+date=`echo $date | tr -d '\r'`
+url=`echo $url | tr -d '\r'`
+srcChecksum=`echo $srcChecksum | tr -d '\r'`
+
+cat << EOF | \
+ sed -e "s/VERSION/$version/" -e "s/SHORTVERSION/$shortversion/" \
+ -e "s/USER/$user/" -e "s/DATE/$date/" \
+ -e "s|URL|$url|" -e "s/REV/$revision/" \
+ -e "s|BRANCH|$branch|" -e "s/SRCCHECKSUM/$srcChecksum/" \
+ > $src_dir/gen/org/apache/hadoop/hive/metastore/annotation/package-info.java
+/*
+ * Generated by saveVersion.sh
+ */
+@MetastoreVersionAnnotation(version="VERSION", shortVersion="SHORTVERSION",
+ revision="REV", branch="BRANCH",
+ user="USER", date="DATE", url="URL",
+ srcChecksum="SRCCHECKSUM")
+package org.apache.hadoop.hive.metastore.annotation;
+EOF
http://git-wip-us.apache.org/repos/asf/hive/blob/b0b6db73/standalone-metastore/src/test/java/org/apache/hadoop/hive/metastore/TestAggregateStatsCache.java
----------------------------------------------------------------------
diff --git a/standalone-metastore/src/test/java/org/apache/hadoop/hive/metastore/TestAggregateStatsCache.java b/standalone-metastore/src/test/java/org/apache/hadoop/hive/metastore/TestAggregateStatsCache.java
new file mode 100644
index 0000000..6e06026
--- /dev/null
+++ b/standalone-metastore/src/test/java/org/apache/hadoop/hive/metastore/TestAggregateStatsCache.java
@@ -0,0 +1,267 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.hadoop.hive.metastore;
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.concurrent.TimeUnit;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hive.metastore.api.ColumnStatisticsData;
+import org.apache.hadoop.hive.metastore.api.ColumnStatisticsObj;
+import org.apache.hadoop.hive.metastore.api.LongColumnStatsData;
+import org.apache.hadoop.hive.metastore.AggregateStatsCache.AggrColStats;
+import org.apache.hadoop.hive.metastore.AggregateStatsCache.Key;
+import org.apache.hadoop.hive.metastore.conf.MetastoreConf;
+import org.apache.hadoop.hive.metastore.conf.MetastoreConf.ConfVars;
+import org.apache.hive.common.util.BloomFilter;
+import org.junit.After;
+import org.junit.AfterClass;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.BeforeClass;
+import org.junit.Test;
+
+public class TestAggregateStatsCache {
+ static String DB_NAME = "db";
+ static String TAB_PREFIX = "tab";
+ static String PART_PREFIX = "part";
+ static String COL_PREFIX = "col";
+ static int NUM_TABS = 2;
+ static int NUM_PARTS = 20;
+ static int NUM_COLS = 5;
+ static int MAX_CACHE_NODES = 10;
+ static int MAX_PARTITIONS_PER_CACHE_NODE = 10;
+ static long TIME_TO_LIVE = 2;
+ static long MAX_WRITER_WAIT = 1;
+ static long MAX_READER_WAIT = 1;
+ static double FALSE_POSITIVE_PROBABILITY = 0.01;
+ static double MAX_VARIANCE = 0.5;
+ static AggregateStatsCache cache;
+ static List<String> tables = new ArrayList<>();
+ static List<String> tabParts = new ArrayList<>();
+ static List<String> tabCols = new ArrayList<>();
+
+ @BeforeClass
+ public static void beforeTest() {
+ // All data intitializations
+ initializeTables();
+ initializePartitions();
+ initializeColumns();
+ }
+
+ // tab1, tab2
+ private static void initializeTables() {
+ for (int i = 1; i <= NUM_TABS; i++) {
+ tables.add(TAB_PREFIX + i);
+ }
+ }
+
+ // part1 ... part20
+ private static void initializePartitions() {
+ for (int i = 1; i <= NUM_PARTS; i++) {
+ tabParts.add(PART_PREFIX + i);
+ }
+ }
+
+ // col1 ... col5
+ private static void initializeColumns() {
+ for (int i = 1; i <= NUM_COLS; i++) {
+ tabCols.add(COL_PREFIX + i);
+ }
+ }
+
+ @AfterClass
+ public static void afterTest() {
+ }
+
+ @Before
+ public void setUp() {
+ Configuration conf = MetastoreConf.newMetastoreConf();
+ MetastoreConf.setLongVar(conf, ConfVars.AGGREGATE_STATS_CACHE_SIZE, MAX_CACHE_NODES);
+ MetastoreConf.setLongVar(conf, ConfVars.AGGREGATE_STATS_CACHE_MAX_PARTITIONS,
+ MAX_PARTITIONS_PER_CACHE_NODE);
+ MetastoreConf.setDoubleVar(conf, ConfVars.AGGREGATE_STATS_CACHE_FPP, FALSE_POSITIVE_PROBABILITY);
+ MetastoreConf.setDoubleVar(conf, ConfVars.AGGREGATE_STATS_CACHE_MAX_VARIANCE, MAX_VARIANCE);
+ MetastoreConf.setTimeVar(conf, ConfVars.AGGREGATE_STATS_CACHE_TTL, TIME_TO_LIVE, TimeUnit.SECONDS);
+ MetastoreConf.setTimeVar(conf, ConfVars.AGGREGATE_STATS_CACHE_MAX_WRITER_WAIT,
+ MAX_WRITER_WAIT, TimeUnit.SECONDS);
+ MetastoreConf.setTimeVar(conf, ConfVars.AGGREGATE_STATS_CACHE_MAX_READER_WAIT,
+ MAX_READER_WAIT, TimeUnit.SECONDS);
+ cache = AggregateStatsCache.getInstance(conf);
+ }
+
+ @After
+ public void tearDown() {
+ }
+
+ @Test
+ public void testCacheKey() {
+ Key k1 = new Key("db", "tbl1", "col");
+ Key k2 = new Key("db", "tbl1", "col");
+ // k1 equals k2
+ Assert.assertEquals(k1, k2);
+ Key k3 = new Key("db", "tbl2", "col");
+ // k1 not equals k3
+ Assert.assertNotEquals(k1, k3);
+ }
+
+ @Test
+ public void testBasicAddAndGet() throws Exception {
+ // Partnames: [tab1part1...tab1part9]
+ List<String> partNames = preparePartNames(tables.get(0), 1, 9);
+ // Prepare the bloom filter
+ BloomFilter bloomFilter = prepareBloomFilter(partNames);
+ // Add a dummy aggregate stats object for the above parts (part1...part9) of tab1 for col1
+ String tblName = tables.get(0);
+ String colName = tabCols.get(0);
+ int highVal = 100, lowVal = 10, numDVs = 50, numNulls = 5;
+ // We'll treat this as the aggregate col stats for part1...part9 of tab1, col1
+ ColumnStatisticsObj aggrColStats =
+ getDummyLongColStat(colName, highVal, lowVal, numDVs, numNulls);
+ // Now add to cache the dummy colstats for these 10 partitions
+ cache.add(DB_NAME, tblName, colName, 10, aggrColStats, bloomFilter);
+ // Now get from cache
+ AggrColStats aggrStatsCached = cache.get(DB_NAME, tblName, colName, partNames);
+ Assert.assertNotNull(aggrStatsCached);
+
+ ColumnStatisticsObj aggrColStatsCached = aggrStatsCached.getColStats();
+ Assert.assertEquals(aggrColStats, aggrColStatsCached);
+
+ // Now get a non-existant entry
+ aggrStatsCached = cache.get("dbNotThere", tblName, colName, partNames);
+ Assert.assertNull(aggrStatsCached);
+ }
+
+ @Test
+ public void testAddGetWithVariance() throws Exception {
+ // Partnames: [tab1part1...tab1part9]
+ List<String> partNames = preparePartNames(tables.get(0), 1, 9);
+ // Prepare the bloom filter
+ BloomFilter bloomFilter = prepareBloomFilter(partNames);
+ // Add a dummy aggregate stats object for the above parts (part1...part9) of tab1 for col1
+ String tblName = tables.get(0);
+ String colName = tabCols.get(0);
+ int highVal = 100, lowVal = 10, numDVs = 50, numNulls = 5;
+ // We'll treat this as the aggregate col stats for part1...part9 of tab1, col1
+ ColumnStatisticsObj aggrColStats =
+ getDummyLongColStat(colName, highVal, lowVal, numDVs, numNulls);
+ // Now add to cache
+ cache.add(DB_NAME, tblName, colName, 10, aggrColStats, bloomFilter);
+
+ // Now prepare partnames with only 5 partitions: [tab1part1...tab1part5]
+ partNames = preparePartNames(tables.get(0), 1, 5);
+ // This get should fail because its variance ((10-5)/5) is way past MAX_VARIANCE (0.5)
+ AggrColStats aggrStatsCached = cache.get(DB_NAME, tblName, colName, partNames);
+ Assert.assertNull(aggrStatsCached);
+
+ // Now prepare partnames with 10 partitions: [tab1part11...tab1part20], but with no overlap
+ partNames = preparePartNames(tables.get(0), 11, 20);
+ // This get should fail because its variance ((10-0)/10) is way past MAX_VARIANCE (0.5)
+ aggrStatsCached = cache.get(DB_NAME, tblName, colName, partNames);
+ Assert.assertNull(aggrStatsCached);
+
+ // Now prepare partnames with 9 partitions: [tab1part1...tab1part8], which are contained in the
+ // object that we added to the cache
+ partNames = preparePartNames(tables.get(0), 1, 8);
+ // This get should succeed because its variance ((10-9)/9) is within past MAX_VARIANCE (0.5)
+ aggrStatsCached = cache.get(DB_NAME, tblName, colName, partNames);
+ Assert.assertNotNull(aggrStatsCached);
+ ColumnStatisticsObj aggrColStatsCached = aggrStatsCached.getColStats();
+ Assert.assertEquals(aggrColStats, aggrColStatsCached);
+ }
+
+ @Test
+ public void testTimeToLive() throws Exception {
+ // Add a dummy node to cache
+ // Partnames: [tab1part1...tab1part9]
+ List<String> partNames = preparePartNames(tables.get(0), 1, 9);
+ // Prepare the bloom filter
+ BloomFilter bloomFilter = prepareBloomFilter(partNames);
+ // Add a dummy aggregate stats object for the above parts (part1...part9) of tab1 for col1
+ String tblName = tables.get(0);
+ String colName = tabCols.get(0);
+ int highVal = 100, lowVal = 10, numDVs = 50, numNulls = 5;
+ // We'll treat this as the aggregate col stats for part1...part9 of tab1, col1
+ ColumnStatisticsObj aggrColStats =
+ getDummyLongColStat(colName, highVal, lowVal, numDVs, numNulls);
+ // Now add to cache
+ cache.add(DB_NAME, tblName, colName, 10, aggrColStats, bloomFilter);
+
+ // Sleep for 3 seconds
+ Thread.sleep(3000);
+
+ // Get should fail now (since TTL is 2s) and we've snoozed for 3 seconds
+ AggrColStats aggrStatsCached = cache.get(DB_NAME, tblName, colName, partNames);
+ Assert.assertNull(aggrStatsCached);
+ }
+
+ /**
+ * Prepares an array of partition names by getting partitions from minPart ... maxPart and
+ * prepending with table name
+ * Example: [tab1part1, tab1part2 ...]
+ *
+ * @param tabName
+ * @param minPart
+ * @param maxPart
+ * @return
+ * @throws Exception
+ */
+ private List<String> preparePartNames(String tabName, int minPart, int maxPart) throws Exception {
+ if ((minPart < 1) || (maxPart > NUM_PARTS)) {
+ throw new Exception("tabParts does not have these partition numbers");
+ }
+ List<String> partNames = new ArrayList<>();
+ for (int i = minPart; i <= maxPart; i++) {
+ String partName = tabParts.get(i-1);
+ partNames.add(tabName + partName);
+ }
+ return partNames;
+ }
+
+ /**
+ * Prepares a bloom filter from the list of partition names
+ * @param partNames
+ * @return
+ */
+ private BloomFilter prepareBloomFilter(List <String> partNames) {
+ BloomFilter bloomFilter =
+ new BloomFilter(MAX_PARTITIONS_PER_CACHE_NODE, FALSE_POSITIVE_PROBABILITY);
+ for (String partName: partNames) {
+ bloomFilter.add(partName.getBytes());
+ }
+ return bloomFilter;
+ }
+
+ private ColumnStatisticsObj getDummyLongColStat(String colName, int highVal, int lowVal, int numDVs, int numNulls) {
+ ColumnStatisticsObj aggrColStats = new ColumnStatisticsObj();
+ aggrColStats.setColName(colName);
+ aggrColStats.setColType("long");
+ LongColumnStatsData longStatsData = new LongColumnStatsData();
+ longStatsData.setHighValue(highVal);
+ longStatsData.setLowValue(lowVal);
+ longStatsData.setNumDVs(numDVs);
+ longStatsData.setNumNulls(numNulls);
+ ColumnStatisticsData aggrColStatsData = new ColumnStatisticsData();
+ aggrColStatsData.setLongStats(longStatsData);
+ aggrColStats.setStatsData(aggrColStatsData);
+ return aggrColStats;
+ }
+}
[6/7] hive git commit: HIVE-17488 Move first set of classes to
standalone metastore. This closes #244. (Alan Gates,
reviewed by Owen O'Malley)
Posted by ga...@apache.org.
http://git-wip-us.apache.org/repos/asf/hive/blob/b0b6db73/metastore/src/java/org/apache/hadoop/hive/metastore/AggregateStatsCache.java
----------------------------------------------------------------------
diff --git a/metastore/src/java/org/apache/hadoop/hive/metastore/AggregateStatsCache.java b/metastore/src/java/org/apache/hadoop/hive/metastore/AggregateStatsCache.java
deleted file mode 100644
index 4ab178c..0000000
--- a/metastore/src/java/org/apache/hadoop/hive/metastore/AggregateStatsCache.java
+++ /dev/null
@@ -1,573 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.hadoop.hive.metastore;
-
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.hive.conf.HiveConf;
-import org.apache.hadoop.hive.metastore.api.ColumnStatisticsObj;
-import org.apache.hive.common.util.BloomFilter;
-
-import java.util.ArrayList;
-import java.util.HashMap;
-import java.util.Iterator;
-import java.util.List;
-import java.util.Map;
-import java.util.concurrent.ConcurrentHashMap;
-import java.util.concurrent.TimeUnit;
-import java.util.concurrent.atomic.AtomicInteger;
-import java.util.concurrent.atomic.AtomicLong;
-import java.util.concurrent.locks.Lock;
-import java.util.concurrent.locks.ReadWriteLock;
-import java.util.concurrent.locks.ReentrantReadWriteLock;
-
-public class AggregateStatsCache {
-
- private static final Logger LOG = LoggerFactory.getLogger(AggregateStatsCache.class.getName());
- private static AggregateStatsCache self = null;
-
- // Backing store for this cache
- private final ConcurrentHashMap<Key, AggrColStatsList> cacheStore;
- // Cache size
- private final int maxCacheNodes;
- // Current nodes in the cache
- private final AtomicInteger currentNodes = new AtomicInteger(0);
- // Run the cleaner thread when the cache is maxFull% full
- private final float maxFull;
- // Run the cleaner thread until cache is cleanUntil% occupied
- private final float cleanUntil;
- // Nodes go stale after this
- private final long timeToLiveMs;
- // Max time when waiting for write locks on node list
- private final long maxWriterWaitTime;
- // Max time when waiting for read locks on node list
- private final long maxReaderWaitTime;
- // Maximum number of paritions aggregated per cache node
- private final int maxPartsPerCacheNode;
- // Bloom filter false positive probability
- private final float falsePositiveProbability;
- // Max tolerable variance for matches
- private final float maxVariance;
- // Used to determine if cleaner thread is already running
- private boolean isCleaning = false;
- private final AtomicLong cacheHits = new AtomicLong(0);
- private final AtomicLong cacheMisses = new AtomicLong(0);
- // To track cleaner metrics
- int numRemovedTTL = 0, numRemovedLRU = 0;
-
- private AggregateStatsCache(int maxCacheNodes, int maxPartsPerCacheNode, long timeToLiveMs,
- float falsePositiveProbability, float maxVariance, long maxWriterWaitTime,
- long maxReaderWaitTime, float maxFull, float cleanUntil) {
- this.maxCacheNodes = maxCacheNodes;
- this.maxPartsPerCacheNode = maxPartsPerCacheNode;
- this.timeToLiveMs = timeToLiveMs;
- this.falsePositiveProbability = falsePositiveProbability;
- this.maxVariance = maxVariance;
- this.maxWriterWaitTime = maxWriterWaitTime;
- this.maxReaderWaitTime = maxReaderWaitTime;
- this.maxFull = maxFull;
- this.cleanUntil = cleanUntil;
- this.cacheStore = new ConcurrentHashMap<Key, AggrColStatsList>();
- }
-
- public static synchronized AggregateStatsCache getInstance(Configuration conf) {
- if (self == null) {
- int maxCacheNodes =
- HiveConf.getIntVar(conf, HiveConf.ConfVars.METASTORE_AGGREGATE_STATS_CACHE_SIZE);
- // The number of partitions aggregated per cache node
- // If the number of partitions requested is > this value, we'll fetch directly from Metastore
- int maxPartitionsPerCacheNode =
- HiveConf
- .getIntVar(conf, HiveConf.ConfVars.METASTORE_AGGREGATE_STATS_CACHE_MAX_PARTITIONS);
- long timeToLiveMs =
- HiveConf.getTimeVar(conf, HiveConf.ConfVars.METASTORE_AGGREGATE_STATS_CACHE_TTL,
- TimeUnit.SECONDS)*1000;
- // False positives probability we are ready to tolerate for the underlying bloom filter
- float falsePositiveProbability =
- HiveConf.getFloatVar(conf, HiveConf.ConfVars.METASTORE_AGGREGATE_STATS_CACHE_FPP);
- // Maximum tolerable variance in number of partitions between cached node and our request
- float maxVariance =
- HiveConf
- .getFloatVar(conf, HiveConf.ConfVars.METASTORE_AGGREGATE_STATS_CACHE_MAX_VARIANCE);
- long maxWriterWaitTime =
- HiveConf.getTimeVar(conf,
- HiveConf.ConfVars.METASTORE_AGGREGATE_STATS_CACHE_MAX_WRITER_WAIT,
- TimeUnit.MILLISECONDS);
- long maxReaderWaitTime =
- HiveConf.getTimeVar(conf,
- HiveConf.ConfVars.METASTORE_AGGREGATE_STATS_CACHE_MAX_READER_WAIT,
- TimeUnit.MILLISECONDS);
- float maxFull =
- HiveConf.getFloatVar(conf, HiveConf.ConfVars.METASTORE_AGGREGATE_STATS_CACHE_MAX_FULL);
- float cleanUntil =
- HiveConf.getFloatVar(conf, HiveConf.ConfVars.METASTORE_AGGREGATE_STATS_CACHE_CLEAN_UNTIL);
- self =
- new AggregateStatsCache(maxCacheNodes, maxPartitionsPerCacheNode, timeToLiveMs,
- falsePositiveProbability, maxVariance, maxWriterWaitTime, maxReaderWaitTime, maxFull,
- cleanUntil);
- }
- return self;
- }
-
- public int getMaxCacheNodes() {
- return maxCacheNodes;
- }
-
- public int getCurrentNodes() {
- return currentNodes.intValue();
- }
-
- public float getFullPercent() {
- return (currentNodes.intValue() / (float) maxCacheNodes) * 100;
- }
-
- public int getMaxPartsPerCacheNode() {
- return maxPartsPerCacheNode;
- }
-
- public float getFalsePositiveProbability() {
- return falsePositiveProbability;
- }
-
- public Float getHitRatio() {
- if (cacheHits.longValue() + cacheMisses.longValue() > 0) {
- return (float) (cacheHits.longValue()) / (cacheHits.longValue() + cacheMisses.longValue());
- }
- return null;
- }
-
- /**
- * Return aggregate stats for a column from the cache or null.
- * While reading from the nodelist for a key, we wait maxReaderWaitTime to acquire the lock,
- * failing which we return a cache miss (i.e. null)
- *
- * @param dbName
- * @param tblName
- * @param colName
- * @param partNames
- * @return
- */
- public AggrColStats get(String dbName, String tblName, String colName, List<String> partNames) {
- // Cache key
- Key key = new Key(dbName, tblName, colName);
- AggrColStatsList candidateList = cacheStore.get(key);
- // No key, or no nodes in candidate list
- if ((candidateList == null) || (candidateList.nodes.size() == 0)) {
- LOG.debug("No aggregate stats cached for " + key.toString());
- return null;
- }
- // Find the value object
- // Update the timestamp of the key,value if value matches the criteria
- // Return the value
- AggrColStats match = null;
- boolean isLocked = false;
- try {
- // Try to readlock the candidateList; timeout after maxReaderWaitTime
- isLocked = candidateList.readLock.tryLock(maxReaderWaitTime, TimeUnit.MILLISECONDS);
- if (isLocked) {
- match = findBestMatch(partNames, candidateList.nodes);
- }
- if (match != null) {
- // Ok to not lock the list for this and use a volatile lastAccessTime instead
- candidateList.updateLastAccessTime();
- cacheHits.incrementAndGet();
- LOG.info("Returning aggregate stats from the cache; total hits: " + cacheHits.longValue()
- + ", total misses: " + cacheMisses.longValue() + ", hit ratio: " + getHitRatio());
- }
- else {
- cacheMisses.incrementAndGet();
- }
- } catch (InterruptedException e) {
- LOG.debug("Interrupted Exception ignored ",e);
- } finally {
- if (isLocked) {
- candidateList.readLock.unlock();
- }
- }
- return match;
- }
-
- /**
- * Find the best match using the configurable error tolerance and time to live value
- *
- * @param partNames
- * @param candidates
- * @return best matched node or null
- */
- private AggrColStats findBestMatch(List<String> partNames, List<AggrColStats> candidates) {
- // Hits, misses tracked for a candidate node
- MatchStats matchStats;
- // MatchStats for each candidate
- Map<AggrColStats, MatchStats> candidateMatchStats = new HashMap<AggrColStats, MatchStats>();
- // The final match we intend to return
- AggrColStats bestMatch = null;
- // To compare among potentially multiple matches
- int bestMatchHits = 0;
- int numPartsRequested = partNames.size();
- // 1st pass at marking invalid candidates
- // Checks based on variance and TTL
- // Note: we're not creating a copy of the list for saving memory
- for (AggrColStats candidate : candidates) {
- // Variance check
- if (Math.abs((candidate.getNumPartsCached() - numPartsRequested) / numPartsRequested)
- > maxVariance) {
- continue;
- }
- // TTL check
- if (isExpired(candidate)) {
- continue;
- } else {
- candidateMatchStats.put(candidate, new MatchStats(0, 0));
- }
- }
- // We'll count misses as we iterate
- int maxMisses = (int) maxVariance * numPartsRequested;
- for (String partName : partNames) {
- for (Iterator<Map.Entry<AggrColStats, MatchStats>> iterator = candidateMatchStats.entrySet().iterator(); iterator.hasNext();) {
- Map.Entry<AggrColStats, MatchStats> entry = iterator.next();
- AggrColStats candidate = entry.getKey();
- matchStats = entry.getValue();
- if (candidate.getBloomFilter().test(partName.getBytes())) {
- ++matchStats.hits;
- } else {
- ++matchStats.misses;
- }
- // 2nd pass at removing invalid candidates
- // If misses so far exceed max tolerable misses
- if (matchStats.misses > maxMisses) {
- iterator.remove();
- continue;
- }
- // Check if this is the best match so far
- if (matchStats.hits > bestMatchHits) {
- bestMatch = candidate;
- }
- }
- }
- if (bestMatch != null) {
- // Update the last access time for this node
- bestMatch.updateLastAccessTime();
- }
- return bestMatch;
- }
-
- /**
- * Add a new node to the cache; may trigger the cleaner thread if the cache is near full capacity.
- * We'll however add the node even if we temporaily exceed maxCacheNodes, because the cleaner
- * will eventually create space from expired nodes or by removing LRU nodes.
- *
- * @param dbName
- * @param tblName
- * @param colName
- * @param numPartsCached
- * @param colStats
- * @param bloomFilter
- */
- // TODO: make add asynchronous: add shouldn't block the higher level calls
- public void add(String dbName, String tblName, String colName, long numPartsCached,
- ColumnStatisticsObj colStats, BloomFilter bloomFilter) {
- // If we have no space in the cache, run cleaner thread
- if (getCurrentNodes() / maxCacheNodes > maxFull) {
- spawnCleaner();
- }
- // Cache key
- Key key = new Key(dbName, tblName, colName);
- // Add new node to the cache
- AggrColStats node = new AggrColStats(numPartsCached, bloomFilter, colStats);
- AggrColStatsList nodeList;
- AggrColStatsList newNodeList = new AggrColStatsList();
- newNodeList.nodes = new ArrayList<AggrColStats>();
- nodeList = cacheStore.putIfAbsent(key, newNodeList);
- if (nodeList == null) {
- nodeList = newNodeList;
- }
- boolean isLocked = false;
- try {
- isLocked = nodeList.writeLock.tryLock(maxWriterWaitTime, TimeUnit.MILLISECONDS);
- if (isLocked) {
- nodeList.nodes.add(node);
- node.updateLastAccessTime();
- nodeList.updateLastAccessTime();
- currentNodes.getAndIncrement();
- }
- } catch (InterruptedException e) {
- LOG.debug("Interrupted Exception ignored ", e);
- } finally {
- if (isLocked) {
- nodeList.writeLock.unlock();
- }
- }
- }
-
- /**
- * Cleans the expired nodes or removes LRU nodes of the cache,
- * until the cache size reduces to cleanUntil% full.
- */
- private void spawnCleaner() {
- // This spawns a separate thread to walk through the cache and removes expired nodes.
- // Only one cleaner thread should be running at any point.
- synchronized (this) {
- if (isCleaning) {
- return;
- }
- isCleaning = true;
- }
- Thread cleaner = new Thread("AggregateStatsCache-CleanerThread") {
- @Override
- public void run() {
- numRemovedTTL = 0;
- numRemovedLRU = 0;
- long cleanerStartTime = System.currentTimeMillis();
- LOG.info("AggregateStatsCache is " + getFullPercent() + "% full, with "
- + getCurrentNodes() + " nodes; starting cleaner thread");
- try {
- Iterator<Map.Entry<Key, AggrColStatsList>> mapIterator = cacheStore.entrySet().iterator();
- while (mapIterator.hasNext()) {
- Map.Entry<Key, AggrColStatsList> pair =
- mapIterator.next();
- AggrColStats node;
- AggrColStatsList candidateList = pair.getValue();
- List<AggrColStats> nodes = candidateList.nodes;
- if (nodes.size() == 0) {
- mapIterator.remove();
- continue;
- }
- boolean isLocked = false;
- try {
- isLocked = candidateList.writeLock.tryLock(maxWriterWaitTime, TimeUnit.MILLISECONDS);
- if (isLocked) {
- for (Iterator<AggrColStats> listIterator = nodes.iterator(); listIterator.hasNext();) {
- node = listIterator.next();
- // Remove the node if it has expired
- if (isExpired(node)) {
- listIterator.remove();
- numRemovedTTL++;
- currentNodes.getAndDecrement();
- }
- }
- }
- } catch (InterruptedException e) {
- LOG.debug("Interrupted Exception ignored ",e);
- } finally {
- if (isLocked) {
- candidateList.writeLock.unlock();
- }
- }
- // We want to make sure this runs at a low priority in the background
- Thread.yield();
- }
- // If the expired nodes did not result in cache being cleanUntil% in size,
- // start removing LRU nodes
- while (getCurrentNodes() / maxCacheNodes > cleanUntil) {
- evictOneNode();
- }
- } finally {
- isCleaning = false;
- LOG.info("Stopping cleaner thread; AggregateStatsCache is now " + getFullPercent()
- + "% full, with " + getCurrentNodes() + " nodes");
- LOG.info("Number of expired nodes removed: " + numRemovedTTL);
- LOG.info("Number of LRU nodes removed: " + numRemovedLRU);
- LOG.info("Cleaner ran for: " + (System.currentTimeMillis() - cleanerStartTime) + "ms");
- }
- }
- };
- cleaner.setPriority(Thread.MIN_PRIORITY);
- cleaner.setDaemon(true);
- cleaner.start();
- }
-
- /**
- * Evict an LRU node or expired node whichever we find first
- */
- private void evictOneNode() {
- // Get the LRU key, value
- Key lruKey = null;
- AggrColStatsList lruValue = null;
- for (Map.Entry<Key, AggrColStatsList> entry : cacheStore.entrySet()) {
- Key key = entry.getKey();
- AggrColStatsList value = entry.getValue();
- if (lruKey == null) {
- lruKey = key;
- lruValue = value;
- continue;
- }
- if ((value.lastAccessTime < lruValue.lastAccessTime) && !(value.nodes.isEmpty())) {
- lruKey = key;
- lruValue = value;
- }
- }
- // Now delete a node for this key's list
- AggrColStatsList candidateList = cacheStore.get(lruKey);
- boolean isLocked = false;
- try {
- isLocked = candidateList.writeLock.tryLock(maxWriterWaitTime, TimeUnit.MILLISECONDS);
- if (isLocked) {
- AggrColStats candidate;
- AggrColStats lruNode = null;
- int currentIndex = 0;
- int deleteIndex = 0;
- for (Iterator<AggrColStats> iterator = candidateList.nodes.iterator(); iterator.hasNext();) {
- candidate = iterator.next();
- // Since we have to create space for 1, if we find an expired node we will remove it &
- // return
- if (isExpired(candidate)) {
- iterator.remove();
- currentNodes.getAndDecrement();
- numRemovedTTL++;
- return;
- }
- // Sorry, too many ifs but this form looks optimal
- // Update the LRU node from what we've seen so far
- if (lruNode == null) {
- lruNode = candidate;
- ++currentIndex;
- continue;
- }
- if (lruNode != null) {
- if (candidate.lastAccessTime < lruNode.lastAccessTime) {
- lruNode = candidate;
- deleteIndex = currentIndex;
- }
- }
- }
- candidateList.nodes.remove(deleteIndex);
- currentNodes.getAndDecrement();
- numRemovedLRU++;
- }
- } catch (InterruptedException e) {
- LOG.debug("Interrupted Exception ignored ",e);
- } finally {
- if (isLocked) {
- candidateList.writeLock.unlock();
- }
- }
- }
-
- private boolean isExpired(AggrColStats aggrColStats) {
- return (System.currentTimeMillis() - aggrColStats.lastAccessTime) > timeToLiveMs;
- }
-
- /**
- * Key object for the stats cache hashtable
- */
- static class Key {
- private final String dbName;
- private final String tblName;
- private final String colName;
-
- Key(String db, String table, String col) {
- // Don't construct an illegal cache key
- if ((db == null) || (table == null) || (col == null)) {
- throw new IllegalArgumentException("dbName, tblName, colName can't be null");
- }
- dbName = db;
- tblName = table;
- colName = col;
- }
-
- @Override
- public boolean equals(Object other) {
- if ((other == null) || !(other instanceof Key)) {
- return false;
- }
- Key that = (Key) other;
- return dbName.equals(that.dbName) && tblName.equals(that.tblName)
- && colName.equals(that.colName);
- }
-
- @Override
- public int hashCode() {
- return dbName.hashCode() * 31 + tblName.hashCode() * 31 + colName.hashCode();
- }
-
- @Override
- public String toString() {
- return "database:" + dbName + ", table:" + tblName + ", column:" + colName;
- }
-
- }
-
- static class AggrColStatsList {
- // TODO: figure out a better data structure for node list(?)
- private List<AggrColStats> nodes = new ArrayList<AggrColStats>();
- private final ReadWriteLock lock = new ReentrantReadWriteLock();
- // Read lock for get operation
- private final Lock readLock = lock.readLock();
- // Write lock for add, evict and clean operation
- private final Lock writeLock = lock.writeLock();
- // Using volatile instead of locking updates to this variable,
- // since we can rely on approx lastAccessTime but don't want a performance hit
- private volatile long lastAccessTime = 0;
-
- List<AggrColStats> getNodes() {
- return nodes;
- }
-
- void updateLastAccessTime() {
- this.lastAccessTime = System.currentTimeMillis();
- }
- }
-
- public static class AggrColStats {
- private final long numPartsCached;
- private final BloomFilter bloomFilter;
- private final ColumnStatisticsObj colStats;
- private volatile long lastAccessTime;
-
- public AggrColStats(long numPartsCached, BloomFilter bloomFilter,
- ColumnStatisticsObj colStats) {
- this.numPartsCached = numPartsCached;
- this.bloomFilter = bloomFilter;
- this.colStats = colStats;
- this.lastAccessTime = System.currentTimeMillis();
- }
-
- public long getNumPartsCached() {
- return numPartsCached;
- }
-
- public ColumnStatisticsObj getColStats() {
- return colStats;
- }
-
- public BloomFilter getBloomFilter() {
- return bloomFilter;
- }
-
- void updateLastAccessTime() {
- this.lastAccessTime = System.currentTimeMillis();
- }
- }
-
- /**
- * Intermediate object, used to collect hits & misses for each cache node that is evaluate for an
- * incoming request
- */
- private static class MatchStats {
- private int hits = 0;
- private int misses = 0;
-
- MatchStats(int hits, int misses) {
- this.hits = hits;
- this.misses = misses;
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/hive/blob/b0b6db73/metastore/src/java/org/apache/hadoop/hive/metastore/DatabaseProduct.java
----------------------------------------------------------------------
diff --git a/metastore/src/java/org/apache/hadoop/hive/metastore/DatabaseProduct.java b/metastore/src/java/org/apache/hadoop/hive/metastore/DatabaseProduct.java
deleted file mode 100644
index 7634852..0000000
--- a/metastore/src/java/org/apache/hadoop/hive/metastore/DatabaseProduct.java
+++ /dev/null
@@ -1,75 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.hive.metastore;
-
-import java.sql.SQLException;
-import java.sql.SQLTransactionRollbackException;
-
-/** Database product infered via JDBC. */
-public enum DatabaseProduct {
- DERBY, MYSQL, POSTGRES, ORACLE, SQLSERVER, OTHER;
-
- /**
- * Determine the database product type
- * @param productName string to defer database connection
- * @return database product type
- */
- public static DatabaseProduct determineDatabaseProduct(String productName) throws SQLException {
- if (productName == null) {
- return OTHER;
- }
- productName = productName.toLowerCase();
- if (productName.contains("derby")) {
- return DERBY;
- } else if (productName.contains("microsoft sql server")) {
- return SQLSERVER;
- } else if (productName.contains("mysql")) {
- return MYSQL;
- } else if (productName.contains("oracle")) {
- return ORACLE;
- } else if (productName.contains("postgresql")) {
- return POSTGRES;
- } else {
- return OTHER;
- }
- }
-
- public static boolean isDeadlock(DatabaseProduct dbProduct, SQLException e) {
- return e instanceof SQLTransactionRollbackException
- || ((dbProduct == MYSQL || dbProduct == POSTGRES || dbProduct == SQLSERVER)
- && e.getSQLState().equals("40001"))
- || (dbProduct == POSTGRES && e.getSQLState().equals("40P01"))
- || (dbProduct == ORACLE && (e.getMessage().contains("deadlock detected")
- || e.getMessage().contains("can't serialize access for this transaction")));
- }
-
- /**
- * Whether the RDBMS has restrictions on IN list size (explicit, or poor perf-based).
- */
- public static boolean needsInBatching(DatabaseProduct dbType) {
- return dbType == ORACLE || dbType == SQLSERVER;
- }
-
- /**
- * Whether the RDBMS has a bug in join and filter operation order described in DERBY-6358.
- */
- public static boolean hasJoinOperationOrderBug(DatabaseProduct dbType) {
- return dbType == DERBY || dbType == ORACLE;
- }
-};
http://git-wip-us.apache.org/repos/asf/hive/blob/b0b6db73/metastore/src/java/org/apache/hadoop/hive/metastore/DefaultHiveMetaHook.java
----------------------------------------------------------------------
diff --git a/metastore/src/java/org/apache/hadoop/hive/metastore/DefaultHiveMetaHook.java b/metastore/src/java/org/apache/hadoop/hive/metastore/DefaultHiveMetaHook.java
deleted file mode 100644
index 0957945..0000000
--- a/metastore/src/java/org/apache/hadoop/hive/metastore/DefaultHiveMetaHook.java
+++ /dev/null
@@ -1,51 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.hive.metastore;
-
-import org.apache.hadoop.hive.metastore.api.MetaException;
-import org.apache.hadoop.hive.metastore.api.Table;
-
-public abstract class DefaultHiveMetaHook implements HiveMetaHook {
- /**
- * Called after successfully after INSERT [OVERWRITE] statement is executed.
- * @param table table definition
- * @param overwrite true if it is INSERT OVERWRITE
- *
- * @throws MetaException
- */
- public abstract void commitInsertTable(Table table, boolean overwrite) throws MetaException;
-
- /**
- * called before commit insert method is called
- * @param table table definition
- * @param overwrite true if it is INSERT OVERWRITE
- *
- * @throws MetaException
- */
- public abstract void preInsertTable(Table table, boolean overwrite) throws MetaException;
-
- /**
- * called in case pre commit or commit insert fail.
- * @param table table definition
- * @param overwrite true if it is INSERT OVERWRITE
- *
- * @throws MetaException
- */
- public abstract void rollbackInsertTable(Table table, boolean overwrite) throws MetaException;
-}
http://git-wip-us.apache.org/repos/asf/hive/blob/b0b6db73/metastore/src/java/org/apache/hadoop/hive/metastore/DefaultMetaStoreFilterHookImpl.java
----------------------------------------------------------------------
diff --git a/metastore/src/java/org/apache/hadoop/hive/metastore/DefaultMetaStoreFilterHookImpl.java b/metastore/src/java/org/apache/hadoop/hive/metastore/DefaultMetaStoreFilterHookImpl.java
deleted file mode 100644
index f0cab26..0000000
--- a/metastore/src/java/org/apache/hadoop/hive/metastore/DefaultMetaStoreFilterHookImpl.java
+++ /dev/null
@@ -1,102 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.hive.metastore;
-
-import java.util.List;
-
-import org.apache.hadoop.hive.conf.HiveConf;
-import org.apache.hadoop.hive.metastore.api.Database;
-import org.apache.hadoop.hive.metastore.api.Index;
-import org.apache.hadoop.hive.metastore.api.MetaException;
-import org.apache.hadoop.hive.metastore.api.NoSuchObjectException;
-import org.apache.hadoop.hive.metastore.api.Partition;
-import org.apache.hadoop.hive.metastore.api.PartitionSpec;
-import org.apache.hadoop.hive.metastore.api.Table;
-
-/**
- * Default no-op implementation of the MetaStoreFilterHook that returns the result as is
- */
-public class DefaultMetaStoreFilterHookImpl implements MetaStoreFilterHook {
-
- public DefaultMetaStoreFilterHookImpl(HiveConf conf) {
- }
-
- @Override
- public List<String> filterDatabases(List<String> dbList) throws MetaException {
- return dbList;
- }
-
- @Override
- public Database filterDatabase(Database dataBase) throws NoSuchObjectException {
- return dataBase;
- }
-
- @Override
- public List<String> filterTableNames(String dbName, List<String> tableList) throws MetaException {
- return tableList;
- }
-
- @Override
- public Table filterTable(Table table) throws NoSuchObjectException {
- return table;
- }
-
- @Override
- public List<Table> filterTables(List<Table> tableList) throws MetaException {
- return tableList;
- }
-
- @Override
- public List<Partition> filterPartitions(List<Partition> partitionList) throws MetaException {
- return partitionList;
- }
-
- @Override
- public List<PartitionSpec> filterPartitionSpecs(
- List<PartitionSpec> partitionSpecList) throws MetaException {
- return partitionSpecList;
- }
-
- @Override
- public Partition filterPartition(Partition partition) throws NoSuchObjectException {
- return partition;
- }
-
- @Override
- public List<String> filterPartitionNames(String dbName, String tblName,
- List<String> partitionNames) throws MetaException {
- return partitionNames;
- }
-
- @Override
- public Index filterIndex(Index index) throws NoSuchObjectException {
- return index;
- }
-
- @Override
- public List<String> filterIndexNames(String dbName, String tblName,
- List<String> indexList) throws MetaException {
- return indexList;
- }
-
- @Override
- public List<Index> filterIndexes(List<Index> indexeList) throws MetaException {
- return indexeList;
- }
-}
http://git-wip-us.apache.org/repos/asf/hive/blob/b0b6db73/metastore/src/java/org/apache/hadoop/hive/metastore/FileFormatProxy.java
----------------------------------------------------------------------
diff --git a/metastore/src/java/org/apache/hadoop/hive/metastore/FileFormatProxy.java b/metastore/src/java/org/apache/hadoop/hive/metastore/FileFormatProxy.java
deleted file mode 100644
index 14ff187..0000000
--- a/metastore/src/java/org/apache/hadoop/hive/metastore/FileFormatProxy.java
+++ /dev/null
@@ -1,64 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.hive.metastore;
-
-import java.io.IOException;
-import java.nio.ByteBuffer;
-import java.util.List;
-
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.hive.metastore.Metastore.SplitInfos;
-import org.apache.hadoop.hive.ql.io.sarg.SearchArgument;
-
-/**
- * Same as PartitionExpressionProxy, but for file format specific methods for metadata cache.
- */
-public interface FileFormatProxy {
-
- /**
- * Applies SARG to file metadata, and produces some result for this file.
- * @param sarg SARG
- * @param fileMetadata File metadata from metastore cache.
- * @return The result to return to client for this file, or null if file is eliminated.
- */
- SplitInfos applySargToMetadata(SearchArgument sarg, ByteBuffer fileMetadata) throws IOException;
-
- /**
- * @param fs The filesystem of the file.
- * @param path The file path.
- * @param addedVals Output parameter; additional column values for columns returned by
- * getAddedColumnsToCache to cache in MS.
- * @return The ORC file metadata for a given file.
- */
- ByteBuffer getMetadataToCache(
- FileSystem fs, Path path, ByteBuffer[] addedVals) throws IOException;
-
- /**
- * @return Additional column names to cache in MS for this format.
- */
- ByteBuffer[] getAddedColumnsToCache();
-
- /**
- * @param metadata File metadatas.
- * @return Additional values for columns returned by getAddedColumnsToCache to cache in MS
- * for respective metadatas.
- */
- ByteBuffer[][] getAddedValuesToCache(List<ByteBuffer> metadata);
-
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hive/blob/b0b6db73/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaException.java
----------------------------------------------------------------------
diff --git a/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaException.java b/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaException.java
deleted file mode 100644
index 05cc1fa..0000000
--- a/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaException.java
+++ /dev/null
@@ -1,42 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.hive.metastore;
-
-public class HiveMetaException extends Exception {
-
- public HiveMetaException() {
- // TODO Auto-generated constructor stub
- }
-
- public HiveMetaException(String message) {
- super(message);
- // TODO Auto-generated constructor stub
- }
-
- public HiveMetaException(Throwable cause) {
- super(cause);
- // TODO Auto-generated constructor stub
- }
-
- public HiveMetaException(String message, Throwable cause) {
- super(message, cause);
- // TODO Auto-generated constructor stub
- }
-
-}
http://git-wip-us.apache.org/repos/asf/hive/blob/b0b6db73/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaHook.java
----------------------------------------------------------------------
diff --git a/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaHook.java b/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaHook.java
deleted file mode 100644
index 3317bc5..0000000
--- a/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaHook.java
+++ /dev/null
@@ -1,96 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.hive.metastore;
-
-import org.apache.hadoop.hive.common.classification.InterfaceAudience;
-import org.apache.hadoop.hive.common.classification.InterfaceStability;
-import org.apache.hadoop.hive.metastore.api.MetaException;
-import org.apache.hadoop.hive.metastore.api.Partition;
-import org.apache.hadoop.hive.metastore.api.Table;
-
-/**
- * HiveMetaHook defines notification methods which are invoked as part
- * of transactions against the metastore, allowing external catalogs
- * such as HBase to be kept in sync with Hive's metastore.
- *
- *<p>
- *
- * Implementations can use {@link MetaStoreUtils#isExternalTable} to
- * distinguish external tables from managed tables.
- */
-@InterfaceAudience.Public
-@InterfaceStability.Stable
-public interface HiveMetaHook {
- /**
- * Called before a new table definition is added to the metastore
- * during CREATE TABLE.
- *
- * @param table new table definition
- */
- public void preCreateTable(Table table)
- throws MetaException;
-
- /**
- * Called after failure adding a new table definition to the metastore
- * during CREATE TABLE.
- *
- * @param table new table definition
- */
- public void rollbackCreateTable(Table table)
- throws MetaException;
-
- /**
- * Called after successfully adding a new table definition to the metastore
- * during CREATE TABLE.
- *
- * @param table new table definition
- */
- public void commitCreateTable(Table table)
- throws MetaException;
-
- /**
- * Called before a table definition is removed from the metastore
- * during DROP TABLE.
- *
- * @param table table definition
- */
- public void preDropTable(Table table)
- throws MetaException;
-
- /**
- * Called after failure removing a table definition from the metastore
- * during DROP TABLE.
- *
- * @param table table definition
- */
- public void rollbackDropTable(Table table)
- throws MetaException;
-
- /**
- * Called after successfully removing a table definition from the metastore
- * during DROP TABLE.
- *
- * @param table table definition
- *
- * @param deleteData whether to delete data as well; this should typically
- * be ignored in the case of an external table
- */
- public void commitDropTable(Table table, boolean deleteData)
- throws MetaException;
-}
http://git-wip-us.apache.org/repos/asf/hive/blob/b0b6db73/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaHookLoader.java
----------------------------------------------------------------------
diff --git a/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaHookLoader.java b/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaHookLoader.java
deleted file mode 100644
index 1cdae9b..0000000
--- a/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaHookLoader.java
+++ /dev/null
@@ -1,39 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.hive.metastore;
-
-import org.apache.hadoop.hive.metastore.api.MetaException;
-import org.apache.hadoop.hive.metastore.api.Table;
-
-/**
- * HiveMetaHookLoader is responsible for loading a {@link HiveMetaHook}
- * for a given table.
- */
-public interface HiveMetaHookLoader {
- /**
- * Loads a hook for the specified table.
- *
- * @param tbl table of interest
- *
- * @return hook, or null if none registered
- */
- public HiveMetaHook getHook(Table tbl) throws MetaException;
-}
-
-// End HiveMetaHookLoader.java
http://git-wip-us.apache.org/repos/asf/hive/blob/b0b6db73/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java
----------------------------------------------------------------------
diff --git a/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java b/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java
index 53cb3d9..cf33cca 100644
--- a/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java
+++ b/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java
@@ -7907,7 +7907,7 @@ public class HiveMetaStore extends ThriftHiveMetastore {
private static void initializeAndStartThread(MetaStoreThread thread, HiveConf conf) throws
MetaException {
LOG.info("Starting metastore thread of type " + thread.getClass().getName());
- thread.setHiveConf(conf);
+ thread.setConf(conf);
thread.setThreadId(nextThreadId++);
thread.init(new AtomicBoolean(), new AtomicBoolean());
thread.start();
http://git-wip-us.apache.org/repos/asf/hive/blob/b0b6db73/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStoreClient.java
----------------------------------------------------------------------
diff --git a/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStoreClient.java b/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStoreClient.java
index 751dd91..75e4180 100644
--- a/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStoreClient.java
+++ b/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStoreClient.java
@@ -49,6 +49,7 @@ import java.security.PrivilegedExceptionAction;
import javax.security.auth.login.LoginException;
+import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hive.common.ObjectPair;
import org.apache.hadoop.hive.common.StatsSetupConst;
import org.apache.hadoop.hive.common.ValidTxnList;
@@ -263,7 +264,7 @@ public class HiveMetaStoreClient implements IMetaStoreClient, AutoCloseable {
String msg = "Unable to create instance of " + authProviderClass.getName() + ": ";
try {
Constructor<? extends MetaStoreFilterHook> constructor =
- authProviderClass.getConstructor(HiveConf.class);
+ authProviderClass.getConstructor(Configuration.class);
return constructor.newInstance(conf);
} catch (NoSuchMethodException e) {
throw new IllegalStateException(msg + e.getMessage(), e);
http://git-wip-us.apache.org/repos/asf/hive/blob/b0b6db73/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStoreFsImpl.java
----------------------------------------------------------------------
diff --git a/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStoreFsImpl.java b/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStoreFsImpl.java
deleted file mode 100644
index b7d7b50..0000000
--- a/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStoreFsImpl.java
+++ /dev/null
@@ -1,52 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.hive.metastore;
-
-import java.io.FileNotFoundException;
-
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.hive.common.FileUtils;
-import org.apache.hadoop.hive.metastore.api.MetaException;
-
-public class HiveMetaStoreFsImpl implements MetaStoreFS {
-
- public static final Logger LOG = LoggerFactory
- .getLogger("hive.metastore.hivemetastoreFsimpl");
-
- @Override
- public boolean deleteDir(FileSystem fs, Path f, boolean recursive,
- boolean ifPurge, Configuration conf) throws MetaException {
- try {
- FileUtils.moveToTrash(fs, f, conf, ifPurge);
- if (fs.exists(f)) {
- throw new MetaException("Unable to delete directory: " + f);
- }
- return true;
- } catch (FileNotFoundException e) {
- return true; // ok even if there is not data
- } catch (Exception e) {
- MetaStoreUtils.logAndThrowMetaException(e);
- }
- return false;
- }
-}
http://git-wip-us.apache.org/repos/asf/hive/blob/b0b6db73/metastore/src/java/org/apache/hadoop/hive/metastore/IExtrapolatePartStatus.java
----------------------------------------------------------------------
diff --git a/metastore/src/java/org/apache/hadoop/hive/metastore/IExtrapolatePartStatus.java b/metastore/src/java/org/apache/hadoop/hive/metastore/IExtrapolatePartStatus.java
deleted file mode 100644
index d0569fb..0000000
--- a/metastore/src/java/org/apache/hadoop/hive/metastore/IExtrapolatePartStatus.java
+++ /dev/null
@@ -1,85 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.hive.metastore;
-
-import java.util.HashMap;
-import java.util.Map;
-
-public interface IExtrapolatePartStatus {
- /**
- * The sequence of colStatNames.
- */
- static String[] colStatNames = new String[] { "LONG_LOW_VALUE", "LONG_HIGH_VALUE",
- "DOUBLE_LOW_VALUE", "DOUBLE_HIGH_VALUE", "BIG_DECIMAL_LOW_VALUE", "BIG_DECIMAL_HIGH_VALUE",
- "NUM_NULLS", "NUM_DISTINCTS", "AVG_COL_LEN", "MAX_COL_LEN", "NUM_TRUES", "NUM_FALSES",
- "AVG_NDV_LONG", "AVG_NDV_DOUBLE", "AVG_NDV_DECIMAL", "SUM_NUM_DISTINCTS" };
-
- /**
- * The indexes for colstats.
- */
- static HashMap<String, Integer[]> indexMaps = new HashMap<String, Integer[]>() {
- {
- put("bigint", new Integer[] { 0, 1, 6, 7, 12, 15 });
- put("int", new Integer[] { 0, 1, 6, 7, 12, 15 });
- put("smallint", new Integer[] { 0, 1, 6, 7, 12, 15 });
- put("tinyint", new Integer[] { 0, 1, 6, 7, 12, 15 });
- put("date", new Integer[] { 0, 1, 6, 7, 12, 15 });
- put("timestamp", new Integer[] { 0, 1, 6, 7, 12, 15 });
- put("long", new Integer[] { 0, 1, 6, 7, 12, 15 });
- put("double", new Integer[] { 2, 3, 6, 7, 13, 15 });
- put("float", new Integer[] { 2, 3, 6, 7, 13, 15 });
- put("varchar", new Integer[] { 8, 9, 6, 7, 15 });
- put("char", new Integer[] { 8, 9, 6, 7, 15 });
- put("string", new Integer[] { 8, 9, 6, 7, 15 });
- put("boolean", new Integer[] { 10, 11, 6, 15 });
- put("binary", new Integer[] { 8, 9, 6, 15 });
- put("decimal", new Integer[] { 4, 5, 6, 7, 14, 15 });
- put("default", new Integer[] { 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 15 });
- }
- };
-
- /**
- * The sequence of colStatTypes.
- */
- static enum ColStatType {
- Long, Double, Decimal
- }
-
- static ColStatType[] colStatTypes = new ColStatType[] { ColStatType.Long, ColStatType.Long,
- ColStatType.Double, ColStatType.Double, ColStatType.Decimal, ColStatType.Decimal,
- ColStatType.Long, ColStatType.Long, ColStatType.Double, ColStatType.Long, ColStatType.Long,
- ColStatType.Long, ColStatType.Double, ColStatType.Double, ColStatType.Double,
- ColStatType.Long };
-
- /**
- * The sequence of aggregation function on colStats.
- */
- static enum AggrType {
- Min, Max, Sum, Avg
- }
-
- static AggrType[] aggrTypes = new AggrType[] { AggrType.Min, AggrType.Max, AggrType.Min,
- AggrType.Max, AggrType.Min, AggrType.Max, AggrType.Sum, AggrType.Max, AggrType.Max,
- AggrType.Max, AggrType.Sum, AggrType.Sum, AggrType.Avg, AggrType.Avg, AggrType.Avg,
- AggrType.Sum };
-
- public Object extrapolate(Object[] min, Object[] max, int colStatIndex,
- Map<String, Integer> indexMap);
-
-}
http://git-wip-us.apache.org/repos/asf/hive/blob/b0b6db73/metastore/src/java/org/apache/hadoop/hive/metastore/IHMSHandler.java
----------------------------------------------------------------------
diff --git a/metastore/src/java/org/apache/hadoop/hive/metastore/IHMSHandler.java b/metastore/src/java/org/apache/hadoop/hive/metastore/IHMSHandler.java
deleted file mode 100644
index cc7ae1d..0000000
--- a/metastore/src/java/org/apache/hadoop/hive/metastore/IHMSHandler.java
+++ /dev/null
@@ -1,28 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.hive.metastore;
-
-import org.apache.hadoop.conf.Configurable;
-import org.apache.hadoop.hive.metastore.api.MetaException;
-import org.apache.hadoop.hive.metastore.api.ThriftHiveMetastore;
-
-public interface IHMSHandler extends ThriftHiveMetastore.Iface, Configurable {
-
- void init() throws MetaException;
-}
http://git-wip-us.apache.org/repos/asf/hive/blob/b0b6db73/metastore/src/java/org/apache/hadoop/hive/metastore/IMetaStoreSchemaInfo.java
----------------------------------------------------------------------
diff --git a/metastore/src/java/org/apache/hadoop/hive/metastore/IMetaStoreSchemaInfo.java b/metastore/src/java/org/apache/hadoop/hive/metastore/IMetaStoreSchemaInfo.java
deleted file mode 100644
index 0a5f978..0000000
--- a/metastore/src/java/org/apache/hadoop/hive/metastore/IMetaStoreSchemaInfo.java
+++ /dev/null
@@ -1,107 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.hive.metastore;
-
-import org.apache.hadoop.hive.common.classification.InterfaceAudience;
-
-import java.sql.Connection;
-import java.util.List;
-import org.apache.hadoop.hive.metastore.tools.HiveSchemaHelper;
-
-/**
- * Defines the method which must be implemented to be used using schema tool to support metastore
- * schema upgrades. The configuration hive.metastore.schema.info.class is used to create instances
- * of this type by SchemaTool.
- *
- * Instances of this interface should be created using MetaStoreSchemaInfoFactory class which uses
- * two Strings argument constructor to instantiate the implementations of this interface
- */
-@InterfaceAudience.Private
-public interface IMetaStoreSchemaInfo {
- String SQL_FILE_EXTENSION = ".sql";
-
- /***
- * Get the list of sql scripts required to upgrade from the give version to current.
- *
- * @param fromVersion
- * @return
- * @throws HiveMetaException
- */
- List<String> getUpgradeScripts(String fromVersion) throws HiveMetaException;
-
- /***
- * Get the name of the script to initialize the schema for given version
- *
- * @param toVersion Target version. If it's null, then the current server version is used
- * @return
- * @throws HiveMetaException
- */
- String generateInitFileName(String toVersion) throws HiveMetaException;
-
- /**
- * Find the directory of metastore scripts
- *
- * @return the path of directory where the sql scripts are
- */
- String getMetaStoreScriptDir();
-
- /**
- * Get the pre-upgrade script for a given script name. Schema tool runs the pre-upgrade scripts
- * returned by this method before running any upgrade scripts. These scripts could contain setup
- * statements may fail on some database versions and failure is ignorable.
- *
- * @param index - index number of the file. The preupgrade script name is derived using the given
- * index
- * @param scriptName - upgrade script name
- * @return name of the pre-upgrade script to be run before running upgrade script
- */
- String getPreUpgradeScriptName(int index, String scriptName);
-
- /**
- * Get hive distribution schema version. Schematool uses this version to identify
- * the Hive version. It compares this version with the version found in metastore database
- * to determine the upgrade or initialization scripts
- * @return Hive schema version
- */
- String getHiveSchemaVersion();
-
- /**
- * Get the schema version from the backend database. This version is used by SchemaTool to to
- * compare the version returned by getHiveSchemaVersion and determine the upgrade order and
- * scripts needed to upgrade the metastore schema
- *
- * @param metastoreDbConnectionInfo Connection information needed to connect to the backend
- * database
- * @return
- * @throws HiveMetaException when unable to fetch the schema version
- */
- String getMetaStoreSchemaVersion(
- HiveSchemaHelper.MetaStoreConnectionInfo metastoreDbConnectionInfo) throws HiveMetaException;
- /**
- * A dbVersion is compatible with hive version if it is greater or equal to the hive version. This
- * is result of the db schema upgrade design principles followed in hive project. The state where
- * db schema version is ahead of hive software version is often seen when a 'rolling upgrade' or
- * 'rolling downgrade' is happening. This is a state where hive is functional and returning non
- * zero status for it is misleading.
- *
- * @param productVersion version of hive software
- * @param dbVersion version of metastore rdbms schema
- * @return true if versions are compatible
- */
- boolean isVersionCompatible(String productVersion, String dbVersion);
-}
http://git-wip-us.apache.org/repos/asf/hive/blob/b0b6db73/metastore/src/java/org/apache/hadoop/hive/metastore/LinearExtrapolatePartStatus.java
----------------------------------------------------------------------
diff --git a/metastore/src/java/org/apache/hadoop/hive/metastore/LinearExtrapolatePartStatus.java b/metastore/src/java/org/apache/hadoop/hive/metastore/LinearExtrapolatePartStatus.java
deleted file mode 100644
index f4e5ef7..0000000
--- a/metastore/src/java/org/apache/hadoop/hive/metastore/LinearExtrapolatePartStatus.java
+++ /dev/null
@@ -1,106 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.hive.metastore;
-
-import java.math.BigDecimal;
-import java.util.Map;
-
-public class LinearExtrapolatePartStatus implements IExtrapolatePartStatus {
-
- @Override
- public Object extrapolate(Object[] min, Object[] max, int colStatIndex,
- Map<String, Integer> indexMap) {
- int rightBorderInd = indexMap.size() - 1;
- int minInd = indexMap.get((String) min[1]);
- int maxInd = indexMap.get((String) max[1]);
- if (minInd == maxInd) {
- return min[0];
- }
- //note that recent metastore stores decimal in string.
- double decimalmin= 0;
- double decimalmax = 0;
- if (colStatTypes[colStatIndex] == ColStatType.Decimal) {
- BigDecimal bdmin = new BigDecimal(min[0].toString());
- decimalmin = bdmin.doubleValue();
- BigDecimal bdmax = new BigDecimal(max[0].toString());
- decimalmax = bdmax.doubleValue();
- }
- if (aggrTypes[colStatIndex] == AggrType.Max) {
- if (minInd < maxInd) {
- // right border is the max
- if (colStatTypes[colStatIndex] == ColStatType.Long) {
- return (Long) ((Long) min[0] + (((Long) max[0] - (Long) min[0])
- * (rightBorderInd - minInd) / (maxInd - minInd)));
- } else if (colStatTypes[colStatIndex] == ColStatType.Double) {
- return (Double) ((Double) min[0] + (((Double) max[0] - (Double) min[0])
- * (rightBorderInd - minInd) / (maxInd - minInd)));
- } else {
- double ret = decimalmin + (decimalmax - decimalmin)
- * (rightBorderInd - minInd) / (maxInd - minInd);
- return String.valueOf(ret);
- }
- } else {
- // left border is the max
- if (colStatTypes[colStatIndex] == ColStatType.Long) {
- return (Long) ((Long) min[0] + ((Long) max[0] - (Long) min[0])
- * minInd / (minInd - maxInd));
- } else if (colStatTypes[colStatIndex] == ColStatType.Double) {
- return (Double) ((Double) min[0] + ((Double) max[0] - (Double) min[0])
- * minInd / (minInd - maxInd));
- } else {
- double ret = decimalmin + (decimalmax - decimalmin) * minInd
- / (minInd - maxInd);
- return String.valueOf(ret);
- }
- }
- } else {
- if (minInd < maxInd) {
- // left border is the min
- if (colStatTypes[colStatIndex] == ColStatType.Long) {
- Long ret = (Long) max[0] - ((Long) max[0] - (Long) min[0]) * maxInd
- / (maxInd - minInd);
- return ret;
- } else if (colStatTypes[colStatIndex] == ColStatType.Double) {
- Double ret = (Double) max[0] - ((Double) max[0] - (Double) min[0])
- * maxInd / (maxInd - minInd);
- return ret;
- } else {
- double ret = decimalmax - (decimalmax - decimalmin) * maxInd
- / (maxInd - minInd);
- return String.valueOf(ret);
- }
- } else {
- // right border is the min
- if (colStatTypes[colStatIndex] == ColStatType.Long) {
- Long ret = (Long) max[0] - ((Long) max[0] - (Long) min[0])
- * (rightBorderInd - maxInd) / (minInd - maxInd);
- return ret;
- } else if (colStatTypes[colStatIndex] == ColStatType.Double) {
- Double ret = (Double) max[0] - ((Double) max[0] - (Double) min[0])
- * (rightBorderInd - maxInd) / (minInd - maxInd);
- return ret;
- } else {
- double ret = decimalmax - (decimalmax - decimalmin)
- * (rightBorderInd - maxInd) / (minInd - maxInd);
- return String.valueOf(ret);
- }
- }
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/hive/blob/b0b6db73/metastore/src/java/org/apache/hadoop/hive/metastore/LockComponentBuilder.java
----------------------------------------------------------------------
diff --git a/metastore/src/java/org/apache/hadoop/hive/metastore/LockComponentBuilder.java b/metastore/src/java/org/apache/hadoop/hive/metastore/LockComponentBuilder.java
deleted file mode 100644
index e074152..0000000
--- a/metastore/src/java/org/apache/hadoop/hive/metastore/LockComponentBuilder.java
+++ /dev/null
@@ -1,121 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.hive.metastore;
-
-import org.apache.hadoop.hive.metastore.api.DataOperationType;
-import org.apache.hadoop.hive.metastore.api.LockComponent;
-import org.apache.hadoop.hive.metastore.api.LockLevel;
-import org.apache.hadoop.hive.metastore.api.LockType;
-
-/**
- * A builder for {@link LockComponent}s
- */
-public class LockComponentBuilder {
- private LockComponent component;
- private boolean tableNameSet;
- private boolean partNameSet;
-
- public LockComponentBuilder() {
- component = new LockComponent();
- tableNameSet = partNameSet = false;
- }
-
- /**
- * Set the lock to be exclusive.
- * @return reference to this builder
- */
- public LockComponentBuilder setExclusive() {
- component.setType(LockType.EXCLUSIVE);
- return this;
- }
-
- /**
- * Set the lock to be semi-shared.
- * @return reference to this builder
- */
- public LockComponentBuilder setSemiShared() {
- component.setType(LockType.SHARED_WRITE);
- return this;
- }
-
- /**
- * Set the lock to be shared.
- * @return reference to this builder
- */
- public LockComponentBuilder setShared() {
- component.setType(LockType.SHARED_READ);
- return this;
- }
-
- /**
- * Set the database name.
- * @param dbName database name
- * @return reference to this builder
- */
- public LockComponentBuilder setDbName(String dbName) {
- component.setDbname(dbName);
- return this;
- }
-
- public LockComponentBuilder setOperationType(DataOperationType dop) {
- component.setOperationType(dop);
- return this;
- }
-
- public LockComponentBuilder setIsAcid(boolean t) {
- component.setIsAcid(t);
- return this;
- }
- /**
- * Set the table name.
- * @param tableName table name
- * @return reference to this builder
- */
- public LockComponentBuilder setTableName(String tableName) {
- component.setTablename(tableName);
- tableNameSet = true;
- return this;
- }
-
- /**
- * Set the partition name.
- * @param partitionName partition name
- * @return reference to this builder
- */
- public LockComponentBuilder setPartitionName(String partitionName) {
- component.setPartitionname(partitionName);
- partNameSet = true;
- return this;
- }
- public LockComponentBuilder setIsDynamicPartitionWrite(boolean t) {
- component.setIsDynamicPartitionWrite(t);
- return this;
- }
-
- /**
- * Get the constructed lock component.
- * @return lock component.
- */
- public LockComponent build() {
- LockLevel level = LockLevel.DB;
- if (tableNameSet) level = LockLevel.TABLE;
- if (partNameSet) level = LockLevel.PARTITION;
- component.setLevel(level);
- return component;
- }
-}
http://git-wip-us.apache.org/repos/asf/hive/blob/b0b6db73/metastore/src/java/org/apache/hadoop/hive/metastore/LockRequestBuilder.java
----------------------------------------------------------------------
diff --git a/metastore/src/java/org/apache/hadoop/hive/metastore/LockRequestBuilder.java b/metastore/src/java/org/apache/hadoop/hive/metastore/LockRequestBuilder.java
deleted file mode 100644
index 00a1907..0000000
--- a/metastore/src/java/org/apache/hadoop/hive/metastore/LockRequestBuilder.java
+++ /dev/null
@@ -1,168 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.hive.metastore;
-
-import org.apache.hadoop.hive.metastore.api.LockComponent;
-import org.apache.hadoop.hive.metastore.api.LockRequest;
-import org.apache.hadoop.hive.metastore.api.LockType;
-
-import java.net.InetAddress;
-import java.net.UnknownHostException;
-import java.util.LinkedHashMap;
-import java.util.Map;
-
-/**
- * Builder class to make constructing {@link LockRequest} easier.
- */
-public class LockRequestBuilder {
-
- private LockRequest req;
- private LockTrie trie;
- private boolean userSet;
-
- /**
- * @deprecated
- */
- public LockRequestBuilder() {
- this(null);
- }
- public LockRequestBuilder(String agentInfo) {
- req = new LockRequest();
- trie = new LockTrie();
- userSet = false;
- if(agentInfo != null) {
- req.setAgentInfo(agentInfo);
- }
- }
-
- /**
- * Get the constructed LockRequest.
- * @return lock request
- */
- public LockRequest build() {
- if (!userSet) {
- throw new RuntimeException("Cannot build a lock without giving a user");
- }
- trie.addLocksToRequest(req);
- try {
- req.setHostname(InetAddress.getLocalHost().getHostName());
- } catch (UnknownHostException e) {
- throw new RuntimeException("Unable to determine our local host!");
- }
- return req;
- }
-
- /**
- * Set the transaction id.
- * @param txnid transaction id
- * @return reference to this builder
- */
- public LockRequestBuilder setTransactionId(long txnid) {
- req.setTxnid(txnid);
- return this;
- }
-
- public LockRequestBuilder setUser(String user) {
- if (user == null) user = "unknown";
- req.setUser(user);
- userSet = true;
- return this;
- }
-
- /**
- * Add a lock component to the lock request
- * @param component to add
- * @return reference to this builder
- */
- public LockRequestBuilder addLockComponent(LockComponent component) {
- trie.add(component);
- return this;
- }
-
- // For reasons that are completely incomprehensible to me the semantic
- // analyzers often ask for multiple locks on the same entity (for example
- // a shared_read and an exlcusive lock). The db locking system gets confused
- // by this and dead locks on it. To resolve that, we'll make sure in the
- // request that multiple locks are coalesced and promoted to the higher
- // level of locking. To do this we put all locks components in trie based
- // on dbname, tablename, partition name and handle the promotion as new
- // requests come in. This structure depends on the fact that null is a
- // valid key in a LinkedHashMap. So a database lock will map to (dbname, null,
- // null).
- private static class LockTrie {
- Map<String, TableTrie> trie;
-
- LockTrie() {
- trie = new LinkedHashMap<String, TableTrie>();
- }
-
- public void add(LockComponent comp) {
- TableTrie tabs = trie.get(comp.getDbname());
- if (tabs == null) {
- tabs = new TableTrie();
- trie.put(comp.getDbname(), tabs);
- }
- setTable(comp, tabs);
- }
-
- public void addLocksToRequest(LockRequest request) {
- for (TableTrie tab : trie.values()) {
- for (PartTrie part : tab.values()) {
- for (LockComponent lock : part.values()) {
- request.addToComponent(lock);
- }
- }
- }
- }
-
- private void setTable(LockComponent comp, TableTrie tabs) {
- PartTrie parts = tabs.get(comp.getTablename());
- if (parts == null) {
- parts = new PartTrie();
- tabs.put(comp.getTablename(), parts);
- }
- setPart(comp, parts);
- }
-
- private void setPart(LockComponent comp, PartTrie parts) {
- LockComponent existing = parts.get(comp.getPartitionname());
- if (existing == null) {
- // No existing lock for this partition.
- parts.put(comp.getPartitionname(), comp);
- } else if (existing.getType() != LockType.EXCLUSIVE &&
- (comp.getType() == LockType.EXCLUSIVE ||
- comp.getType() == LockType.SHARED_WRITE)) {
- // We only need to promote if comp.type is > existing.type. For
- // efficiency we check if existing is exclusive (in which case we
- // need never promote) or if comp is exclusive or shared_write (in
- // which case we can promote even though they may both be shared
- // write). If comp is shared_read there's never a need to promote.
- parts.put(comp.getPartitionname(), comp);
- }
- }
-
- private static class TableTrie extends LinkedHashMap<String, PartTrie> {
- }
-
- private static class PartTrie extends LinkedHashMap<String, LockComponent> {
- }
-
-
-
- }
-}
http://git-wip-us.apache.org/repos/asf/hive/blob/b0b6db73/metastore/src/java/org/apache/hadoop/hive/metastore/MetaStoreDirectSql.java
----------------------------------------------------------------------
diff --git a/metastore/src/java/org/apache/hadoop/hive/metastore/MetaStoreDirectSql.java b/metastore/src/java/org/apache/hadoop/hive/metastore/MetaStoreDirectSql.java
index dc1245e..c1a8efe 100644
--- a/metastore/src/java/org/apache/hadoop/hive/metastore/MetaStoreDirectSql.java
+++ b/metastore/src/java/org/apache/hadoop/hive/metastore/MetaStoreDirectSql.java
@@ -1314,7 +1314,7 @@ class MetaStoreDirectSql {
AggrColStats colStatsAggrCached;
List<ColumnStatisticsObj> colStatsAggrFromDB;
int maxPartsPerCacheNode = aggrStatsCache.getMaxPartsPerCacheNode();
- float fpp = aggrStatsCache.getFalsePositiveProbability();
+ double fpp = aggrStatsCache.getFalsePositiveProbability();
colStatsList = new ArrayList<ColumnStatisticsObj>();
// Bloom filter for the new node that we will eventually add to the cache
BloomFilter bloomFilter = createPartsBloomFilter(maxPartsPerCacheNode, fpp, partNames);
@@ -1356,7 +1356,7 @@ class MetaStoreDirectSql {
return new AggrStats(colStatsList, partsFound);
}
- private BloomFilter createPartsBloomFilter(int maxPartsPerCacheNode, float fpp,
+ private BloomFilter createPartsBloomFilter(int maxPartsPerCacheNode, double fpp,
List<String> partNames) {
BloomFilter bloomFilter = new BloomFilter(maxPartsPerCacheNode, fpp);
for (String partName : partNames) {
http://git-wip-us.apache.org/repos/asf/hive/blob/b0b6db73/metastore/src/java/org/apache/hadoop/hive/metastore/MetaStoreEndFunctionContext.java
----------------------------------------------------------------------
diff --git a/metastore/src/java/org/apache/hadoop/hive/metastore/MetaStoreEndFunctionContext.java b/metastore/src/java/org/apache/hadoop/hive/metastore/MetaStoreEndFunctionContext.java
deleted file mode 100644
index ae75200..0000000
--- a/metastore/src/java/org/apache/hadoop/hive/metastore/MetaStoreEndFunctionContext.java
+++ /dev/null
@@ -1,59 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.hive.metastore;
-
-/**
- * Base class which provides context to implementations of MetaStoreEndFunctionListener
- */
-
-public class MetaStoreEndFunctionContext {
-
- /**
- * whether method was successful or not.
- */
- private final boolean success;
- private final Exception e;
- private final String inputTableName;
-
- public MetaStoreEndFunctionContext(boolean success, Exception e, String inputTableName) {
- this.success = success;
- this.e = e;
- this.inputTableName = inputTableName;
- }
-
- public MetaStoreEndFunctionContext(boolean success) {
- this(success, null, null);
- }
-
- /**
- * @return whether or not the method succeeded.
- */
- public boolean isSuccess() {
- return success;
- }
-
- public Exception getException() {
- return e;
- }
-
- public String getInputTableName() {
- return inputTableName;
- }
-
-}
http://git-wip-us.apache.org/repos/asf/hive/blob/b0b6db73/metastore/src/java/org/apache/hadoop/hive/metastore/MetaStoreEndFunctionListener.java
----------------------------------------------------------------------
diff --git a/metastore/src/java/org/apache/hadoop/hive/metastore/MetaStoreEndFunctionListener.java b/metastore/src/java/org/apache/hadoop/hive/metastore/MetaStoreEndFunctionListener.java
deleted file mode 100644
index 429efab..0000000
--- a/metastore/src/java/org/apache/hadoop/hive/metastore/MetaStoreEndFunctionListener.java
+++ /dev/null
@@ -1,58 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.hive.metastore;
-
-import java.util.AbstractMap;
-
-import org.apache.hadoop.conf.Configurable;
-import org.apache.hadoop.conf.Configuration;
-
-/**
- * This abstract class needs to be extended to provide implementation of actions that need
- * to be performed when a function ends. These methods are called whenever a function ends.
- *
- * It also provides a way to add fb303 counters through the exportCounters method.
- */
-
-public abstract class MetaStoreEndFunctionListener implements Configurable {
-
- private Configuration conf;
-
- public MetaStoreEndFunctionListener(Configuration config){
- this.conf = config;
- }
-
- public abstract void onEndFunction(String functionName, MetaStoreEndFunctionContext context);
-
- // Unless this is overridden, it does nothing
- public void exportCounters(AbstractMap<String, Long> counters) {
- }
-
- @Override
- public Configuration getConf() {
- return this.conf;
- }
-
- @Override
- public void setConf(Configuration config) {
- this.conf = config;
- }
-
-
-}
http://git-wip-us.apache.org/repos/asf/hive/blob/b0b6db73/metastore/src/java/org/apache/hadoop/hive/metastore/MetaStoreFS.java
----------------------------------------------------------------------
diff --git a/metastore/src/java/org/apache/hadoop/hive/metastore/MetaStoreFS.java b/metastore/src/java/org/apache/hadoop/hive/metastore/MetaStoreFS.java
deleted file mode 100644
index 4f525a4..0000000
--- a/metastore/src/java/org/apache/hadoop/hive/metastore/MetaStoreFS.java
+++ /dev/null
@@ -1,43 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.hive.metastore;
-
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.hive.metastore.api.MetaException;
-
-/**
- * Define a set of APIs that may vary in different environments
- */
-public interface MetaStoreFS {
-
- /**
- * delete a directory
- *
- * @param f
- * @param ifPurge
- * @param recursive
- * @return true on success
- * @throws MetaException
- */
- public boolean deleteDir(FileSystem fs, Path f, boolean recursive,
- boolean ifPurge, Configuration conf) throws MetaException;
-
-}
http://git-wip-us.apache.org/repos/asf/hive/blob/b0b6db73/metastore/src/java/org/apache/hadoop/hive/metastore/MetaStoreFilterHook.java
----------------------------------------------------------------------
diff --git a/metastore/src/java/org/apache/hadoop/hive/metastore/MetaStoreFilterHook.java b/metastore/src/java/org/apache/hadoop/hive/metastore/MetaStoreFilterHook.java
deleted file mode 100644
index 2263837..0000000
--- a/metastore/src/java/org/apache/hadoop/hive/metastore/MetaStoreFilterHook.java
+++ /dev/null
@@ -1,132 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.hive.metastore;
-
-import java.util.List;
-
-import org.apache.hadoop.hive.common.classification.InterfaceAudience.LimitedPrivate;
-import org.apache.hadoop.hive.common.classification.InterfaceStability.Evolving;
-import org.apache.hadoop.hive.metastore.api.Database;
-import org.apache.hadoop.hive.metastore.api.Index;
-import org.apache.hadoop.hive.metastore.api.MetaException;
-import org.apache.hadoop.hive.metastore.api.NoSuchObjectException;
-import org.apache.hadoop.hive.metastore.api.Partition;
-import org.apache.hadoop.hive.metastore.api.PartitionSpec;
-import org.apache.hadoop.hive.metastore.api.Table;
-
-/**
- * Metadata filter hook for metastore client. This will be useful for authorization
- * plugins on hiveserver2 to filter metadata results, especially in case of
- * non-impersonation mode where the metastore doesn't know the end user's identity.
- */
-@LimitedPrivate(value = { "Apache Sentry (Incubating)" })
-@Evolving
-public interface MetaStoreFilterHook {
-
- /**
- * Filter given list of databases
- * @param dbList
- * @return List of filtered Db names
- */
- public List<String> filterDatabases(List<String> dbList) throws MetaException;
-
- /**
- * filter to given database object if applicable
- * @param dataBase
- * @return the same database if it's not filtered out
- * @throws NoSuchObjectException
- */
- public Database filterDatabase(Database dataBase) throws MetaException, NoSuchObjectException;
-
- /**
- * Filter given list of tables
- * @param dbName
- * @param tableList
- * @return List of filtered table names
- */
- public List<String> filterTableNames(String dbName, List<String> tableList) throws MetaException;
-
- /**
- * filter to given table object if applicable
- * @param table
- * @return the same table if it's not filtered out
- * @throws NoSuchObjectException
- */
- public Table filterTable(Table table) throws MetaException, NoSuchObjectException;
-
- /**
- * Filter given list of tables
- * @param tableList
- * @return List of filtered table names
- */
- public List<Table> filterTables(List<Table> tableList) throws MetaException;
-
- /**
- * Filter given list of partitions
- * @param partitionList
- * @return
- */
- public List<Partition> filterPartitions(List<Partition> partitionList) throws MetaException;
-
- /**
- * Filter given list of partition specs
- * @param partitionSpecList
- * @return
- */
- public List<PartitionSpec> filterPartitionSpecs(List<PartitionSpec> partitionSpecList)
- throws MetaException;
-
- /**
- * filter to given partition object if applicable
- * @param partition
- * @return the same partition object if it's not filtered out
- * @throws NoSuchObjectException
- */
- public Partition filterPartition(Partition partition) throws MetaException, NoSuchObjectException;
-
- /**
- * Filter given list of partition names
- * @param dbName
- * @param tblName
- * @param partitionNames
- * @return
- */
- public List<String> filterPartitionNames(String dbName, String tblName,
- List<String> partitionNames) throws MetaException;
-
- public Index filterIndex(Index index) throws MetaException, NoSuchObjectException;
-
- /**
- * Filter given list of index names
- * @param dbName
- * @param tblName
- * @param indexList
- * @return
- */
- public List<String> filterIndexNames(String dbName, String tblName,
- List<String> indexList) throws MetaException;
-
- /**
- * Filter given list of index objects
- * @param indexeList
- * @return
- */
- public List<Index> filterIndexes(List<Index> indexeList) throws MetaException;
-}
-