You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by ga...@apache.org on 2017/09/14 22:10:21 UTC
[6/7] hive git commit: HIVE-17488 Move first set of classes to
standalone metastore. This closes #244. (Alan Gates,
reviewed by Owen O'Malley)
http://git-wip-us.apache.org/repos/asf/hive/blob/b0b6db73/metastore/src/java/org/apache/hadoop/hive/metastore/AggregateStatsCache.java
----------------------------------------------------------------------
diff --git a/metastore/src/java/org/apache/hadoop/hive/metastore/AggregateStatsCache.java b/metastore/src/java/org/apache/hadoop/hive/metastore/AggregateStatsCache.java
deleted file mode 100644
index 4ab178c..0000000
--- a/metastore/src/java/org/apache/hadoop/hive/metastore/AggregateStatsCache.java
+++ /dev/null
@@ -1,573 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.hadoop.hive.metastore;
-
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.hive.conf.HiveConf;
-import org.apache.hadoop.hive.metastore.api.ColumnStatisticsObj;
-import org.apache.hive.common.util.BloomFilter;
-
-import java.util.ArrayList;
-import java.util.HashMap;
-import java.util.Iterator;
-import java.util.List;
-import java.util.Map;
-import java.util.concurrent.ConcurrentHashMap;
-import java.util.concurrent.TimeUnit;
-import java.util.concurrent.atomic.AtomicInteger;
-import java.util.concurrent.atomic.AtomicLong;
-import java.util.concurrent.locks.Lock;
-import java.util.concurrent.locks.ReadWriteLock;
-import java.util.concurrent.locks.ReentrantReadWriteLock;
-
-public class AggregateStatsCache {
-
- private static final Logger LOG = LoggerFactory.getLogger(AggregateStatsCache.class.getName());
- private static AggregateStatsCache self = null;
-
- // Backing store for this cache
- private final ConcurrentHashMap<Key, AggrColStatsList> cacheStore;
- // Cache size
- private final int maxCacheNodes;
- // Current nodes in the cache
- private final AtomicInteger currentNodes = new AtomicInteger(0);
- // Run the cleaner thread when the cache is maxFull% full
- private final float maxFull;
- // Run the cleaner thread until cache is cleanUntil% occupied
- private final float cleanUntil;
- // Nodes go stale after this
- private final long timeToLiveMs;
- // Max time when waiting for write locks on node list
- private final long maxWriterWaitTime;
- // Max time when waiting for read locks on node list
- private final long maxReaderWaitTime;
- // Maximum number of paritions aggregated per cache node
- private final int maxPartsPerCacheNode;
- // Bloom filter false positive probability
- private final float falsePositiveProbability;
- // Max tolerable variance for matches
- private final float maxVariance;
- // Used to determine if cleaner thread is already running
- private boolean isCleaning = false;
- private final AtomicLong cacheHits = new AtomicLong(0);
- private final AtomicLong cacheMisses = new AtomicLong(0);
- // To track cleaner metrics
- int numRemovedTTL = 0, numRemovedLRU = 0;
-
- private AggregateStatsCache(int maxCacheNodes, int maxPartsPerCacheNode, long timeToLiveMs,
- float falsePositiveProbability, float maxVariance, long maxWriterWaitTime,
- long maxReaderWaitTime, float maxFull, float cleanUntil) {
- this.maxCacheNodes = maxCacheNodes;
- this.maxPartsPerCacheNode = maxPartsPerCacheNode;
- this.timeToLiveMs = timeToLiveMs;
- this.falsePositiveProbability = falsePositiveProbability;
- this.maxVariance = maxVariance;
- this.maxWriterWaitTime = maxWriterWaitTime;
- this.maxReaderWaitTime = maxReaderWaitTime;
- this.maxFull = maxFull;
- this.cleanUntil = cleanUntil;
- this.cacheStore = new ConcurrentHashMap<Key, AggrColStatsList>();
- }
-
- public static synchronized AggregateStatsCache getInstance(Configuration conf) {
- if (self == null) {
- int maxCacheNodes =
- HiveConf.getIntVar(conf, HiveConf.ConfVars.METASTORE_AGGREGATE_STATS_CACHE_SIZE);
- // The number of partitions aggregated per cache node
- // If the number of partitions requested is > this value, we'll fetch directly from Metastore
- int maxPartitionsPerCacheNode =
- HiveConf
- .getIntVar(conf, HiveConf.ConfVars.METASTORE_AGGREGATE_STATS_CACHE_MAX_PARTITIONS);
- long timeToLiveMs =
- HiveConf.getTimeVar(conf, HiveConf.ConfVars.METASTORE_AGGREGATE_STATS_CACHE_TTL,
- TimeUnit.SECONDS)*1000;
- // False positives probability we are ready to tolerate for the underlying bloom filter
- float falsePositiveProbability =
- HiveConf.getFloatVar(conf, HiveConf.ConfVars.METASTORE_AGGREGATE_STATS_CACHE_FPP);
- // Maximum tolerable variance in number of partitions between cached node and our request
- float maxVariance =
- HiveConf
- .getFloatVar(conf, HiveConf.ConfVars.METASTORE_AGGREGATE_STATS_CACHE_MAX_VARIANCE);
- long maxWriterWaitTime =
- HiveConf.getTimeVar(conf,
- HiveConf.ConfVars.METASTORE_AGGREGATE_STATS_CACHE_MAX_WRITER_WAIT,
- TimeUnit.MILLISECONDS);
- long maxReaderWaitTime =
- HiveConf.getTimeVar(conf,
- HiveConf.ConfVars.METASTORE_AGGREGATE_STATS_CACHE_MAX_READER_WAIT,
- TimeUnit.MILLISECONDS);
- float maxFull =
- HiveConf.getFloatVar(conf, HiveConf.ConfVars.METASTORE_AGGREGATE_STATS_CACHE_MAX_FULL);
- float cleanUntil =
- HiveConf.getFloatVar(conf, HiveConf.ConfVars.METASTORE_AGGREGATE_STATS_CACHE_CLEAN_UNTIL);
- self =
- new AggregateStatsCache(maxCacheNodes, maxPartitionsPerCacheNode, timeToLiveMs,
- falsePositiveProbability, maxVariance, maxWriterWaitTime, maxReaderWaitTime, maxFull,
- cleanUntil);
- }
- return self;
- }
-
- public int getMaxCacheNodes() {
- return maxCacheNodes;
- }
-
- public int getCurrentNodes() {
- return currentNodes.intValue();
- }
-
- public float getFullPercent() {
- return (currentNodes.intValue() / (float) maxCacheNodes) * 100;
- }
-
- public int getMaxPartsPerCacheNode() {
- return maxPartsPerCacheNode;
- }
-
- public float getFalsePositiveProbability() {
- return falsePositiveProbability;
- }
-
- public Float getHitRatio() {
- if (cacheHits.longValue() + cacheMisses.longValue() > 0) {
- return (float) (cacheHits.longValue()) / (cacheHits.longValue() + cacheMisses.longValue());
- }
- return null;
- }
-
- /**
- * Return aggregate stats for a column from the cache or null.
- * While reading from the nodelist for a key, we wait maxReaderWaitTime to acquire the lock,
- * failing which we return a cache miss (i.e. null)
- *
- * @param dbName
- * @param tblName
- * @param colName
- * @param partNames
- * @return
- */
- public AggrColStats get(String dbName, String tblName, String colName, List<String> partNames) {
- // Cache key
- Key key = new Key(dbName, tblName, colName);
- AggrColStatsList candidateList = cacheStore.get(key);
- // No key, or no nodes in candidate list
- if ((candidateList == null) || (candidateList.nodes.size() == 0)) {
- LOG.debug("No aggregate stats cached for " + key.toString());
- return null;
- }
- // Find the value object
- // Update the timestamp of the key,value if value matches the criteria
- // Return the value
- AggrColStats match = null;
- boolean isLocked = false;
- try {
- // Try to readlock the candidateList; timeout after maxReaderWaitTime
- isLocked = candidateList.readLock.tryLock(maxReaderWaitTime, TimeUnit.MILLISECONDS);
- if (isLocked) {
- match = findBestMatch(partNames, candidateList.nodes);
- }
- if (match != null) {
- // Ok to not lock the list for this and use a volatile lastAccessTime instead
- candidateList.updateLastAccessTime();
- cacheHits.incrementAndGet();
- LOG.info("Returning aggregate stats from the cache; total hits: " + cacheHits.longValue()
- + ", total misses: " + cacheMisses.longValue() + ", hit ratio: " + getHitRatio());
- }
- else {
- cacheMisses.incrementAndGet();
- }
- } catch (InterruptedException e) {
- LOG.debug("Interrupted Exception ignored ",e);
- } finally {
- if (isLocked) {
- candidateList.readLock.unlock();
- }
- }
- return match;
- }
-
- /**
- * Find the best match using the configurable error tolerance and time to live value
- *
- * @param partNames
- * @param candidates
- * @return best matched node or null
- */
- private AggrColStats findBestMatch(List<String> partNames, List<AggrColStats> candidates) {
- // Hits, misses tracked for a candidate node
- MatchStats matchStats;
- // MatchStats for each candidate
- Map<AggrColStats, MatchStats> candidateMatchStats = new HashMap<AggrColStats, MatchStats>();
- // The final match we intend to return
- AggrColStats bestMatch = null;
- // To compare among potentially multiple matches
- int bestMatchHits = 0;
- int numPartsRequested = partNames.size();
- // 1st pass at marking invalid candidates
- // Checks based on variance and TTL
- // Note: we're not creating a copy of the list for saving memory
- for (AggrColStats candidate : candidates) {
- // Variance check
- if (Math.abs((candidate.getNumPartsCached() - numPartsRequested) / numPartsRequested)
- > maxVariance) {
- continue;
- }
- // TTL check
- if (isExpired(candidate)) {
- continue;
- } else {
- candidateMatchStats.put(candidate, new MatchStats(0, 0));
- }
- }
- // We'll count misses as we iterate
- int maxMisses = (int) maxVariance * numPartsRequested;
- for (String partName : partNames) {
- for (Iterator<Map.Entry<AggrColStats, MatchStats>> iterator = candidateMatchStats.entrySet().iterator(); iterator.hasNext();) {
- Map.Entry<AggrColStats, MatchStats> entry = iterator.next();
- AggrColStats candidate = entry.getKey();
- matchStats = entry.getValue();
- if (candidate.getBloomFilter().test(partName.getBytes())) {
- ++matchStats.hits;
- } else {
- ++matchStats.misses;
- }
- // 2nd pass at removing invalid candidates
- // If misses so far exceed max tolerable misses
- if (matchStats.misses > maxMisses) {
- iterator.remove();
- continue;
- }
- // Check if this is the best match so far
- if (matchStats.hits > bestMatchHits) {
- bestMatch = candidate;
- }
- }
- }
- if (bestMatch != null) {
- // Update the last access time for this node
- bestMatch.updateLastAccessTime();
- }
- return bestMatch;
- }
-
- /**
- * Add a new node to the cache; may trigger the cleaner thread if the cache is near full capacity.
- * We'll however add the node even if we temporaily exceed maxCacheNodes, because the cleaner
- * will eventually create space from expired nodes or by removing LRU nodes.
- *
- * @param dbName
- * @param tblName
- * @param colName
- * @param numPartsCached
- * @param colStats
- * @param bloomFilter
- */
- // TODO: make add asynchronous: add shouldn't block the higher level calls
- public void add(String dbName, String tblName, String colName, long numPartsCached,
- ColumnStatisticsObj colStats, BloomFilter bloomFilter) {
- // If we have no space in the cache, run cleaner thread
- if (getCurrentNodes() / maxCacheNodes > maxFull) {
- spawnCleaner();
- }
- // Cache key
- Key key = new Key(dbName, tblName, colName);
- // Add new node to the cache
- AggrColStats node = new AggrColStats(numPartsCached, bloomFilter, colStats);
- AggrColStatsList nodeList;
- AggrColStatsList newNodeList = new AggrColStatsList();
- newNodeList.nodes = new ArrayList<AggrColStats>();
- nodeList = cacheStore.putIfAbsent(key, newNodeList);
- if (nodeList == null) {
- nodeList = newNodeList;
- }
- boolean isLocked = false;
- try {
- isLocked = nodeList.writeLock.tryLock(maxWriterWaitTime, TimeUnit.MILLISECONDS);
- if (isLocked) {
- nodeList.nodes.add(node);
- node.updateLastAccessTime();
- nodeList.updateLastAccessTime();
- currentNodes.getAndIncrement();
- }
- } catch (InterruptedException e) {
- LOG.debug("Interrupted Exception ignored ", e);
- } finally {
- if (isLocked) {
- nodeList.writeLock.unlock();
- }
- }
- }
-
- /**
- * Cleans the expired nodes or removes LRU nodes of the cache,
- * until the cache size reduces to cleanUntil% full.
- */
- private void spawnCleaner() {
- // This spawns a separate thread to walk through the cache and removes expired nodes.
- // Only one cleaner thread should be running at any point.
- synchronized (this) {
- if (isCleaning) {
- return;
- }
- isCleaning = true;
- }
- Thread cleaner = new Thread("AggregateStatsCache-CleanerThread") {
- @Override
- public void run() {
- numRemovedTTL = 0;
- numRemovedLRU = 0;
- long cleanerStartTime = System.currentTimeMillis();
- LOG.info("AggregateStatsCache is " + getFullPercent() + "% full, with "
- + getCurrentNodes() + " nodes; starting cleaner thread");
- try {
- Iterator<Map.Entry<Key, AggrColStatsList>> mapIterator = cacheStore.entrySet().iterator();
- while (mapIterator.hasNext()) {
- Map.Entry<Key, AggrColStatsList> pair =
- mapIterator.next();
- AggrColStats node;
- AggrColStatsList candidateList = pair.getValue();
- List<AggrColStats> nodes = candidateList.nodes;
- if (nodes.size() == 0) {
- mapIterator.remove();
- continue;
- }
- boolean isLocked = false;
- try {
- isLocked = candidateList.writeLock.tryLock(maxWriterWaitTime, TimeUnit.MILLISECONDS);
- if (isLocked) {
- for (Iterator<AggrColStats> listIterator = nodes.iterator(); listIterator.hasNext();) {
- node = listIterator.next();
- // Remove the node if it has expired
- if (isExpired(node)) {
- listIterator.remove();
- numRemovedTTL++;
- currentNodes.getAndDecrement();
- }
- }
- }
- } catch (InterruptedException e) {
- LOG.debug("Interrupted Exception ignored ",e);
- } finally {
- if (isLocked) {
- candidateList.writeLock.unlock();
- }
- }
- // We want to make sure this runs at a low priority in the background
- Thread.yield();
- }
- // If the expired nodes did not result in cache being cleanUntil% in size,
- // start removing LRU nodes
- while (getCurrentNodes() / maxCacheNodes > cleanUntil) {
- evictOneNode();
- }
- } finally {
- isCleaning = false;
- LOG.info("Stopping cleaner thread; AggregateStatsCache is now " + getFullPercent()
- + "% full, with " + getCurrentNodes() + " nodes");
- LOG.info("Number of expired nodes removed: " + numRemovedTTL);
- LOG.info("Number of LRU nodes removed: " + numRemovedLRU);
- LOG.info("Cleaner ran for: " + (System.currentTimeMillis() - cleanerStartTime) + "ms");
- }
- }
- };
- cleaner.setPriority(Thread.MIN_PRIORITY);
- cleaner.setDaemon(true);
- cleaner.start();
- }
-
- /**
- * Evict an LRU node or expired node whichever we find first
- */
- private void evictOneNode() {
- // Get the LRU key, value
- Key lruKey = null;
- AggrColStatsList lruValue = null;
- for (Map.Entry<Key, AggrColStatsList> entry : cacheStore.entrySet()) {
- Key key = entry.getKey();
- AggrColStatsList value = entry.getValue();
- if (lruKey == null) {
- lruKey = key;
- lruValue = value;
- continue;
- }
- if ((value.lastAccessTime < lruValue.lastAccessTime) && !(value.nodes.isEmpty())) {
- lruKey = key;
- lruValue = value;
- }
- }
- // Now delete a node for this key's list
- AggrColStatsList candidateList = cacheStore.get(lruKey);
- boolean isLocked = false;
- try {
- isLocked = candidateList.writeLock.tryLock(maxWriterWaitTime, TimeUnit.MILLISECONDS);
- if (isLocked) {
- AggrColStats candidate;
- AggrColStats lruNode = null;
- int currentIndex = 0;
- int deleteIndex = 0;
- for (Iterator<AggrColStats> iterator = candidateList.nodes.iterator(); iterator.hasNext();) {
- candidate = iterator.next();
- // Since we have to create space for 1, if we find an expired node we will remove it &
- // return
- if (isExpired(candidate)) {
- iterator.remove();
- currentNodes.getAndDecrement();
- numRemovedTTL++;
- return;
- }
- // Sorry, too many ifs but this form looks optimal
- // Update the LRU node from what we've seen so far
- if (lruNode == null) {
- lruNode = candidate;
- ++currentIndex;
- continue;
- }
- if (lruNode != null) {
- if (candidate.lastAccessTime < lruNode.lastAccessTime) {
- lruNode = candidate;
- deleteIndex = currentIndex;
- }
- }
- }
- candidateList.nodes.remove(deleteIndex);
- currentNodes.getAndDecrement();
- numRemovedLRU++;
- }
- } catch (InterruptedException e) {
- LOG.debug("Interrupted Exception ignored ",e);
- } finally {
- if (isLocked) {
- candidateList.writeLock.unlock();
- }
- }
- }
-
- private boolean isExpired(AggrColStats aggrColStats) {
- return (System.currentTimeMillis() - aggrColStats.lastAccessTime) > timeToLiveMs;
- }
-
- /**
- * Key object for the stats cache hashtable
- */
- static class Key {
- private final String dbName;
- private final String tblName;
- private final String colName;
-
- Key(String db, String table, String col) {
- // Don't construct an illegal cache key
- if ((db == null) || (table == null) || (col == null)) {
- throw new IllegalArgumentException("dbName, tblName, colName can't be null");
- }
- dbName = db;
- tblName = table;
- colName = col;
- }
-
- @Override
- public boolean equals(Object other) {
- if ((other == null) || !(other instanceof Key)) {
- return false;
- }
- Key that = (Key) other;
- return dbName.equals(that.dbName) && tblName.equals(that.tblName)
- && colName.equals(that.colName);
- }
-
- @Override
- public int hashCode() {
- return dbName.hashCode() * 31 + tblName.hashCode() * 31 + colName.hashCode();
- }
-
- @Override
- public String toString() {
- return "database:" + dbName + ", table:" + tblName + ", column:" + colName;
- }
-
- }
-
- static class AggrColStatsList {
- // TODO: figure out a better data structure for node list(?)
- private List<AggrColStats> nodes = new ArrayList<AggrColStats>();
- private final ReadWriteLock lock = new ReentrantReadWriteLock();
- // Read lock for get operation
- private final Lock readLock = lock.readLock();
- // Write lock for add, evict and clean operation
- private final Lock writeLock = lock.writeLock();
- // Using volatile instead of locking updates to this variable,
- // since we can rely on approx lastAccessTime but don't want a performance hit
- private volatile long lastAccessTime = 0;
-
- List<AggrColStats> getNodes() {
- return nodes;
- }
-
- void updateLastAccessTime() {
- this.lastAccessTime = System.currentTimeMillis();
- }
- }
-
- public static class AggrColStats {
- private final long numPartsCached;
- private final BloomFilter bloomFilter;
- private final ColumnStatisticsObj colStats;
- private volatile long lastAccessTime;
-
- public AggrColStats(long numPartsCached, BloomFilter bloomFilter,
- ColumnStatisticsObj colStats) {
- this.numPartsCached = numPartsCached;
- this.bloomFilter = bloomFilter;
- this.colStats = colStats;
- this.lastAccessTime = System.currentTimeMillis();
- }
-
- public long getNumPartsCached() {
- return numPartsCached;
- }
-
- public ColumnStatisticsObj getColStats() {
- return colStats;
- }
-
- public BloomFilter getBloomFilter() {
- return bloomFilter;
- }
-
- void updateLastAccessTime() {
- this.lastAccessTime = System.currentTimeMillis();
- }
- }
-
- /**
- * Intermediate object, used to collect hits & misses for each cache node that is evaluate for an
- * incoming request
- */
- private static class MatchStats {
- private int hits = 0;
- private int misses = 0;
-
- MatchStats(int hits, int misses) {
- this.hits = hits;
- this.misses = misses;
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/hive/blob/b0b6db73/metastore/src/java/org/apache/hadoop/hive/metastore/DatabaseProduct.java
----------------------------------------------------------------------
diff --git a/metastore/src/java/org/apache/hadoop/hive/metastore/DatabaseProduct.java b/metastore/src/java/org/apache/hadoop/hive/metastore/DatabaseProduct.java
deleted file mode 100644
index 7634852..0000000
--- a/metastore/src/java/org/apache/hadoop/hive/metastore/DatabaseProduct.java
+++ /dev/null
@@ -1,75 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.hive.metastore;
-
-import java.sql.SQLException;
-import java.sql.SQLTransactionRollbackException;
-
-/** Database product infered via JDBC. */
-public enum DatabaseProduct {
- DERBY, MYSQL, POSTGRES, ORACLE, SQLSERVER, OTHER;
-
- /**
- * Determine the database product type
- * @param productName string to defer database connection
- * @return database product type
- */
- public static DatabaseProduct determineDatabaseProduct(String productName) throws SQLException {
- if (productName == null) {
- return OTHER;
- }
- productName = productName.toLowerCase();
- if (productName.contains("derby")) {
- return DERBY;
- } else if (productName.contains("microsoft sql server")) {
- return SQLSERVER;
- } else if (productName.contains("mysql")) {
- return MYSQL;
- } else if (productName.contains("oracle")) {
- return ORACLE;
- } else if (productName.contains("postgresql")) {
- return POSTGRES;
- } else {
- return OTHER;
- }
- }
-
- public static boolean isDeadlock(DatabaseProduct dbProduct, SQLException e) {
- return e instanceof SQLTransactionRollbackException
- || ((dbProduct == MYSQL || dbProduct == POSTGRES || dbProduct == SQLSERVER)
- && e.getSQLState().equals("40001"))
- || (dbProduct == POSTGRES && e.getSQLState().equals("40P01"))
- || (dbProduct == ORACLE && (e.getMessage().contains("deadlock detected")
- || e.getMessage().contains("can't serialize access for this transaction")));
- }
-
- /**
- * Whether the RDBMS has restrictions on IN list size (explicit, or poor perf-based).
- */
- public static boolean needsInBatching(DatabaseProduct dbType) {
- return dbType == ORACLE || dbType == SQLSERVER;
- }
-
- /**
- * Whether the RDBMS has a bug in join and filter operation order described in DERBY-6358.
- */
- public static boolean hasJoinOperationOrderBug(DatabaseProduct dbType) {
- return dbType == DERBY || dbType == ORACLE;
- }
-};
http://git-wip-us.apache.org/repos/asf/hive/blob/b0b6db73/metastore/src/java/org/apache/hadoop/hive/metastore/DefaultHiveMetaHook.java
----------------------------------------------------------------------
diff --git a/metastore/src/java/org/apache/hadoop/hive/metastore/DefaultHiveMetaHook.java b/metastore/src/java/org/apache/hadoop/hive/metastore/DefaultHiveMetaHook.java
deleted file mode 100644
index 0957945..0000000
--- a/metastore/src/java/org/apache/hadoop/hive/metastore/DefaultHiveMetaHook.java
+++ /dev/null
@@ -1,51 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.hive.metastore;
-
-import org.apache.hadoop.hive.metastore.api.MetaException;
-import org.apache.hadoop.hive.metastore.api.Table;
-
-public abstract class DefaultHiveMetaHook implements HiveMetaHook {
- /**
- * Called after successfully after INSERT [OVERWRITE] statement is executed.
- * @param table table definition
- * @param overwrite true if it is INSERT OVERWRITE
- *
- * @throws MetaException
- */
- public abstract void commitInsertTable(Table table, boolean overwrite) throws MetaException;
-
- /**
- * called before commit insert method is called
- * @param table table definition
- * @param overwrite true if it is INSERT OVERWRITE
- *
- * @throws MetaException
- */
- public abstract void preInsertTable(Table table, boolean overwrite) throws MetaException;
-
- /**
- * called in case pre commit or commit insert fail.
- * @param table table definition
- * @param overwrite true if it is INSERT OVERWRITE
- *
- * @throws MetaException
- */
- public abstract void rollbackInsertTable(Table table, boolean overwrite) throws MetaException;
-}
http://git-wip-us.apache.org/repos/asf/hive/blob/b0b6db73/metastore/src/java/org/apache/hadoop/hive/metastore/DefaultMetaStoreFilterHookImpl.java
----------------------------------------------------------------------
diff --git a/metastore/src/java/org/apache/hadoop/hive/metastore/DefaultMetaStoreFilterHookImpl.java b/metastore/src/java/org/apache/hadoop/hive/metastore/DefaultMetaStoreFilterHookImpl.java
deleted file mode 100644
index f0cab26..0000000
--- a/metastore/src/java/org/apache/hadoop/hive/metastore/DefaultMetaStoreFilterHookImpl.java
+++ /dev/null
@@ -1,102 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.hive.metastore;
-
-import java.util.List;
-
-import org.apache.hadoop.hive.conf.HiveConf;
-import org.apache.hadoop.hive.metastore.api.Database;
-import org.apache.hadoop.hive.metastore.api.Index;
-import org.apache.hadoop.hive.metastore.api.MetaException;
-import org.apache.hadoop.hive.metastore.api.NoSuchObjectException;
-import org.apache.hadoop.hive.metastore.api.Partition;
-import org.apache.hadoop.hive.metastore.api.PartitionSpec;
-import org.apache.hadoop.hive.metastore.api.Table;
-
-/**
- * Default no-op implementation of the MetaStoreFilterHook that returns the result as is
- */
-public class DefaultMetaStoreFilterHookImpl implements MetaStoreFilterHook {
-
- public DefaultMetaStoreFilterHookImpl(HiveConf conf) {
- }
-
- @Override
- public List<String> filterDatabases(List<String> dbList) throws MetaException {
- return dbList;
- }
-
- @Override
- public Database filterDatabase(Database dataBase) throws NoSuchObjectException {
- return dataBase;
- }
-
- @Override
- public List<String> filterTableNames(String dbName, List<String> tableList) throws MetaException {
- return tableList;
- }
-
- @Override
- public Table filterTable(Table table) throws NoSuchObjectException {
- return table;
- }
-
- @Override
- public List<Table> filterTables(List<Table> tableList) throws MetaException {
- return tableList;
- }
-
- @Override
- public List<Partition> filterPartitions(List<Partition> partitionList) throws MetaException {
- return partitionList;
- }
-
- @Override
- public List<PartitionSpec> filterPartitionSpecs(
- List<PartitionSpec> partitionSpecList) throws MetaException {
- return partitionSpecList;
- }
-
- @Override
- public Partition filterPartition(Partition partition) throws NoSuchObjectException {
- return partition;
- }
-
- @Override
- public List<String> filterPartitionNames(String dbName, String tblName,
- List<String> partitionNames) throws MetaException {
- return partitionNames;
- }
-
- @Override
- public Index filterIndex(Index index) throws NoSuchObjectException {
- return index;
- }
-
- @Override
- public List<String> filterIndexNames(String dbName, String tblName,
- List<String> indexList) throws MetaException {
- return indexList;
- }
-
- @Override
- public List<Index> filterIndexes(List<Index> indexeList) throws MetaException {
- return indexeList;
- }
-}
http://git-wip-us.apache.org/repos/asf/hive/blob/b0b6db73/metastore/src/java/org/apache/hadoop/hive/metastore/FileFormatProxy.java
----------------------------------------------------------------------
diff --git a/metastore/src/java/org/apache/hadoop/hive/metastore/FileFormatProxy.java b/metastore/src/java/org/apache/hadoop/hive/metastore/FileFormatProxy.java
deleted file mode 100644
index 14ff187..0000000
--- a/metastore/src/java/org/apache/hadoop/hive/metastore/FileFormatProxy.java
+++ /dev/null
@@ -1,64 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.hive.metastore;
-
-import java.io.IOException;
-import java.nio.ByteBuffer;
-import java.util.List;
-
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.hive.metastore.Metastore.SplitInfos;
-import org.apache.hadoop.hive.ql.io.sarg.SearchArgument;
-
-/**
- * Same as PartitionExpressionProxy, but for file format specific methods for metadata cache.
- */
-public interface FileFormatProxy {
-
- /**
- * Applies SARG to file metadata, and produces some result for this file.
- * @param sarg SARG
- * @param fileMetadata File metadata from metastore cache.
- * @return The result to return to client for this file, or null if file is eliminated.
- */
- SplitInfos applySargToMetadata(SearchArgument sarg, ByteBuffer fileMetadata) throws IOException;
-
- /**
- * @param fs The filesystem of the file.
- * @param path The file path.
- * @param addedVals Output parameter; additional column values for columns returned by
- * getAddedColumnsToCache to cache in MS.
- * @return The ORC file metadata for a given file.
- */
- ByteBuffer getMetadataToCache(
- FileSystem fs, Path path, ByteBuffer[] addedVals) throws IOException;
-
- /**
- * @return Additional column names to cache in MS for this format.
- */
- ByteBuffer[] getAddedColumnsToCache();
-
- /**
- * @param metadata File metadatas.
- * @return Additional values for columns returned by getAddedColumnsToCache to cache in MS
- * for respective metadatas.
- */
- ByteBuffer[][] getAddedValuesToCache(List<ByteBuffer> metadata);
-
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hive/blob/b0b6db73/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaException.java
----------------------------------------------------------------------
diff --git a/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaException.java b/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaException.java
deleted file mode 100644
index 05cc1fa..0000000
--- a/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaException.java
+++ /dev/null
@@ -1,42 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.hive.metastore;
-
-public class HiveMetaException extends Exception {
-
- public HiveMetaException() {
- // TODO Auto-generated constructor stub
- }
-
- public HiveMetaException(String message) {
- super(message);
- // TODO Auto-generated constructor stub
- }
-
- public HiveMetaException(Throwable cause) {
- super(cause);
- // TODO Auto-generated constructor stub
- }
-
- public HiveMetaException(String message, Throwable cause) {
- super(message, cause);
- // TODO Auto-generated constructor stub
- }
-
-}
http://git-wip-us.apache.org/repos/asf/hive/blob/b0b6db73/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaHook.java
----------------------------------------------------------------------
diff --git a/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaHook.java b/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaHook.java
deleted file mode 100644
index 3317bc5..0000000
--- a/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaHook.java
+++ /dev/null
@@ -1,96 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.hive.metastore;
-
-import org.apache.hadoop.hive.common.classification.InterfaceAudience;
-import org.apache.hadoop.hive.common.classification.InterfaceStability;
-import org.apache.hadoop.hive.metastore.api.MetaException;
-import org.apache.hadoop.hive.metastore.api.Partition;
-import org.apache.hadoop.hive.metastore.api.Table;
-
-/**
- * HiveMetaHook defines notification methods which are invoked as part
- * of transactions against the metastore, allowing external catalogs
- * such as HBase to be kept in sync with Hive's metastore.
- *
- *<p>
- *
- * Implementations can use {@link MetaStoreUtils#isExternalTable} to
- * distinguish external tables from managed tables.
- */
-@InterfaceAudience.Public
-@InterfaceStability.Stable
-public interface HiveMetaHook {
- /**
- * Called before a new table definition is added to the metastore
- * during CREATE TABLE.
- *
- * @param table new table definition
- */
- public void preCreateTable(Table table)
- throws MetaException;
-
- /**
- * Called after failure adding a new table definition to the metastore
- * during CREATE TABLE.
- *
- * @param table new table definition
- */
- public void rollbackCreateTable(Table table)
- throws MetaException;
-
- /**
- * Called after successfully adding a new table definition to the metastore
- * during CREATE TABLE.
- *
- * @param table new table definition
- */
- public void commitCreateTable(Table table)
- throws MetaException;
-
- /**
- * Called before a table definition is removed from the metastore
- * during DROP TABLE.
- *
- * @param table table definition
- */
- public void preDropTable(Table table)
- throws MetaException;
-
- /**
- * Called after failure removing a table definition from the metastore
- * during DROP TABLE.
- *
- * @param table table definition
- */
- public void rollbackDropTable(Table table)
- throws MetaException;
-
- /**
- * Called after successfully removing a table definition from the metastore
- * during DROP TABLE.
- *
- * @param table table definition
- *
- * @param deleteData whether to delete data as well; this should typically
- * be ignored in the case of an external table
- */
- public void commitDropTable(Table table, boolean deleteData)
- throws MetaException;
-}
http://git-wip-us.apache.org/repos/asf/hive/blob/b0b6db73/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaHookLoader.java
----------------------------------------------------------------------
diff --git a/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaHookLoader.java b/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaHookLoader.java
deleted file mode 100644
index 1cdae9b..0000000
--- a/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaHookLoader.java
+++ /dev/null
@@ -1,39 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.hive.metastore;
-
-import org.apache.hadoop.hive.metastore.api.MetaException;
-import org.apache.hadoop.hive.metastore.api.Table;
-
-/**
- * HiveMetaHookLoader is responsible for loading a {@link HiveMetaHook}
- * for a given table.
- */
-public interface HiveMetaHookLoader {
- /**
- * Loads a hook for the specified table.
- *
- * @param tbl table of interest
- *
- * @return hook, or null if none registered
- */
- public HiveMetaHook getHook(Table tbl) throws MetaException;
-}
-
-// End HiveMetaHookLoader.java
http://git-wip-us.apache.org/repos/asf/hive/blob/b0b6db73/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java
----------------------------------------------------------------------
diff --git a/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java b/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java
index 53cb3d9..cf33cca 100644
--- a/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java
+++ b/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java
@@ -7907,7 +7907,7 @@ public class HiveMetaStore extends ThriftHiveMetastore {
private static void initializeAndStartThread(MetaStoreThread thread, HiveConf conf) throws
MetaException {
LOG.info("Starting metastore thread of type " + thread.getClass().getName());
- thread.setHiveConf(conf);
+ thread.setConf(conf);
thread.setThreadId(nextThreadId++);
thread.init(new AtomicBoolean(), new AtomicBoolean());
thread.start();
http://git-wip-us.apache.org/repos/asf/hive/blob/b0b6db73/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStoreClient.java
----------------------------------------------------------------------
diff --git a/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStoreClient.java b/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStoreClient.java
index 751dd91..75e4180 100644
--- a/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStoreClient.java
+++ b/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStoreClient.java
@@ -49,6 +49,7 @@ import java.security.PrivilegedExceptionAction;
import javax.security.auth.login.LoginException;
+import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hive.common.ObjectPair;
import org.apache.hadoop.hive.common.StatsSetupConst;
import org.apache.hadoop.hive.common.ValidTxnList;
@@ -263,7 +264,7 @@ public class HiveMetaStoreClient implements IMetaStoreClient, AutoCloseable {
String msg = "Unable to create instance of " + authProviderClass.getName() + ": ";
try {
Constructor<? extends MetaStoreFilterHook> constructor =
- authProviderClass.getConstructor(HiveConf.class);
+ authProviderClass.getConstructor(Configuration.class);
return constructor.newInstance(conf);
} catch (NoSuchMethodException e) {
throw new IllegalStateException(msg + e.getMessage(), e);
http://git-wip-us.apache.org/repos/asf/hive/blob/b0b6db73/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStoreFsImpl.java
----------------------------------------------------------------------
diff --git a/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStoreFsImpl.java b/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStoreFsImpl.java
deleted file mode 100644
index b7d7b50..0000000
--- a/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStoreFsImpl.java
+++ /dev/null
@@ -1,52 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.hive.metastore;
-
-import java.io.FileNotFoundException;
-
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.hive.common.FileUtils;
-import org.apache.hadoop.hive.metastore.api.MetaException;
-
-public class HiveMetaStoreFsImpl implements MetaStoreFS {
-
- public static final Logger LOG = LoggerFactory
- .getLogger("hive.metastore.hivemetastoreFsimpl");
-
- @Override
- public boolean deleteDir(FileSystem fs, Path f, boolean recursive,
- boolean ifPurge, Configuration conf) throws MetaException {
- try {
- FileUtils.moveToTrash(fs, f, conf, ifPurge);
- if (fs.exists(f)) {
- throw new MetaException("Unable to delete directory: " + f);
- }
- return true;
- } catch (FileNotFoundException e) {
- return true; // ok even if there is not data
- } catch (Exception e) {
- MetaStoreUtils.logAndThrowMetaException(e);
- }
- return false;
- }
-}
http://git-wip-us.apache.org/repos/asf/hive/blob/b0b6db73/metastore/src/java/org/apache/hadoop/hive/metastore/IExtrapolatePartStatus.java
----------------------------------------------------------------------
diff --git a/metastore/src/java/org/apache/hadoop/hive/metastore/IExtrapolatePartStatus.java b/metastore/src/java/org/apache/hadoop/hive/metastore/IExtrapolatePartStatus.java
deleted file mode 100644
index d0569fb..0000000
--- a/metastore/src/java/org/apache/hadoop/hive/metastore/IExtrapolatePartStatus.java
+++ /dev/null
@@ -1,85 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.hive.metastore;
-
-import java.util.HashMap;
-import java.util.Map;
-
-public interface IExtrapolatePartStatus {
- /**
- * The sequence of colStatNames.
- */
- static String[] colStatNames = new String[] { "LONG_LOW_VALUE", "LONG_HIGH_VALUE",
- "DOUBLE_LOW_VALUE", "DOUBLE_HIGH_VALUE", "BIG_DECIMAL_LOW_VALUE", "BIG_DECIMAL_HIGH_VALUE",
- "NUM_NULLS", "NUM_DISTINCTS", "AVG_COL_LEN", "MAX_COL_LEN", "NUM_TRUES", "NUM_FALSES",
- "AVG_NDV_LONG", "AVG_NDV_DOUBLE", "AVG_NDV_DECIMAL", "SUM_NUM_DISTINCTS" };
-
- /**
- * The indexes for colstats.
- */
- static HashMap<String, Integer[]> indexMaps = new HashMap<String, Integer[]>() {
- {
- put("bigint", new Integer[] { 0, 1, 6, 7, 12, 15 });
- put("int", new Integer[] { 0, 1, 6, 7, 12, 15 });
- put("smallint", new Integer[] { 0, 1, 6, 7, 12, 15 });
- put("tinyint", new Integer[] { 0, 1, 6, 7, 12, 15 });
- put("date", new Integer[] { 0, 1, 6, 7, 12, 15 });
- put("timestamp", new Integer[] { 0, 1, 6, 7, 12, 15 });
- put("long", new Integer[] { 0, 1, 6, 7, 12, 15 });
- put("double", new Integer[] { 2, 3, 6, 7, 13, 15 });
- put("float", new Integer[] { 2, 3, 6, 7, 13, 15 });
- put("varchar", new Integer[] { 8, 9, 6, 7, 15 });
- put("char", new Integer[] { 8, 9, 6, 7, 15 });
- put("string", new Integer[] { 8, 9, 6, 7, 15 });
- put("boolean", new Integer[] { 10, 11, 6, 15 });
- put("binary", new Integer[] { 8, 9, 6, 15 });
- put("decimal", new Integer[] { 4, 5, 6, 7, 14, 15 });
- put("default", new Integer[] { 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 15 });
- }
- };
-
- /**
- * The sequence of colStatTypes.
- */
- static enum ColStatType {
- Long, Double, Decimal
- }
-
- static ColStatType[] colStatTypes = new ColStatType[] { ColStatType.Long, ColStatType.Long,
- ColStatType.Double, ColStatType.Double, ColStatType.Decimal, ColStatType.Decimal,
- ColStatType.Long, ColStatType.Long, ColStatType.Double, ColStatType.Long, ColStatType.Long,
- ColStatType.Long, ColStatType.Double, ColStatType.Double, ColStatType.Double,
- ColStatType.Long };
-
- /**
- * The sequence of aggregation function on colStats.
- */
- static enum AggrType {
- Min, Max, Sum, Avg
- }
-
- static AggrType[] aggrTypes = new AggrType[] { AggrType.Min, AggrType.Max, AggrType.Min,
- AggrType.Max, AggrType.Min, AggrType.Max, AggrType.Sum, AggrType.Max, AggrType.Max,
- AggrType.Max, AggrType.Sum, AggrType.Sum, AggrType.Avg, AggrType.Avg, AggrType.Avg,
- AggrType.Sum };
-
- public Object extrapolate(Object[] min, Object[] max, int colStatIndex,
- Map<String, Integer> indexMap);
-
-}
http://git-wip-us.apache.org/repos/asf/hive/blob/b0b6db73/metastore/src/java/org/apache/hadoop/hive/metastore/IHMSHandler.java
----------------------------------------------------------------------
diff --git a/metastore/src/java/org/apache/hadoop/hive/metastore/IHMSHandler.java b/metastore/src/java/org/apache/hadoop/hive/metastore/IHMSHandler.java
deleted file mode 100644
index cc7ae1d..0000000
--- a/metastore/src/java/org/apache/hadoop/hive/metastore/IHMSHandler.java
+++ /dev/null
@@ -1,28 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.hive.metastore;
-
-import org.apache.hadoop.conf.Configurable;
-import org.apache.hadoop.hive.metastore.api.MetaException;
-import org.apache.hadoop.hive.metastore.api.ThriftHiveMetastore;
-
-public interface IHMSHandler extends ThriftHiveMetastore.Iface, Configurable {
-
- void init() throws MetaException;
-}
http://git-wip-us.apache.org/repos/asf/hive/blob/b0b6db73/metastore/src/java/org/apache/hadoop/hive/metastore/IMetaStoreSchemaInfo.java
----------------------------------------------------------------------
diff --git a/metastore/src/java/org/apache/hadoop/hive/metastore/IMetaStoreSchemaInfo.java b/metastore/src/java/org/apache/hadoop/hive/metastore/IMetaStoreSchemaInfo.java
deleted file mode 100644
index 0a5f978..0000000
--- a/metastore/src/java/org/apache/hadoop/hive/metastore/IMetaStoreSchemaInfo.java
+++ /dev/null
@@ -1,107 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.hive.metastore;
-
-import org.apache.hadoop.hive.common.classification.InterfaceAudience;
-
-import java.sql.Connection;
-import java.util.List;
-import org.apache.hadoop.hive.metastore.tools.HiveSchemaHelper;
-
-/**
- * Defines the method which must be implemented to be used using schema tool to support metastore
- * schema upgrades. The configuration hive.metastore.schema.info.class is used to create instances
- * of this type by SchemaTool.
- *
- * Instances of this interface should be created using MetaStoreSchemaInfoFactory class which uses
- * two Strings argument constructor to instantiate the implementations of this interface
- */
-@InterfaceAudience.Private
-public interface IMetaStoreSchemaInfo {
- String SQL_FILE_EXTENSION = ".sql";
-
- /***
- * Get the list of sql scripts required to upgrade from the give version to current.
- *
- * @param fromVersion
- * @return
- * @throws HiveMetaException
- */
- List<String> getUpgradeScripts(String fromVersion) throws HiveMetaException;
-
- /***
- * Get the name of the script to initialize the schema for given version
- *
- * @param toVersion Target version. If it's null, then the current server version is used
- * @return
- * @throws HiveMetaException
- */
- String generateInitFileName(String toVersion) throws HiveMetaException;
-
- /**
- * Find the directory of metastore scripts
- *
- * @return the path of directory where the sql scripts are
- */
- String getMetaStoreScriptDir();
-
- /**
- * Get the pre-upgrade script for a given script name. Schema tool runs the pre-upgrade scripts
- * returned by this method before running any upgrade scripts. These scripts could contain setup
- * statements may fail on some database versions and failure is ignorable.
- *
- * @param index - index number of the file. The preupgrade script name is derived using the given
- * index
- * @param scriptName - upgrade script name
- * @return name of the pre-upgrade script to be run before running upgrade script
- */
- String getPreUpgradeScriptName(int index, String scriptName);
-
- /**
- * Get hive distribution schema version. Schematool uses this version to identify
- * the Hive version. It compares this version with the version found in metastore database
- * to determine the upgrade or initialization scripts
- * @return Hive schema version
- */
- String getHiveSchemaVersion();
-
- /**
- * Get the schema version from the backend database. This version is used by SchemaTool to to
- * compare the version returned by getHiveSchemaVersion and determine the upgrade order and
- * scripts needed to upgrade the metastore schema
- *
- * @param metastoreDbConnectionInfo Connection information needed to connect to the backend
- * database
- * @return
- * @throws HiveMetaException when unable to fetch the schema version
- */
- String getMetaStoreSchemaVersion(
- HiveSchemaHelper.MetaStoreConnectionInfo metastoreDbConnectionInfo) throws HiveMetaException;
- /**
- * A dbVersion is compatible with hive version if it is greater or equal to the hive version. This
- * is result of the db schema upgrade design principles followed in hive project. The state where
- * db schema version is ahead of hive software version is often seen when a 'rolling upgrade' or
- * 'rolling downgrade' is happening. This is a state where hive is functional and returning non
- * zero status for it is misleading.
- *
- * @param productVersion version of hive software
- * @param dbVersion version of metastore rdbms schema
- * @return true if versions are compatible
- */
- boolean isVersionCompatible(String productVersion, String dbVersion);
-}
http://git-wip-us.apache.org/repos/asf/hive/blob/b0b6db73/metastore/src/java/org/apache/hadoop/hive/metastore/LinearExtrapolatePartStatus.java
----------------------------------------------------------------------
diff --git a/metastore/src/java/org/apache/hadoop/hive/metastore/LinearExtrapolatePartStatus.java b/metastore/src/java/org/apache/hadoop/hive/metastore/LinearExtrapolatePartStatus.java
deleted file mode 100644
index f4e5ef7..0000000
--- a/metastore/src/java/org/apache/hadoop/hive/metastore/LinearExtrapolatePartStatus.java
+++ /dev/null
@@ -1,106 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.hive.metastore;
-
-import java.math.BigDecimal;
-import java.util.Map;
-
-public class LinearExtrapolatePartStatus implements IExtrapolatePartStatus {
-
- @Override
- public Object extrapolate(Object[] min, Object[] max, int colStatIndex,
- Map<String, Integer> indexMap) {
- int rightBorderInd = indexMap.size() - 1;
- int minInd = indexMap.get((String) min[1]);
- int maxInd = indexMap.get((String) max[1]);
- if (minInd == maxInd) {
- return min[0];
- }
- //note that recent metastore stores decimal in string.
- double decimalmin= 0;
- double decimalmax = 0;
- if (colStatTypes[colStatIndex] == ColStatType.Decimal) {
- BigDecimal bdmin = new BigDecimal(min[0].toString());
- decimalmin = bdmin.doubleValue();
- BigDecimal bdmax = new BigDecimal(max[0].toString());
- decimalmax = bdmax.doubleValue();
- }
- if (aggrTypes[colStatIndex] == AggrType.Max) {
- if (minInd < maxInd) {
- // right border is the max
- if (colStatTypes[colStatIndex] == ColStatType.Long) {
- return (Long) ((Long) min[0] + (((Long) max[0] - (Long) min[0])
- * (rightBorderInd - minInd) / (maxInd - minInd)));
- } else if (colStatTypes[colStatIndex] == ColStatType.Double) {
- return (Double) ((Double) min[0] + (((Double) max[0] - (Double) min[0])
- * (rightBorderInd - minInd) / (maxInd - minInd)));
- } else {
- double ret = decimalmin + (decimalmax - decimalmin)
- * (rightBorderInd - minInd) / (maxInd - minInd);
- return String.valueOf(ret);
- }
- } else {
- // left border is the max
- if (colStatTypes[colStatIndex] == ColStatType.Long) {
- return (Long) ((Long) min[0] + ((Long) max[0] - (Long) min[0])
- * minInd / (minInd - maxInd));
- } else if (colStatTypes[colStatIndex] == ColStatType.Double) {
- return (Double) ((Double) min[0] + ((Double) max[0] - (Double) min[0])
- * minInd / (minInd - maxInd));
- } else {
- double ret = decimalmin + (decimalmax - decimalmin) * minInd
- / (minInd - maxInd);
- return String.valueOf(ret);
- }
- }
- } else {
- if (minInd < maxInd) {
- // left border is the min
- if (colStatTypes[colStatIndex] == ColStatType.Long) {
- Long ret = (Long) max[0] - ((Long) max[0] - (Long) min[0]) * maxInd
- / (maxInd - minInd);
- return ret;
- } else if (colStatTypes[colStatIndex] == ColStatType.Double) {
- Double ret = (Double) max[0] - ((Double) max[0] - (Double) min[0])
- * maxInd / (maxInd - minInd);
- return ret;
- } else {
- double ret = decimalmax - (decimalmax - decimalmin) * maxInd
- / (maxInd - minInd);
- return String.valueOf(ret);
- }
- } else {
- // right border is the min
- if (colStatTypes[colStatIndex] == ColStatType.Long) {
- Long ret = (Long) max[0] - ((Long) max[0] - (Long) min[0])
- * (rightBorderInd - maxInd) / (minInd - maxInd);
- return ret;
- } else if (colStatTypes[colStatIndex] == ColStatType.Double) {
- Double ret = (Double) max[0] - ((Double) max[0] - (Double) min[0])
- * (rightBorderInd - maxInd) / (minInd - maxInd);
- return ret;
- } else {
- double ret = decimalmax - (decimalmax - decimalmin)
- * (rightBorderInd - maxInd) / (minInd - maxInd);
- return String.valueOf(ret);
- }
- }
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/hive/blob/b0b6db73/metastore/src/java/org/apache/hadoop/hive/metastore/LockComponentBuilder.java
----------------------------------------------------------------------
diff --git a/metastore/src/java/org/apache/hadoop/hive/metastore/LockComponentBuilder.java b/metastore/src/java/org/apache/hadoop/hive/metastore/LockComponentBuilder.java
deleted file mode 100644
index e074152..0000000
--- a/metastore/src/java/org/apache/hadoop/hive/metastore/LockComponentBuilder.java
+++ /dev/null
@@ -1,121 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.hive.metastore;
-
-import org.apache.hadoop.hive.metastore.api.DataOperationType;
-import org.apache.hadoop.hive.metastore.api.LockComponent;
-import org.apache.hadoop.hive.metastore.api.LockLevel;
-import org.apache.hadoop.hive.metastore.api.LockType;
-
-/**
- * A builder for {@link LockComponent}s
- */
-public class LockComponentBuilder {
- private LockComponent component;
- private boolean tableNameSet;
- private boolean partNameSet;
-
- public LockComponentBuilder() {
- component = new LockComponent();
- tableNameSet = partNameSet = false;
- }
-
- /**
- * Set the lock to be exclusive.
- * @return reference to this builder
- */
- public LockComponentBuilder setExclusive() {
- component.setType(LockType.EXCLUSIVE);
- return this;
- }
-
- /**
- * Set the lock to be semi-shared.
- * @return reference to this builder
- */
- public LockComponentBuilder setSemiShared() {
- component.setType(LockType.SHARED_WRITE);
- return this;
- }
-
- /**
- * Set the lock to be shared.
- * @return reference to this builder
- */
- public LockComponentBuilder setShared() {
- component.setType(LockType.SHARED_READ);
- return this;
- }
-
- /**
- * Set the database name.
- * @param dbName database name
- * @return reference to this builder
- */
- public LockComponentBuilder setDbName(String dbName) {
- component.setDbname(dbName);
- return this;
- }
-
- public LockComponentBuilder setOperationType(DataOperationType dop) {
- component.setOperationType(dop);
- return this;
- }
-
- public LockComponentBuilder setIsAcid(boolean t) {
- component.setIsAcid(t);
- return this;
- }
- /**
- * Set the table name.
- * @param tableName table name
- * @return reference to this builder
- */
- public LockComponentBuilder setTableName(String tableName) {
- component.setTablename(tableName);
- tableNameSet = true;
- return this;
- }
-
- /**
- * Set the partition name.
- * @param partitionName partition name
- * @return reference to this builder
- */
- public LockComponentBuilder setPartitionName(String partitionName) {
- component.setPartitionname(partitionName);
- partNameSet = true;
- return this;
- }
- public LockComponentBuilder setIsDynamicPartitionWrite(boolean t) {
- component.setIsDynamicPartitionWrite(t);
- return this;
- }
-
- /**
- * Get the constructed lock component.
- * @return lock component.
- */
- public LockComponent build() {
- LockLevel level = LockLevel.DB;
- if (tableNameSet) level = LockLevel.TABLE;
- if (partNameSet) level = LockLevel.PARTITION;
- component.setLevel(level);
- return component;
- }
-}
http://git-wip-us.apache.org/repos/asf/hive/blob/b0b6db73/metastore/src/java/org/apache/hadoop/hive/metastore/LockRequestBuilder.java
----------------------------------------------------------------------
diff --git a/metastore/src/java/org/apache/hadoop/hive/metastore/LockRequestBuilder.java b/metastore/src/java/org/apache/hadoop/hive/metastore/LockRequestBuilder.java
deleted file mode 100644
index 00a1907..0000000
--- a/metastore/src/java/org/apache/hadoop/hive/metastore/LockRequestBuilder.java
+++ /dev/null
@@ -1,168 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.hive.metastore;
-
-import org.apache.hadoop.hive.metastore.api.LockComponent;
-import org.apache.hadoop.hive.metastore.api.LockRequest;
-import org.apache.hadoop.hive.metastore.api.LockType;
-
-import java.net.InetAddress;
-import java.net.UnknownHostException;
-import java.util.LinkedHashMap;
-import java.util.Map;
-
-/**
- * Builder class to make constructing {@link LockRequest} easier.
- */
-public class LockRequestBuilder {
-
- private LockRequest req;
- private LockTrie trie;
- private boolean userSet;
-
- /**
- * @deprecated
- */
- public LockRequestBuilder() {
- this(null);
- }
- public LockRequestBuilder(String agentInfo) {
- req = new LockRequest();
- trie = new LockTrie();
- userSet = false;
- if(agentInfo != null) {
- req.setAgentInfo(agentInfo);
- }
- }
-
- /**
- * Get the constructed LockRequest.
- * @return lock request
- */
- public LockRequest build() {
- if (!userSet) {
- throw new RuntimeException("Cannot build a lock without giving a user");
- }
- trie.addLocksToRequest(req);
- try {
- req.setHostname(InetAddress.getLocalHost().getHostName());
- } catch (UnknownHostException e) {
- throw new RuntimeException("Unable to determine our local host!");
- }
- return req;
- }
-
- /**
- * Set the transaction id.
- * @param txnid transaction id
- * @return reference to this builder
- */
- public LockRequestBuilder setTransactionId(long txnid) {
- req.setTxnid(txnid);
- return this;
- }
-
- public LockRequestBuilder setUser(String user) {
- if (user == null) user = "unknown";
- req.setUser(user);
- userSet = true;
- return this;
- }
-
- /**
- * Add a lock component to the lock request
- * @param component to add
- * @return reference to this builder
- */
- public LockRequestBuilder addLockComponent(LockComponent component) {
- trie.add(component);
- return this;
- }
-
- // For reasons that are completely incomprehensible to me the semantic
- // analyzers often ask for multiple locks on the same entity (for example
- // a shared_read and an exlcusive lock). The db locking system gets confused
- // by this and dead locks on it. To resolve that, we'll make sure in the
- // request that multiple locks are coalesced and promoted to the higher
- // level of locking. To do this we put all locks components in trie based
- // on dbname, tablename, partition name and handle the promotion as new
- // requests come in. This structure depends on the fact that null is a
- // valid key in a LinkedHashMap. So a database lock will map to (dbname, null,
- // null).
- private static class LockTrie {
- Map<String, TableTrie> trie;
-
- LockTrie() {
- trie = new LinkedHashMap<String, TableTrie>();
- }
-
- public void add(LockComponent comp) {
- TableTrie tabs = trie.get(comp.getDbname());
- if (tabs == null) {
- tabs = new TableTrie();
- trie.put(comp.getDbname(), tabs);
- }
- setTable(comp, tabs);
- }
-
- public void addLocksToRequest(LockRequest request) {
- for (TableTrie tab : trie.values()) {
- for (PartTrie part : tab.values()) {
- for (LockComponent lock : part.values()) {
- request.addToComponent(lock);
- }
- }
- }
- }
-
- private void setTable(LockComponent comp, TableTrie tabs) {
- PartTrie parts = tabs.get(comp.getTablename());
- if (parts == null) {
- parts = new PartTrie();
- tabs.put(comp.getTablename(), parts);
- }
- setPart(comp, parts);
- }
-
- private void setPart(LockComponent comp, PartTrie parts) {
- LockComponent existing = parts.get(comp.getPartitionname());
- if (existing == null) {
- // No existing lock for this partition.
- parts.put(comp.getPartitionname(), comp);
- } else if (existing.getType() != LockType.EXCLUSIVE &&
- (comp.getType() == LockType.EXCLUSIVE ||
- comp.getType() == LockType.SHARED_WRITE)) {
- // We only need to promote if comp.type is > existing.type. For
- // efficiency we check if existing is exclusive (in which case we
- // need never promote) or if comp is exclusive or shared_write (in
- // which case we can promote even though they may both be shared
- // write). If comp is shared_read there's never a need to promote.
- parts.put(comp.getPartitionname(), comp);
- }
- }
-
- private static class TableTrie extends LinkedHashMap<String, PartTrie> {
- }
-
- private static class PartTrie extends LinkedHashMap<String, LockComponent> {
- }
-
-
-
- }
-}
http://git-wip-us.apache.org/repos/asf/hive/blob/b0b6db73/metastore/src/java/org/apache/hadoop/hive/metastore/MetaStoreDirectSql.java
----------------------------------------------------------------------
diff --git a/metastore/src/java/org/apache/hadoop/hive/metastore/MetaStoreDirectSql.java b/metastore/src/java/org/apache/hadoop/hive/metastore/MetaStoreDirectSql.java
index dc1245e..c1a8efe 100644
--- a/metastore/src/java/org/apache/hadoop/hive/metastore/MetaStoreDirectSql.java
+++ b/metastore/src/java/org/apache/hadoop/hive/metastore/MetaStoreDirectSql.java
@@ -1314,7 +1314,7 @@ class MetaStoreDirectSql {
AggrColStats colStatsAggrCached;
List<ColumnStatisticsObj> colStatsAggrFromDB;
int maxPartsPerCacheNode = aggrStatsCache.getMaxPartsPerCacheNode();
- float fpp = aggrStatsCache.getFalsePositiveProbability();
+ double fpp = aggrStatsCache.getFalsePositiveProbability();
colStatsList = new ArrayList<ColumnStatisticsObj>();
// Bloom filter for the new node that we will eventually add to the cache
BloomFilter bloomFilter = createPartsBloomFilter(maxPartsPerCacheNode, fpp, partNames);
@@ -1356,7 +1356,7 @@ class MetaStoreDirectSql {
return new AggrStats(colStatsList, partsFound);
}
- private BloomFilter createPartsBloomFilter(int maxPartsPerCacheNode, float fpp,
+ private BloomFilter createPartsBloomFilter(int maxPartsPerCacheNode, double fpp,
List<String> partNames) {
BloomFilter bloomFilter = new BloomFilter(maxPartsPerCacheNode, fpp);
for (String partName : partNames) {
http://git-wip-us.apache.org/repos/asf/hive/blob/b0b6db73/metastore/src/java/org/apache/hadoop/hive/metastore/MetaStoreEndFunctionContext.java
----------------------------------------------------------------------
diff --git a/metastore/src/java/org/apache/hadoop/hive/metastore/MetaStoreEndFunctionContext.java b/metastore/src/java/org/apache/hadoop/hive/metastore/MetaStoreEndFunctionContext.java
deleted file mode 100644
index ae75200..0000000
--- a/metastore/src/java/org/apache/hadoop/hive/metastore/MetaStoreEndFunctionContext.java
+++ /dev/null
@@ -1,59 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.hive.metastore;
-
-/**
- * Base class which provides context to implementations of MetaStoreEndFunctionListener
- */
-
-public class MetaStoreEndFunctionContext {
-
- /**
- * whether method was successful or not.
- */
- private final boolean success;
- private final Exception e;
- private final String inputTableName;
-
- public MetaStoreEndFunctionContext(boolean success, Exception e, String inputTableName) {
- this.success = success;
- this.e = e;
- this.inputTableName = inputTableName;
- }
-
- public MetaStoreEndFunctionContext(boolean success) {
- this(success, null, null);
- }
-
- /**
- * @return whether or not the method succeeded.
- */
- public boolean isSuccess() {
- return success;
- }
-
- public Exception getException() {
- return e;
- }
-
- public String getInputTableName() {
- return inputTableName;
- }
-
-}
http://git-wip-us.apache.org/repos/asf/hive/blob/b0b6db73/metastore/src/java/org/apache/hadoop/hive/metastore/MetaStoreEndFunctionListener.java
----------------------------------------------------------------------
diff --git a/metastore/src/java/org/apache/hadoop/hive/metastore/MetaStoreEndFunctionListener.java b/metastore/src/java/org/apache/hadoop/hive/metastore/MetaStoreEndFunctionListener.java
deleted file mode 100644
index 429efab..0000000
--- a/metastore/src/java/org/apache/hadoop/hive/metastore/MetaStoreEndFunctionListener.java
+++ /dev/null
@@ -1,58 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.hive.metastore;
-
-import java.util.AbstractMap;
-
-import org.apache.hadoop.conf.Configurable;
-import org.apache.hadoop.conf.Configuration;
-
-/**
- * This abstract class needs to be extended to provide implementation of actions that need
- * to be performed when a function ends. These methods are called whenever a function ends.
- *
- * It also provides a way to add fb303 counters through the exportCounters method.
- */
-
-public abstract class MetaStoreEndFunctionListener implements Configurable {
-
- private Configuration conf;
-
- public MetaStoreEndFunctionListener(Configuration config){
- this.conf = config;
- }
-
- public abstract void onEndFunction(String functionName, MetaStoreEndFunctionContext context);
-
- // Unless this is overridden, it does nothing
- public void exportCounters(AbstractMap<String, Long> counters) {
- }
-
- @Override
- public Configuration getConf() {
- return this.conf;
- }
-
- @Override
- public void setConf(Configuration config) {
- this.conf = config;
- }
-
-
-}
http://git-wip-us.apache.org/repos/asf/hive/blob/b0b6db73/metastore/src/java/org/apache/hadoop/hive/metastore/MetaStoreFS.java
----------------------------------------------------------------------
diff --git a/metastore/src/java/org/apache/hadoop/hive/metastore/MetaStoreFS.java b/metastore/src/java/org/apache/hadoop/hive/metastore/MetaStoreFS.java
deleted file mode 100644
index 4f525a4..0000000
--- a/metastore/src/java/org/apache/hadoop/hive/metastore/MetaStoreFS.java
+++ /dev/null
@@ -1,43 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.hive.metastore;
-
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.hive.metastore.api.MetaException;
-
-/**
- * Define a set of APIs that may vary in different environments
- */
-public interface MetaStoreFS {
-
- /**
- * delete a directory
- *
- * @param f
- * @param ifPurge
- * @param recursive
- * @return true on success
- * @throws MetaException
- */
- public boolean deleteDir(FileSystem fs, Path f, boolean recursive,
- boolean ifPurge, Configuration conf) throws MetaException;
-
-}
http://git-wip-us.apache.org/repos/asf/hive/blob/b0b6db73/metastore/src/java/org/apache/hadoop/hive/metastore/MetaStoreFilterHook.java
----------------------------------------------------------------------
diff --git a/metastore/src/java/org/apache/hadoop/hive/metastore/MetaStoreFilterHook.java b/metastore/src/java/org/apache/hadoop/hive/metastore/MetaStoreFilterHook.java
deleted file mode 100644
index 2263837..0000000
--- a/metastore/src/java/org/apache/hadoop/hive/metastore/MetaStoreFilterHook.java
+++ /dev/null
@@ -1,132 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.hive.metastore;
-
-import java.util.List;
-
-import org.apache.hadoop.hive.common.classification.InterfaceAudience.LimitedPrivate;
-import org.apache.hadoop.hive.common.classification.InterfaceStability.Evolving;
-import org.apache.hadoop.hive.metastore.api.Database;
-import org.apache.hadoop.hive.metastore.api.Index;
-import org.apache.hadoop.hive.metastore.api.MetaException;
-import org.apache.hadoop.hive.metastore.api.NoSuchObjectException;
-import org.apache.hadoop.hive.metastore.api.Partition;
-import org.apache.hadoop.hive.metastore.api.PartitionSpec;
-import org.apache.hadoop.hive.metastore.api.Table;
-
-/**
- * Metadata filter hook for metastore client. This will be useful for authorization
- * plugins on hiveserver2 to filter metadata results, especially in case of
- * non-impersonation mode where the metastore doesn't know the end user's identity.
- */
-@LimitedPrivate(value = { "Apache Sentry (Incubating)" })
-@Evolving
-public interface MetaStoreFilterHook {
-
- /**
- * Filter given list of databases
- * @param dbList
- * @return List of filtered Db names
- */
- public List<String> filterDatabases(List<String> dbList) throws MetaException;
-
- /**
- * filter to given database object if applicable
- * @param dataBase
- * @return the same database if it's not filtered out
- * @throws NoSuchObjectException
- */
- public Database filterDatabase(Database dataBase) throws MetaException, NoSuchObjectException;
-
- /**
- * Filter given list of tables
- * @param dbName
- * @param tableList
- * @return List of filtered table names
- */
- public List<String> filterTableNames(String dbName, List<String> tableList) throws MetaException;
-
- /**
- * filter to given table object if applicable
- * @param table
- * @return the same table if it's not filtered out
- * @throws NoSuchObjectException
- */
- public Table filterTable(Table table) throws MetaException, NoSuchObjectException;
-
- /**
- * Filter given list of tables
- * @param tableList
- * @return List of filtered table names
- */
- public List<Table> filterTables(List<Table> tableList) throws MetaException;
-
- /**
- * Filter given list of partitions
- * @param partitionList
- * @return
- */
- public List<Partition> filterPartitions(List<Partition> partitionList) throws MetaException;
-
- /**
- * Filter given list of partition specs
- * @param partitionSpecList
- * @return
- */
- public List<PartitionSpec> filterPartitionSpecs(List<PartitionSpec> partitionSpecList)
- throws MetaException;
-
- /**
- * filter to given partition object if applicable
- * @param partition
- * @return the same partition object if it's not filtered out
- * @throws NoSuchObjectException
- */
- public Partition filterPartition(Partition partition) throws MetaException, NoSuchObjectException;
-
- /**
- * Filter given list of partition names
- * @param dbName
- * @param tblName
- * @param partitionNames
- * @return
- */
- public List<String> filterPartitionNames(String dbName, String tblName,
- List<String> partitionNames) throws MetaException;
-
- public Index filterIndex(Index index) throws MetaException, NoSuchObjectException;
-
- /**
- * Filter given list of index names
- * @param dbName
- * @param tblName
- * @param indexList
- * @return
- */
- public List<String> filterIndexNames(String dbName, String tblName,
- List<String> indexList) throws MetaException;
-
- /**
- * Filter given list of index objects
- * @param indexeList
- * @return
- */
- public List<Index> filterIndexes(List<Index> indexeList) throws MetaException;
-}
-