You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by ga...@apache.org on 2017/09/19 23:49:26 UTC
[2/3] hive git commit: HIVE-17537 Move Warehouse class to standalone
metastore. This closes #252. (Alan Gates, reviewed by Zoltan Haindrich)
http://git-wip-us.apache.org/repos/asf/hive/blob/56083008/metastore/src/java/org/apache/hadoop/hive/metastore/Warehouse.java
----------------------------------------------------------------------
diff --git a/metastore/src/java/org/apache/hadoop/hive/metastore/Warehouse.java b/metastore/src/java/org/apache/hadoop/hive/metastore/Warehouse.java
deleted file mode 100755
index 1dd50de..0000000
--- a/metastore/src/java/org/apache/hadoop/hive/metastore/Warehouse.java
+++ /dev/null
@@ -1,635 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.hive.metastore;
-
-import static org.apache.hadoop.hive.metastore.MetaStoreUtils.DATABASE_WAREHOUSE_SUFFIX;
-import static org.apache.hadoop.hive.metastore.MetaStoreUtils.DEFAULT_DATABASE_NAME;
-
-import java.io.FileNotFoundException;
-import java.io.IOException;
-import java.util.AbstractList;
-import java.util.ArrayList;
-import java.util.LinkedHashMap;
-import java.util.List;
-import java.util.Map;
-import java.util.Map.Entry;
-import java.util.regex.Matcher;
-import java.util.regex.Pattern;
-
-import org.apache.commons.lang.StringUtils;
-import org.apache.hadoop.hive.metastore.utils.HdfsUtils;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.ContentSummary;
-import org.apache.hadoop.fs.FileStatus;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.fs.permission.FsAction;
-import org.apache.hadoop.hive.common.FileUtils;
-import org.apache.hadoop.hive.common.HiveStatsUtils;
-import org.apache.hadoop.hive.common.JavaUtils;
-import org.apache.hadoop.hive.conf.HiveConf;
-import org.apache.hadoop.hive.metastore.ReplChangeManager.RecycleType;
-import org.apache.hadoop.hive.metastore.api.Database;
-import org.apache.hadoop.hive.metastore.api.FieldSchema;
-import org.apache.hadoop.hive.metastore.api.MetaException;
-import org.apache.hadoop.hive.metastore.api.Partition;
-import org.apache.hadoop.hive.metastore.api.StorageDescriptor;
-import org.apache.hadoop.hive.metastore.api.Table;
-import org.apache.hadoop.util.ReflectionUtils;
-
-/**
- * This class represents a warehouse where data of Hive tables is stored
- */
-public class Warehouse {
- private Path whRoot;
- private final Configuration conf;
- private final String whRootString;
-
- public static final Logger LOG = LoggerFactory.getLogger("hive.metastore.warehouse");
-
- private MetaStoreFS fsHandler = null;
- private boolean storageAuthCheck = false;
- private ReplChangeManager cm = null;
-
- public Warehouse(Configuration conf) throws MetaException {
- this.conf = conf;
- whRootString = HiveConf.getVar(conf, HiveConf.ConfVars.METASTOREWAREHOUSE);
- if (StringUtils.isBlank(whRootString)) {
- throw new MetaException(HiveConf.ConfVars.METASTOREWAREHOUSE.varname
- + " is not set in the config or blank");
- }
- fsHandler = getMetaStoreFsHandler(conf);
- cm = ReplChangeManager.getInstance((HiveConf)conf);
- storageAuthCheck = HiveConf.getBoolVar(conf,
- HiveConf.ConfVars.METASTORE_AUTHORIZATION_STORAGE_AUTH_CHECKS);
- }
-
- private MetaStoreFS getMetaStoreFsHandler(Configuration conf)
- throws MetaException {
- String handlerClassStr = HiveConf.getVar(conf,
- HiveConf.ConfVars.HIVE_METASTORE_FS_HANDLER_CLS);
- try {
- Class<? extends MetaStoreFS> handlerClass = (Class<? extends MetaStoreFS>) Class
- .forName(handlerClassStr, true, JavaUtils.getClassLoader());
- MetaStoreFS handler = ReflectionUtils.newInstance(handlerClass, conf);
- return handler;
- } catch (ClassNotFoundException e) {
- throw new MetaException("Error in loading MetaStoreFS handler."
- + e.getMessage());
- }
- }
-
-
- /**
- * Helper functions to convert IOException to MetaException
- */
- public static FileSystem getFs(Path f, Configuration conf) throws MetaException {
- try {
- return f.getFileSystem(conf);
- } catch (IOException e) {
- MetaStoreUtils.logAndThrowMetaException(e);
- }
- return null;
- }
-
- public FileSystem getFs(Path f) throws MetaException {
- return getFs(f, conf);
- }
-
-
- /**
- * Hadoop File System reverse lookups paths with raw ip addresses The File
- * System URI always contains the canonical DNS name of the Namenode.
- * Subsequently, operations on paths with raw ip addresses cause an exception
- * since they don't match the file system URI.
- *
- * This routine solves this problem by replacing the scheme and authority of a
- * path with the scheme and authority of the FileSystem that it maps to.
- *
- * @param path
- * Path to be canonicalized
- * @return Path with canonical scheme and authority
- */
- public static Path getDnsPath(Path path, Configuration conf) throws MetaException {
- FileSystem fs = getFs(path, conf);
- return (new Path(fs.getUri().getScheme(), fs.getUri().getAuthority(), path
- .toUri().getPath()));
- }
-
- public Path getDnsPath(Path path) throws MetaException {
- return getDnsPath(path, conf);
- }
-
- /**
- * Resolve the configured warehouse root dir with respect to the configuration
- * This involves opening the FileSystem corresponding to the warehouse root
- * dir (but that should be ok given that this is only called during DDL
- * statements for non-external tables).
- */
- public Path getWhRoot() throws MetaException {
- if (whRoot != null) {
- return whRoot;
- }
- whRoot = getDnsPath(new Path(whRootString));
- return whRoot;
- }
-
- public Path getDatabasePath(Database db) throws MetaException {
- if (db.getName().equalsIgnoreCase(DEFAULT_DATABASE_NAME)) {
- return getWhRoot();
- }
- return new Path(db.getLocationUri());
- }
-
- public Path getDefaultDatabasePath(String dbName) throws MetaException {
- if (dbName.equalsIgnoreCase(DEFAULT_DATABASE_NAME)) {
- return getWhRoot();
- }
- return new Path(getWhRoot(), dbName.toLowerCase() + DATABASE_WAREHOUSE_SUFFIX);
- }
-
- /**
- * Returns the default location of the table path using the parent database's location
- * @param db Database where the table is created
- * @param tableName table name
- * @return
- * @throws MetaException
- */
- public Path getDefaultTablePath(Database db, String tableName)
- throws MetaException {
- return getDnsPath(new Path(getDatabasePath(db), MetaStoreUtils.encodeTableName(tableName.toLowerCase())));
- }
-
- public static String getQualifiedName(Table table) {
- return table.getDbName() + "." + table.getTableName();
- }
-
- public static String getQualifiedName(Partition partition) {
- return partition.getDbName() + "." + partition.getTableName() + partition.getValues();
- }
-
- public boolean mkdirs(Path f) throws MetaException {
- FileSystem fs = null;
- try {
- fs = getFs(f);
- return FileUtils.mkdir(fs, f, conf);
- } catch (IOException e) {
- MetaStoreUtils.logAndThrowMetaException(e);
- }
- return false;
- }
-
- public boolean renameDir(Path sourcePath, Path destPath, boolean needCmRecycle) throws MetaException {
- try {
- if (needCmRecycle) {
- // Copy the source files to cmroot. As the client will move the source files to another
- // location, we should make a copy of the files to cmroot instead of moving it.
- cm.recycle(sourcePath, RecycleType.COPY, true);
- }
- FileSystem fs = getFs(sourcePath);
- return FileUtils.rename(fs, sourcePath, destPath, conf);
- } catch (Exception ex) {
- MetaStoreUtils.logAndThrowMetaException(ex);
- }
- return false;
- }
-
- void addToChangeManagement(Path file) throws MetaException {
- cm.recycle(file, RecycleType.COPY, true);
- }
-
- public boolean deleteDir(Path f, boolean recursive) throws MetaException {
- return deleteDir(f, recursive, false);
- }
-
- public boolean deleteDir(Path f, boolean recursive, boolean ifPurge) throws MetaException {
- cm.recycle(f, RecycleType.MOVE, ifPurge);
- FileSystem fs = getFs(f);
- return fsHandler.deleteDir(fs, f, recursive, ifPurge, conf);
- }
-
- public void recycleDirToCmPath(Path f, boolean ifPurge) throws MetaException {
- cm.recycle(f, RecycleType.MOVE, ifPurge);
- return;
- }
-
- public boolean isEmpty(Path path) throws IOException, MetaException {
- ContentSummary contents = getFs(path).getContentSummary(path);
- if (contents != null && contents.getFileCount() == 0 && contents.getDirectoryCount() == 1) {
- return true;
- }
- return false;
- }
-
- public boolean isWritable(Path path) throws IOException {
- if (!storageAuthCheck) {
- // no checks for non-secure hadoop installations
- return true;
- }
- if (path == null) { //what??!!
- return false;
- }
- final FileStatus stat;
- final FileSystem fs;
- try {
- fs = getFs(path);
- stat = fs.getFileStatus(path);
- HdfsUtils.checkFileAccess(fs, stat, FsAction.WRITE);
- return true;
- } catch (FileNotFoundException fnfe){
- // File named by path doesn't exist; nothing to validate.
- return true;
- } catch (Exception e) {
- // all other exceptions are considered as emanating from
- // unauthorized accesses
- if (LOG.isDebugEnabled()) {
- LOG.debug("Exception when checking if path (" + path + ")", e);
- }
- return false;
- }
- }
-
- /*
- // NOTE: This is for generating the internal path name for partitions. Users
- // should always use the MetaStore API to get the path name for a partition.
- // Users should not directly take partition values and turn it into a path
- // name by themselves, because the logic below may change in the future.
- //
- // In the future, it's OK to add new chars to the escape list, and old data
- // won't be corrupt, because the full path name in metastore is stored.
- // In that case, Hive will continue to read the old data, but when it creates
- // new partitions, it will use new names.
- static BitSet charToEscape = new BitSet(128);
- static {
- for (char c = 0; c < ' '; c++) {
- charToEscape.set(c);
- }
- char[] clist = new char[] { '"', '#', '%', '\'', '*', '/', ':', '=', '?',
- '\\', '\u00FF' };
- for (char c : clist) {
- charToEscape.set(c);
- }
- }
-
- static boolean needsEscaping(char c) {
- return c >= 0 && c < charToEscape.size() && charToEscape.get(c);
- }
- */
-
- static String escapePathName(String path) {
- return FileUtils.escapePathName(path);
- }
-
- static String unescapePathName(String path) {
- return FileUtils.unescapePathName(path);
- }
-
- /**
- * Given a partition specification, return the path corresponding to the
- * partition spec. By default, the specification does not include dynamic partitions.
- * @param spec
- * @return string representation of the partition specification.
- * @throws MetaException
- */
- public static String makePartPath(Map<String, String> spec)
- throws MetaException {
- return makePartName(spec, true);
- }
-
- /**
- * Makes a partition name from a specification
- * @param spec
- * @param addTrailingSeperator if true, adds a trailing separator e.g. 'ds=1/'
- * @return partition name
- * @throws MetaException
- */
- public static String makePartName(Map<String, String> spec,
- boolean addTrailingSeperator)
- throws MetaException {
- StringBuilder suffixBuf = new StringBuilder();
- int i = 0;
- for (Entry<String, String> e : spec.entrySet()) {
- if (e.getValue() == null || e.getValue().length() == 0) {
- throw new MetaException("Partition spec is incorrect. " + spec);
- }
- if (i>0) {
- suffixBuf.append(Path.SEPARATOR);
- }
- suffixBuf.append(escapePathName(e.getKey()));
- suffixBuf.append('=');
- suffixBuf.append(escapePathName(e.getValue()));
- i++;
- }
- if (addTrailingSeperator) {
- suffixBuf.append(Path.SEPARATOR);
- }
- return suffixBuf.toString();
- }
- /**
- * Given a dynamic partition specification, return the path corresponding to the
- * static part of partition specification. This is basically a copy of makePartName
- * but we get rid of MetaException since it is not serializable.
- * @param spec
- * @return string representation of the static part of the partition specification.
- */
- public static String makeDynamicPartName(Map<String, String> spec) {
- StringBuilder suffixBuf = new StringBuilder();
- for (Entry<String, String> e : spec.entrySet()) {
- if (e.getValue() != null && e.getValue().length() > 0) {
- suffixBuf.append(escapePathName(e.getKey()));
- suffixBuf.append('=');
- suffixBuf.append(escapePathName(e.getValue()));
- suffixBuf.append(Path.SEPARATOR);
- } else { // stop once we see a dynamic partition
- break;
- }
- }
- return suffixBuf.toString();
- }
-
- static final Pattern pat = Pattern.compile("([^/]+)=([^/]+)");
-
- private static final Pattern slash = Pattern.compile("/");
-
- /**
- * Extracts values from partition name without the column names.
- * @param name Partition name.
- * @param result The result. Must be pre-sized to the expected number of columns.
- */
- public static AbstractList<String> makeValsFromName(
- String name, AbstractList<String> result) throws MetaException {
- assert name != null;
- String[] parts = slash.split(name, 0);
- if (result == null) {
- result = new ArrayList<>(parts.length);
- for (int i = 0; i < parts.length; ++i) {
- result.add(null);
- }
- } else if (parts.length != result.size()) {
- throw new MetaException(
- "Expected " + result.size() + " components, got " + parts.length + " (" + name + ")");
- }
- for (int i = 0; i < parts.length; ++i) {
- int eq = parts[i].indexOf('=');
- if (eq <= 0) {
- throw new MetaException("Unexpected component " + parts[i]);
- }
- result.set(i, unescapePathName(parts[i].substring(eq + 1)));
- }
- return result;
- }
-
- public static LinkedHashMap<String, String> makeSpecFromName(String name)
- throws MetaException {
- if (name == null || name.isEmpty()) {
- throw new MetaException("Partition name is invalid. " + name);
- }
- LinkedHashMap<String, String> partSpec = new LinkedHashMap<String, String>();
- makeSpecFromName(partSpec, new Path(name));
- return partSpec;
- }
-
- public static void makeSpecFromName(Map<String, String> partSpec, Path currPath) {
- List<String[]> kvs = new ArrayList<String[]>();
- do {
- String component = currPath.getName();
- Matcher m = pat.matcher(component);
- if (m.matches()) {
- String k = unescapePathName(m.group(1));
- String v = unescapePathName(m.group(2));
- String[] kv = new String[2];
- kv[0] = k;
- kv[1] = v;
- kvs.add(kv);
- }
- currPath = currPath.getParent();
- } while (currPath != null && !currPath.getName().isEmpty());
-
- // reverse the list since we checked the part from leaf dir to table's base dir
- for (int i = kvs.size(); i > 0; i--) {
- partSpec.put(kvs.get(i - 1)[0], kvs.get(i - 1)[1]);
- }
- }
-
- public static Map<String, String> makeEscSpecFromName(String name) throws MetaException {
-
- if (name == null || name.isEmpty()) {
- throw new MetaException("Partition name is invalid. " + name);
- }
- LinkedHashMap<String, String> partSpec = new LinkedHashMap<String, String>();
-
- Path currPath = new Path(name);
-
- List<String[]> kvs = new ArrayList<String[]>();
- do {
- String component = currPath.getName();
- Matcher m = pat.matcher(component);
- if (m.matches()) {
- String k = m.group(1);
- String v = m.group(2);
- String[] kv = new String[2];
- kv[0] = k;
- kv[1] = v;
- kvs.add(kv);
- }
- currPath = currPath.getParent();
- } while (currPath != null && !currPath.getName().isEmpty());
-
- // reverse the list since we checked the part from leaf dir to table's base dir
- for (int i = kvs.size(); i > 0; i--) {
- partSpec.put(kvs.get(i - 1)[0], kvs.get(i - 1)[1]);
- }
-
- return partSpec;
- }
-
- /**
- * Returns the default partition path of a table within a given database and partition key value
- * pairs. It uses the database location and appends it the table name and the partition key,value
- * pairs to create the Path for the partition directory
- *
- * @param db - parent database which is used to get the base location of the partition directory
- * @param tableName - table name for the partitions
- * @param pm - Partition key value pairs
- * @return
- * @throws MetaException
- */
- public Path getDefaultPartitionPath(Database db, String tableName,
- Map<String, String> pm) throws MetaException {
- return getPartitionPath(getDefaultTablePath(db, tableName), pm);
- }
-
- /**
- * Returns the path object for the given partition key-value pairs and the base location
- *
- * @param tblPath - the base location for the partitions. Typically the table location
- * @param pm - Partition key value pairs
- * @return
- * @throws MetaException
- */
- public Path getPartitionPath(Path tblPath, Map<String, String> pm)
- throws MetaException {
- return new Path(tblPath, makePartPath(pm));
- }
-
- /**
- * Given a database, a table and the partition key value pairs this method returns the Path object
- * corresponding to the partition key value pairs. It uses the table location if available else
- * uses the database location for constructing the path corresponding to the partition key-value
- * pairs
- *
- * @param db - Parent database of the given table
- * @param table - Table for which the partition key-values are given
- * @param vals - List of values for the partition keys
- * @return Path corresponding to the partition key-value pairs
- * @throws MetaException
- */
- public Path getPartitionPath(Database db, Table table, List<String> vals)
- throws MetaException {
- List<FieldSchema> partKeys = table.getPartitionKeys();
- if (partKeys == null || (partKeys.size() != vals.size())) {
- throw new MetaException("Invalid number of partition keys found for " + table.getTableName());
- }
- Map<String, String> pm = new LinkedHashMap<>(vals.size());
- int i = 0;
- for (FieldSchema key : partKeys) {
- pm.put(key.getName(), vals.get(i));
- i++;
- }
-
- if (table.getSd().getLocation() != null) {
- return getPartitionPath(getDnsPath(new Path(table.getSd().getLocation())), pm);
- } else {
- return getDefaultPartitionPath(db, table.getTableName(), pm);
- }
- }
-
- public boolean isDir(Path f) throws MetaException {
- FileSystem fs = null;
- try {
- fs = getFs(f);
- FileStatus fstatus = fs.getFileStatus(f);
- if (!fstatus.isDir()) {
- return false;
- }
- } catch (FileNotFoundException e) {
- return false;
- } catch (IOException e) {
- MetaStoreUtils.logAndThrowMetaException(e);
- }
- return true;
- }
-
- public static String makePartName(List<FieldSchema> partCols,
- List<String> vals) throws MetaException {
- return makePartName(partCols, vals, null);
- }
-
- /**
- * @param desc
- * @return array of FileStatus objects corresponding to the files
- * making up the passed storage description
- */
- public FileStatus[] getFileStatusesForSD(StorageDescriptor desc)
- throws MetaException {
- return getFileStatusesForLocation(desc.getLocation());
- }
-
- /**
- * @param location
- * @return array of FileStatus objects corresponding to the files
- * making up the passed storage description
- */
- public FileStatus[] getFileStatusesForLocation(String location)
- throws MetaException {
- try {
- Path path = new Path(location);
- FileSystem fileSys = path.getFileSystem(conf);
- return HiveStatsUtils.getFileStatusRecurse(path, -1, fileSys);
- } catch (IOException ioe) {
- MetaStoreUtils.logAndThrowMetaException(ioe);
- }
- return null;
- }
-
- /**
- * @param table
- * @return array of FileStatus objects corresponding to the files making up the passed
- * unpartitioned table
- */
- public FileStatus[] getFileStatusesForUnpartitionedTable(Database db, Table table)
- throws MetaException {
- Path tablePath = getDnsPath(new Path(table.getSd().getLocation()));
- try {
- FileSystem fileSys = tablePath.getFileSystem(conf);
- return HiveStatsUtils.getFileStatusRecurse(tablePath, -1, fileSys);
- } catch (IOException ioe) {
- MetaStoreUtils.logAndThrowMetaException(ioe);
- }
- return null;
- }
-
- /**
- * Makes a valid partition name.
- * @param partCols The partition columns
- * @param vals The partition values
- * @param defaultStr
- * The default name given to a partition value if the respective value is empty or null.
- * @return An escaped, valid partition name.
- * @throws MetaException
- */
- public static String makePartName(List<FieldSchema> partCols,
- List<String> vals, String defaultStr) throws MetaException {
- if ((partCols.size() != vals.size()) || (partCols.size() == 0)) {
- String errorStr = "Invalid partition key & values; keys [";
- for (FieldSchema fs : partCols) {
- errorStr += (fs.getName() + ", ");
- }
- errorStr += "], values [";
- for (String val : vals) {
- errorStr += (val + ", ");
- }
- throw new MetaException(errorStr + "]");
- }
- List<String> colNames = new ArrayList<String>();
- for (FieldSchema col: partCols) {
- colNames.add(col.getName());
- }
- return FileUtils.makePartName(colNames, vals, defaultStr);
- }
-
- public static List<String> getPartValuesFromPartName(String partName)
- throws MetaException {
- LinkedHashMap<String, String> partSpec = Warehouse.makeSpecFromName(partName);
- List<String> values = new ArrayList<String>();
- values.addAll(partSpec.values());
- return values;
- }
-
- public static Map<String, String> makeSpecFromValues(List<FieldSchema> partCols,
- List<String> values) {
- Map<String, String> spec = new LinkedHashMap<String, String>();
- for (int i = 0; i < values.size(); i++) {
- spec.put(partCols.get(i).getName(), values.get(i));
- }
- return spec;
- }
-}
http://git-wip-us.apache.org/repos/asf/hive/blob/56083008/ql/src/java/org/apache/hadoop/hive/ql/exec/DDLTask.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/DDLTask.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/DDLTask.java
index 8aa2d90..714ea1f 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/DDLTask.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/DDLTask.java
@@ -2413,7 +2413,7 @@ public class DDLTask extends Task<DDLWork> implements Serializable {
if (tbl.getStorageHandler() == null) {
// If serialization.format property has the default value, it will not to be included in
// SERDE properties
- if (MetaStoreUtils.DEFAULT_SERIALIZATION_FORMAT.equals(serdeParams.get(
+ if (Warehouse.DEFAULT_SERIALIZATION_FORMAT.equals(serdeParams.get(
serdeConstants.SERIALIZATION_FORMAT))){
serdeParams.remove(serdeConstants.SERIALIZATION_FORMAT);
}
@@ -4823,11 +4823,11 @@ public class DDLTask extends Task<DDLWork> implements Serializable {
if (!sd.isSetLocation())
{
// Location is not set, leave it as-is if this is not a default DB
- if (databaseName.equalsIgnoreCase(MetaStoreUtils.DEFAULT_DATABASE_NAME))
+ if (databaseName.equalsIgnoreCase(Warehouse.DEFAULT_DATABASE_NAME))
{
// Default database name path is always ignored, use METASTOREWAREHOUSE and object name
// instead
- path = new Path(HiveConf.getVar(conf, HiveConf.ConfVars.METASTOREWAREHOUSE), MetaStoreUtils.encodeTableName(name.toLowerCase()));
+ path = new Path(HiveConf.getVar(conf, HiveConf.ConfVars.METASTOREWAREHOUSE), org.apache.hadoop.hive.metastore.utils.MetaStoreUtils.encodeTableName(name.toLowerCase()));
}
}
else
@@ -4855,7 +4855,7 @@ public class DDLTask extends Task<DDLWork> implements Serializable {
if (crtIndex.getLocation() == null) {
// Location is not set, leave it as-is if index doesn't belong to default DB
// Currently all indexes are created in current DB only
- if (Utilities.getDatabaseName(name).equalsIgnoreCase(MetaStoreUtils.DEFAULT_DATABASE_NAME)) {
+ if (Utilities.getDatabaseName(name).equalsIgnoreCase(Warehouse.DEFAULT_DATABASE_NAME)) {
// Default database name path is always ignored, use METASTOREWAREHOUSE and object name
// instead
String warehouse = HiveConf.getVar(conf, ConfVars.METASTOREWAREHOUSE);
http://git-wip-us.apache.org/repos/asf/hive/blob/56083008/ql/src/java/org/apache/hadoop/hive/ql/exec/StatsTask.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/StatsTask.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/StatsTask.java
index c22d69b..4db6806 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/StatsTask.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/StatsTask.java
@@ -22,7 +22,6 @@ package org.apache.hadoop.hive.ql.exec;
import java.io.Serializable;
import java.util.ArrayList;
import java.util.Collection;
-import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
import java.util.concurrent.Callable;
@@ -350,7 +349,7 @@ public class StatsTask extends Task<StatsWork> implements Serializable {
throws MetaException {
// prefix is of the form dbName.tblName
- String prefix = table.getDbName() + "." + MetaStoreUtils.encodeTableName(table.getTableName());
+ String prefix = table.getDbName() + "." + org.apache.hadoop.hive.metastore.utils.MetaStoreUtils.encodeTableName(table.getTableName());
if (partition != null) {
return Utilities.join(prefix, Warehouse.makePartPath(partition.getSpec()));
}
http://git-wip-us.apache.org/repos/asf/hive/blob/56083008/ql/src/java/org/apache/hadoop/hive/ql/exec/repl/bootstrap/load/table/LoadPartitions.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/repl/bootstrap/load/table/LoadPartitions.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/repl/bootstrap/load/table/LoadPartitions.java
index acf9746..cef07ad 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/repl/bootstrap/load/table/LoadPartitions.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/repl/bootstrap/load/table/LoadPartitions.java
@@ -18,10 +18,10 @@
package org.apache.hadoop.hive.ql.exec.repl.bootstrap.load.table;
import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.hive.metastore.MetaStoreUtils;
import org.apache.hadoop.hive.metastore.Warehouse;
import org.apache.hadoop.hive.metastore.api.Database;
import org.apache.hadoop.hive.metastore.api.MetaException;
+import org.apache.hadoop.hive.metastore.utils.MetaStoreUtils;
import org.apache.hadoop.hive.ql.exec.ReplCopyTask;
import org.apache.hadoop.hive.ql.exec.Task;
import org.apache.hadoop.hive.ql.exec.TaskFactory;
http://git-wip-us.apache.org/repos/asf/hive/blob/56083008/ql/src/java/org/apache/hadoop/hive/ql/exec/repl/bootstrap/load/table/LoadTable.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/repl/bootstrap/load/table/LoadTable.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/repl/bootstrap/load/table/LoadTable.java
index 9ffd152..2bf3784 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/repl/bootstrap/load/table/LoadTable.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/repl/bootstrap/load/table/LoadTable.java
@@ -18,10 +18,10 @@
package org.apache.hadoop.hive.ql.exec.repl.bootstrap.load.table;
import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.hive.metastore.MetaStoreUtils;
import org.apache.hadoop.hive.metastore.TableType;
import org.apache.hadoop.hive.metastore.api.Database;
import org.apache.hadoop.hive.metastore.api.MetaException;
+import org.apache.hadoop.hive.metastore.utils.MetaStoreUtils;
import org.apache.hadoop.hive.ql.ErrorMsg;
import org.apache.hadoop.hive.ql.exec.ReplCopyTask;
import org.apache.hadoop.hive.ql.exec.Task;
http://git-wip-us.apache.org/repos/asf/hive/blob/56083008/ql/src/java/org/apache/hadoop/hive/ql/hooks/EnforceReadOnlyTables.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/hooks/EnforceReadOnlyTables.java b/ql/src/java/org/apache/hadoop/hive/ql/hooks/EnforceReadOnlyTables.java
index 4569ed5..107ce68 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/hooks/EnforceReadOnlyTables.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/hooks/EnforceReadOnlyTables.java
@@ -18,7 +18,7 @@
package org.apache.hadoop.hive.ql.hooks;
-import static org.apache.hadoop.hive.metastore.MetaStoreUtils.DEFAULT_DATABASE_NAME;
+import static org.apache.hadoop.hive.metastore.Warehouse.DEFAULT_DATABASE_NAME;
import java.util.HashSet;
import java.util.Set;
http://git-wip-us.apache.org/repos/asf/hive/blob/56083008/ql/src/java/org/apache/hadoop/hive/ql/lockmgr/DummyTxnManager.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/lockmgr/DummyTxnManager.java b/ql/src/java/org/apache/hadoop/hive/ql/lockmgr/DummyTxnManager.java
index 24df25b..9c9a3ab 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/lockmgr/DummyTxnManager.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/lockmgr/DummyTxnManager.java
@@ -22,10 +22,8 @@ import org.slf4j.LoggerFactory;
import org.apache.hadoop.hive.common.ValidTxnList;
import org.apache.hadoop.hive.common.ValidReadTxnList;
import org.apache.hadoop.hive.conf.HiveConf;
-import org.apache.hadoop.hive.metastore.MetaStoreUtils;
import org.apache.hadoop.hive.metastore.api.Database;
import org.apache.hadoop.hive.ql.Context;
-import org.apache.hadoop.hive.ql.Driver.DriverState;
import org.apache.hadoop.hive.ql.Driver.LockedDriverState;
import org.apache.hadoop.hive.ql.ErrorMsg;
import org.apache.hadoop.hive.ql.QueryPlan;
@@ -35,7 +33,6 @@ import org.apache.hadoop.hive.ql.metadata.DummyPartition;
import org.apache.hadoop.hive.ql.metadata.HiveException;
import org.apache.hadoop.hive.ql.metadata.Partition;
import org.apache.hadoop.hive.ql.metadata.Table;
-import org.apache.hadoop.hive.ql.session.SessionState;
import org.apache.hadoop.util.ReflectionUtils;
import java.util.*;
@@ -345,7 +342,7 @@ class DummyTxnManager extends HiveTxnManagerImpl {
try {
locks.add(new HiveLockObj(
new HiveLockObject(new DummyPartition(p.getTable(), p.getTable().getDbName()
- + "/" + MetaStoreUtils.encodeTableName(p.getTable().getTableName())
+ + "/" + org.apache.hadoop.hive.metastore.utils.MetaStoreUtils.encodeTableName(p.getTable().getTableName())
+ "/" + partialName,
partialSpec), lockData), mode));
partialName += "/";
http://git-wip-us.apache.org/repos/asf/hive/blob/56083008/ql/src/java/org/apache/hadoop/hive/ql/lockmgr/HiveLockObject.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/lockmgr/HiveLockObject.java b/ql/src/java/org/apache/hadoop/hive/ql/lockmgr/HiveLockObject.java
index a514339..eb66c31 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/lockmgr/HiveLockObject.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/lockmgr/HiveLockObject.java
@@ -24,7 +24,6 @@ import org.apache.commons.lang.StringUtils;
import org.apache.commons.lang.builder.HashCodeBuilder;
import org.apache.hadoop.hive.common.StringInternUtils;
import org.apache.hadoop.hive.conf.HiveConf;
-import org.apache.hadoop.hive.metastore.MetaStoreUtils;
import org.apache.hadoop.hive.ql.metadata.DummyPartition;
import org.apache.hadoop.hive.ql.metadata.Hive;
import org.apache.hadoop.hive.ql.metadata.HiveException;
@@ -198,12 +197,12 @@ public class HiveLockObject {
}
public HiveLockObject(Table tbl, HiveLockObjectData lockData) {
- this(new String[] {tbl.getDbName(), MetaStoreUtils.encodeTableName(tbl.getTableName())}, lockData);
+ this(new String[] {tbl.getDbName(), org.apache.hadoop.hive.metastore.utils.MetaStoreUtils.encodeTableName(tbl.getTableName())}, lockData);
}
public HiveLockObject(Partition par, HiveLockObjectData lockData) {
this(new String[] {par.getTable().getDbName(),
- MetaStoreUtils.encodeTableName(par.getTable().getTableName()), par.getName()}, lockData);
+ org.apache.hadoop.hive.metastore.utils.MetaStoreUtils.encodeTableName(par.getTable().getTableName()), par.getName()}, lockData);
}
public HiveLockObject(DummyPartition par, HiveLockObjectData lockData) {
http://git-wip-us.apache.org/repos/asf/hive/blob/56083008/ql/src/java/org/apache/hadoop/hive/ql/metadata/HiveMetaStoreChecker.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/metadata/HiveMetaStoreChecker.java b/ql/src/java/org/apache/hadoop/hive/ql/metadata/HiveMetaStoreChecker.java
index 4add836..5737c66 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/metadata/HiveMetaStoreChecker.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/metadata/HiveMetaStoreChecker.java
@@ -48,7 +48,6 @@ import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hive.common.FileUtils;
import org.apache.hadoop.hive.conf.HiveConf;
import org.apache.hadoop.hive.conf.HiveConf.ConfVars;
-import org.apache.hadoop.hive.metastore.MetaStoreUtils;
import org.apache.hadoop.hive.metastore.Warehouse;
import org.apache.hadoop.hive.metastore.api.MetaException;
import org.apache.hadoop.hive.metastore.api.NoSuchObjectException;
@@ -101,7 +100,7 @@ public class HiveMetaStoreChecker {
throws HiveException, IOException {
if (dbName == null || "".equalsIgnoreCase(dbName)) {
- dbName = MetaStoreUtils.DEFAULT_DATABASE_NAME;
+ dbName = Warehouse.DEFAULT_DATABASE_NAME;
}
try {
http://git-wip-us.apache.org/repos/asf/hive/blob/56083008/ql/src/java/org/apache/hadoop/hive/ql/optimizer/index/RewriteQueryUsingAggregateIndexCtx.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/index/RewriteQueryUsingAggregateIndexCtx.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/index/RewriteQueryUsingAggregateIndexCtx.java
index dcea0e5..12e2e24 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/index/RewriteQueryUsingAggregateIndexCtx.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/index/RewriteQueryUsingAggregateIndexCtx.java
@@ -27,7 +27,6 @@ import org.apache.hadoop.hive.ql.optimizer.FieldNode;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.hive.metastore.MetaStoreUtils;
import org.apache.hadoop.hive.ql.exec.ColumnInfo;
import org.apache.hadoop.hive.ql.exec.FunctionRegistry;
import org.apache.hadoop.hive.ql.exec.GroupByOperator;
@@ -181,7 +180,7 @@ public final class RewriteQueryUsingAggregateIndexCtx implements NodeProcessorC
TableScanDesc indexTableScanDesc = new TableScanDesc(indexTableHandle);
indexTableScanDesc.setGatherStats(false);
- String k = MetaStoreUtils.encodeTableName(indexTableName) + Path.SEPARATOR;
+ String k = org.apache.hadoop.hive.metastore.utils.MetaStoreUtils.encodeTableName(indexTableName) + Path.SEPARATOR;
indexTableScanDesc.setStatsAggPrefix(k);
scanOperator.setConf(indexTableScanDesc);
http://git-wip-us.apache.org/repos/asf/hive/blob/56083008/ql/src/java/org/apache/hadoop/hive/ql/parse/ImportSemanticAnalyzer.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/ImportSemanticAnalyzer.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/ImportSemanticAnalyzer.java
index aa4c660..ed004fe 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/parse/ImportSemanticAnalyzer.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/ImportSemanticAnalyzer.java
@@ -39,7 +39,6 @@ import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hive.common.FileUtils;
import org.apache.hadoop.hive.common.StatsSetupConst;
import org.apache.hadoop.hive.conf.HiveConf;
-import org.apache.hadoop.hive.metastore.MetaStoreUtils;
import org.apache.hadoop.hive.metastore.TableType;
import org.apache.hadoop.hive.metastore.Warehouse;
import org.apache.hadoop.hive.metastore.api.Database;
@@ -839,7 +838,7 @@ public class ImportSemanticAnalyzer extends BaseSemanticAnalyzer {
tblDesc.setLocation(
wh.getDnsPath(new Path(
wh.getDefaultDatabasePath(tblDesc.getDatabaseName()),
- MetaStoreUtils.encodeTableName(tblDesc.getTableName().toLowerCase())
+ org.apache.hadoop.hive.metastore.utils.MetaStoreUtils.encodeTableName(tblDesc.getTableName().toLowerCase())
)
).toString());
http://git-wip-us.apache.org/repos/asf/hive/blob/56083008/ql/src/java/org/apache/hadoop/hive/ql/parse/MacroSemanticAnalyzer.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/MacroSemanticAnalyzer.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/MacroSemanticAnalyzer.java
index fe065f8..0ae95be 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/parse/MacroSemanticAnalyzer.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/MacroSemanticAnalyzer.java
@@ -28,11 +28,11 @@ import java.util.Set;
import java.util.Stack;
import java.util.LinkedHashSet;
+import org.apache.hadoop.hive.metastore.Warehouse;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.apache.hadoop.hive.conf.HiveConf;
import org.apache.hadoop.hive.conf.HiveConf.ConfVars;
-import org.apache.hadoop.hive.metastore.MetaStoreUtils;
import org.apache.hadoop.hive.metastore.api.Database;
import org.apache.hadoop.hive.metastore.api.FieldSchema;
import org.apache.hadoop.hive.ql.ErrorMsg;
@@ -170,7 +170,7 @@ public class MacroSemanticAnalyzer extends BaseSemanticAnalyzer {
}
private void addEntities() throws SemanticException {
- Database database = getDatabase(MetaStoreUtils.DEFAULT_DATABASE_NAME);
+ Database database = getDatabase(Warehouse.DEFAULT_DATABASE_NAME);
// This restricts macro creation to privileged users.
outputs.add(new WriteEntity(database, WriteEntity.WriteType.DDL_NO_LOCK));
}
http://git-wip-us.apache.org/repos/asf/hive/blob/56083008/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java
index db29e3a..7bdf39e 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java
@@ -10506,7 +10506,7 @@ public class SemanticAnalyzer extends BaseSemanticAnalyzer {
// db_name.table_name + partitionSec
// as the prefix for easy of read during explain and debugging.
// Currently, partition spec can only be static partition.
- String k = MetaStoreUtils.encodeTableName(tblName) + Path.SEPARATOR;
+ String k = org.apache.hadoop.hive.metastore.utils.MetaStoreUtils.encodeTableName(tblName) + Path.SEPARATOR;
tsDesc.setStatsAggPrefix(tab.getDbName()+"."+k);
// set up WriteEntity for replication
http://git-wip-us.apache.org/repos/asf/hive/blob/56083008/ql/src/java/org/apache/hadoop/hive/ql/security/authorization/plugin/sqlstd/SQLAuthorizationUtils.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/security/authorization/plugin/sqlstd/SQLAuthorizationUtils.java b/ql/src/java/org/apache/hadoop/hive/ql/security/authorization/plugin/sqlstd/SQLAuthorizationUtils.java
index 462963a..6bd29b4 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/security/authorization/plugin/sqlstd/SQLAuthorizationUtils.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/security/authorization/plugin/sqlstd/SQLAuthorizationUtils.java
@@ -29,6 +29,7 @@ import java.util.Locale;
import java.util.Map;
import java.util.Set;
+import org.apache.hadoop.hive.metastore.Warehouse;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.apache.hadoop.fs.FileStatus;
@@ -39,7 +40,6 @@ import org.apache.hadoop.hive.common.FileUtils;
import org.apache.hadoop.hive.conf.HiveConf;
import org.apache.hadoop.hive.conf.HiveConf.ConfVars;
import org.apache.hadoop.hive.metastore.IMetaStoreClient;
-import org.apache.hadoop.hive.metastore.MetaStoreUtils;
import org.apache.hadoop.hive.metastore.api.Database;
import org.apache.hadoop.hive.metastore.api.HiveObjectPrivilege;
import org.apache.hadoop.hive.metastore.api.HiveObjectRef;
@@ -274,7 +274,7 @@ public class SQLAuthorizationUtils {
return userName.equals(thriftTableObj.getOwner());
}
case DATABASE: {
- if (MetaStoreUtils.DEFAULT_DATABASE_NAME.equalsIgnoreCase(hivePrivObject.getDbname())) {
+ if (Warehouse.DEFAULT_DATABASE_NAME.equalsIgnoreCase(hivePrivObject.getDbname())) {
return true;
}
Database db = null;
http://git-wip-us.apache.org/repos/asf/hive/blob/56083008/ql/src/java/org/apache/hadoop/hive/ql/session/SessionState.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/session/SessionState.java b/ql/src/java/org/apache/hadoop/hive/ql/session/SessionState.java
index 8b64407..97c8124 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/session/SessionState.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/session/SessionState.java
@@ -17,7 +17,7 @@
*/
package org.apache.hadoop.hive.ql.session;
-import static org.apache.hadoop.hive.metastore.MetaStoreUtils.DEFAULT_DATABASE_NAME;
+import static org.apache.hadoop.hive.metastore.Warehouse.DEFAULT_DATABASE_NAME;
import java.io.File;
import java.io.IOException;
http://git-wip-us.apache.org/repos/asf/hive/blob/56083008/ql/src/test/org/apache/hadoop/hive/ql/exec/TestExecDriver.java
----------------------------------------------------------------------
diff --git a/ql/src/test/org/apache/hadoop/hive/ql/exec/TestExecDriver.java b/ql/src/test/org/apache/hadoop/hive/ql/exec/TestExecDriver.java
index b4898e2..c0c496f 100644
--- a/ql/src/test/org/apache/hadoop/hive/ql/exec/TestExecDriver.java
+++ b/ql/src/test/org/apache/hadoop/hive/ql/exec/TestExecDriver.java
@@ -26,13 +26,13 @@ import java.util.List;
import junit.framework.TestCase;
+import org.apache.hadoop.hive.metastore.Warehouse;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.apache.hadoop.fs.FSDataInputStream;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hive.conf.HiveConf;
-import org.apache.hadoop.hive.metastore.MetaStoreUtils;
import org.apache.hadoop.hive.ql.CompilationOpContext;
import org.apache.hadoop.hive.ql.DriverContext;
import org.apache.hadoop.hive.ql.QueryState;
@@ -137,7 +137,7 @@ public class TestExecDriver extends TestCase {
cols.add("key");
cols.add("value");
for (String src : srctables) {
- db.dropTable(MetaStoreUtils.DEFAULT_DATABASE_NAME, src, true, true);
+ db.dropTable(Warehouse.DEFAULT_DATABASE_NAME, src, true, true);
db.createTable(src, cols, null, TextInputFormat.class,
HiveIgnoreKeyTextOutputFormat.class);
db.loadTable(hadoopDataFile[i], src, false, true, false, false, false);
@@ -492,7 +492,7 @@ public class TestExecDriver extends TestCase {
public void testMapPlan1() throws Exception {
LOG.info("Beginning testMapPlan1");
- populateMapPlan1(db.getTable(MetaStoreUtils.DEFAULT_DATABASE_NAME, "src"));
+ populateMapPlan1(db.getTable(Warehouse.DEFAULT_DATABASE_NAME, "src"));
executePlan();
fileDiff("lt100.txt.deflate", "mapplan1.out");
}
@@ -500,7 +500,7 @@ public class TestExecDriver extends TestCase {
public void testMapPlan2() throws Exception {
LOG.info("Beginning testMapPlan2");
- populateMapPlan2(db.getTable(MetaStoreUtils.DEFAULT_DATABASE_NAME, "src"));
+ populateMapPlan2(db.getTable(Warehouse.DEFAULT_DATABASE_NAME, "src"));
executePlan();
fileDiff("lt100.txt", "mapplan2.out");
}
@@ -508,7 +508,7 @@ public class TestExecDriver extends TestCase {
public void testMapRedPlan1() throws Exception {
LOG.info("Beginning testMapRedPlan1");
- populateMapRedPlan1(db.getTable(MetaStoreUtils.DEFAULT_DATABASE_NAME,
+ populateMapRedPlan1(db.getTable(Warehouse.DEFAULT_DATABASE_NAME,
"src"));
executePlan();
fileDiff("kv1.val.sorted.txt", "mapredplan1.out");
@@ -517,7 +517,7 @@ public class TestExecDriver extends TestCase {
public void testMapRedPlan2() throws Exception {
LOG.info("Beginning testMapPlan2");
- populateMapRedPlan2(db.getTable(MetaStoreUtils.DEFAULT_DATABASE_NAME,
+ populateMapRedPlan2(db.getTable(Warehouse.DEFAULT_DATABASE_NAME,
"src"));
executePlan();
fileDiff("lt100.sorted.txt", "mapredplan2.out");
@@ -526,8 +526,8 @@ public class TestExecDriver extends TestCase {
public void testMapRedPlan3() throws Exception {
LOG.info("Beginning testMapPlan3");
- populateMapRedPlan3(db.getTable(MetaStoreUtils.DEFAULT_DATABASE_NAME,
- "src"), db.getTable(MetaStoreUtils.DEFAULT_DATABASE_NAME, "src2"));
+ populateMapRedPlan3(db.getTable(Warehouse.DEFAULT_DATABASE_NAME,
+ "src"), db.getTable(Warehouse.DEFAULT_DATABASE_NAME, "src2"));
executePlan();
fileDiff("kv1kv2.cogroup.txt", "mapredplan3.out");
}
@@ -535,7 +535,7 @@ public class TestExecDriver extends TestCase {
public void testMapRedPlan4() throws Exception {
LOG.info("Beginning testMapPlan4");
- populateMapRedPlan4(db.getTable(MetaStoreUtils.DEFAULT_DATABASE_NAME,
+ populateMapRedPlan4(db.getTable(Warehouse.DEFAULT_DATABASE_NAME,
"src"));
executePlan();
fileDiff("kv1.string-sorted.txt", "mapredplan4.out");
@@ -544,7 +544,7 @@ public class TestExecDriver extends TestCase {
public void testMapRedPlan5() throws Exception {
LOG.info("Beginning testMapPlan5");
- populateMapRedPlan5(db.getTable(MetaStoreUtils.DEFAULT_DATABASE_NAME,
+ populateMapRedPlan5(db.getTable(Warehouse.DEFAULT_DATABASE_NAME,
"src"));
executePlan();
fileDiff("kv1.string-sorted.txt", "mapredplan5.out");
@@ -553,7 +553,7 @@ public class TestExecDriver extends TestCase {
public void testMapRedPlan6() throws Exception {
LOG.info("Beginning testMapPlan6");
- populateMapRedPlan6(db.getTable(MetaStoreUtils.DEFAULT_DATABASE_NAME,
+ populateMapRedPlan6(db.getTable(Warehouse.DEFAULT_DATABASE_NAME,
"src"));
executePlan();
fileDiff("lt100.sorted.txt", "mapredplan6.out");
http://git-wip-us.apache.org/repos/asf/hive/blob/56083008/ql/src/test/org/apache/hadoop/hive/ql/metadata/TestHive.java
----------------------------------------------------------------------
diff --git a/ql/src/test/org/apache/hadoop/hive/ql/metadata/TestHive.java b/ql/src/test/org/apache/hadoop/hive/ql/metadata/TestHive.java
index 91eb033..dc7d51d 100755
--- a/ql/src/test/org/apache/hadoop/hive/ql/metadata/TestHive.java
+++ b/ql/src/test/org/apache/hadoop/hive/ql/metadata/TestHive.java
@@ -18,7 +18,7 @@
package org.apache.hadoop.hive.ql.metadata;
-import static org.apache.hadoop.hive.metastore.MetaStoreUtils.DEFAULT_DATABASE_NAME;
+import static org.apache.hadoop.hive.metastore.Warehouse.DEFAULT_DATABASE_NAME;
import java.util.ArrayList;
import java.util.Arrays;
@@ -33,7 +33,6 @@ import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hive.conf.HiveConf;
import org.apache.hadoop.hive.conf.HiveConf.ConfVars;
-import org.apache.hadoop.hive.metastore.MetaStoreUtils;
import org.apache.hadoop.hive.metastore.PartitionDropOptions;
import org.apache.hadoop.hive.metastore.Warehouse;
import org.apache.hadoop.hive.metastore.api.Database;
@@ -118,13 +117,13 @@ public class TestHive extends TestCase {
// create a simple table and test create, drop, get
String tableName = "table_for_testtable";
try {
- hm.dropTable(MetaStoreUtils.DEFAULT_DATABASE_NAME, tableName);
+ hm.dropTable(Warehouse.DEFAULT_DATABASE_NAME, tableName);
} catch (HiveException e1) {
e1.printStackTrace();
assertTrue("Unable to drop table", false);
}
- Table tbl = new Table(MetaStoreUtils.DEFAULT_DATABASE_NAME, tableName);
+ Table tbl = new Table(Warehouse.DEFAULT_DATABASE_NAME, tableName);
List<FieldSchema> fields = tbl.getCols();
fields.add(new FieldSchema("col1", serdeConstants.INT_TYPE_NAME, "int -- first column"));
@@ -184,9 +183,9 @@ public class TestHive extends TestCase {
validateTable(tbl, tableName);
try {
- hm.dropTable(MetaStoreUtils.DEFAULT_DATABASE_NAME, tableName, true,
+ hm.dropTable(Warehouse.DEFAULT_DATABASE_NAME, tableName, true,
false);
- Table ft2 = hm.getTable(MetaStoreUtils.DEFAULT_DATABASE_NAME,
+ Table ft2 = hm.getTable(Warehouse.DEFAULT_DATABASE_NAME,
tableName, false);
assertNull("Unable to drop table ", ft2);
} catch (HiveException e) {
@@ -216,12 +215,12 @@ public class TestHive extends TestCase {
String tableName = "table_for_test_thrifttable";
try {
try {
- hm.dropTable(MetaStoreUtils.DEFAULT_DATABASE_NAME, tableName);
+ hm.dropTable(Warehouse.DEFAULT_DATABASE_NAME, tableName);
} catch (HiveException e1) {
System.err.println(StringUtils.stringifyException(e1));
assertTrue("Unable to drop table", false);
}
- Table tbl = new Table(MetaStoreUtils.DEFAULT_DATABASE_NAME, tableName);
+ Table tbl = new Table(Warehouse.DEFAULT_DATABASE_NAME, tableName);
tbl.setInputFormatClass(SequenceFileInputFormat.class.getName());
tbl.setOutputFormatClass(SequenceFileOutputFormat.class.getName());
tbl.setSerializationLib(ThriftDeserializer.class.getName());
@@ -308,7 +307,7 @@ public class TestHive extends TestCase {
// (create table sets it to empty (non null) structures)
tbl.getTTable().setPrivilegesIsSet(false);
- ft = hm.getTable(MetaStoreUtils.DEFAULT_DATABASE_NAME, tableName);
+ ft = hm.getTable(Warehouse.DEFAULT_DATABASE_NAME, tableName);
assertNotNull("Unable to fetch table", ft);
ft.checkValidity(hiveConf);
assertEquals("Table names didn't match for table: " + tableName, tbl
@@ -526,7 +525,7 @@ public class TestHive extends TestCase {
* @throws Exception on failure.
*/
public void testDropPartitionsWithPurge() throws Exception {
- String dbName = MetaStoreUtils.DEFAULT_DATABASE_NAME;
+ String dbName = Warehouse.DEFAULT_DATABASE_NAME;
String tableName = "table_for_testDropPartitionsWithPurge";
try {
@@ -589,7 +588,7 @@ public class TestHive extends TestCase {
*/
public void testAutoPurgeTablesAndPartitions() throws Throwable {
- String dbName = MetaStoreUtils.DEFAULT_DATABASE_NAME;
+ String dbName = Warehouse.DEFAULT_DATABASE_NAME;
String tableName = "table_for_testAutoPurgeTablesAndPartitions";
try {
@@ -643,7 +642,7 @@ public class TestHive extends TestCase {
try {
String tableName = "table_for_testpartition";
try {
- hm.dropTable(MetaStoreUtils.DEFAULT_DATABASE_NAME, tableName);
+ hm.dropTable(Warehouse.DEFAULT_DATABASE_NAME, tableName);
} catch (HiveException e) {
System.err.println(StringUtils.stringifyException(e));
assertTrue("Unable to drop table: " + tableName, false);
@@ -664,7 +663,7 @@ public class TestHive extends TestCase {
}
Table tbl = null;
try {
- tbl = hm.getTable(MetaStoreUtils.DEFAULT_DATABASE_NAME, tableName);
+ tbl = hm.getTable(Warehouse.DEFAULT_DATABASE_NAME, tableName);
} catch (HiveException e) {
System.err.println(StringUtils.stringifyException(e));
assertTrue("Unable to fetch table: " + tableName, false);
@@ -679,7 +678,7 @@ public class TestHive extends TestCase {
System.err.println(StringUtils.stringifyException(e));
assertTrue("Unable to create parition for table: " + tableName, false);
}
- hm.dropTable(MetaStoreUtils.DEFAULT_DATABASE_NAME, tableName);
+ hm.dropTable(Warehouse.DEFAULT_DATABASE_NAME, tableName);
} catch (Throwable e) {
System.err.println(StringUtils.stringifyException(e));
System.err.println("testPartition() failed");
@@ -696,15 +695,15 @@ public class TestHive extends TestCase {
try{
// create a simple table
String tableName = "table_for_testindex";
- String qTableName = MetaStoreUtils.DEFAULT_DATABASE_NAME + "." + tableName;
+ String qTableName = Warehouse.DEFAULT_DATABASE_NAME + "." + tableName;
try {
- hm.dropTable(MetaStoreUtils.DEFAULT_DATABASE_NAME, tableName);
+ hm.dropTable(Warehouse.DEFAULT_DATABASE_NAME, tableName);
} catch (HiveException e) {
e.printStackTrace();
assertTrue("Unable to drop table", false);
}
- Table tbl = new Table(MetaStoreUtils.DEFAULT_DATABASE_NAME, tableName);
+ Table tbl = new Table(Warehouse.DEFAULT_DATABASE_NAME, tableName);
List<FieldSchema> fields = tbl.getCols();
fields.add(new FieldSchema("col1", serdeConstants.INT_TYPE_NAME, "int -- first column"));
@@ -731,7 +730,7 @@ public class TestHive extends TestCase {
List<String> indexedCols = new ArrayList<String>();
indexedCols.add("col1");
String indexTableName = "index_on_table_for_testindex_table";
- String qIndexTableName = MetaStoreUtils.DEFAULT_DATABASE_NAME + "." + indexTableName;
+ String qIndexTableName = Warehouse.DEFAULT_DATABASE_NAME + "." + indexTableName;
boolean deferredRebuild = true;
String inputFormat = SequenceFileInputFormat.class.getName();
String outputFormat = SequenceFileOutputFormat.class.getName();
@@ -776,7 +775,7 @@ public class TestHive extends TestCase {
// Drop index
try {
- hm.dropIndex(MetaStoreUtils.DEFAULT_DATABASE_NAME, tableName, indexName, false, true);
+ hm.dropIndex(Warehouse.DEFAULT_DATABASE_NAME, tableName, indexName, false, true);
} catch (HiveException e) {
System.err.println(StringUtils.stringifyException(e));
assertTrue("Unable to drop index: " + indexName, false);
http://git-wip-us.apache.org/repos/asf/hive/blob/56083008/ql/src/test/org/apache/hadoop/hive/ql/session/TestSessionState.java
----------------------------------------------------------------------
diff --git a/ql/src/test/org/apache/hadoop/hive/ql/session/TestSessionState.java b/ql/src/test/org/apache/hadoop/hive/ql/session/TestSessionState.java
index fdebb94..64a90c5 100644
--- a/ql/src/test/org/apache/hadoop/hive/ql/session/TestSessionState.java
+++ b/ql/src/test/org/apache/hadoop/hive/ql/session/TestSessionState.java
@@ -20,21 +20,18 @@ package org.apache.hadoop.hive.ql.session;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertNull;
-import java.io.BufferedReader;
import java.io.File;
import java.io.IOException;
-import java.io.InputStreamReader;
import java.lang.reflect.Method;
import java.util.Arrays;
import java.util.Collection;
import org.apache.commons.io.FileUtils;
-import org.apache.commons.io.IOUtils;
+import org.apache.hadoop.hive.metastore.Warehouse;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.apache.hadoop.hive.conf.HiveConf;
import org.apache.hadoop.hive.conf.HiveConf.ConfVars;
-import org.apache.hadoop.hive.metastore.MetaStoreUtils;
import org.apache.hive.common.util.HiveTestUtils;
import org.junit.After;
import org.junit.Assert;
@@ -112,7 +109,7 @@ public class TestSessionState {
@Test
public void testgetDbName() throws Exception {
//check that we start with default db
- assertEquals(MetaStoreUtils.DEFAULT_DATABASE_NAME,
+ assertEquals(Warehouse.DEFAULT_DATABASE_NAME,
SessionState.get().getCurrentDatabase());
final String newdb = "DB_2";
@@ -123,7 +120,7 @@ public class TestSessionState {
//verify that a new sessionstate has default db
SessionState.start(new HiveConf());
- assertEquals(MetaStoreUtils.DEFAULT_DATABASE_NAME,
+ assertEquals(Warehouse.DEFAULT_DATABASE_NAME,
SessionState.get().getCurrentDatabase());
}
http://git-wip-us.apache.org/repos/asf/hive/blob/56083008/standalone-metastore/pom.xml
----------------------------------------------------------------------
diff --git a/standalone-metastore/pom.xml b/standalone-metastore/pom.xml
index 5cdbbcb..d91b22d 100644
--- a/standalone-metastore/pom.xml
+++ b/standalone-metastore/pom.xml
@@ -63,6 +63,11 @@
<version>${dropwizard.version}</version>
</dependency>
<dependency>
+ <groupId>org.apache.commons</groupId>
+ <artifactId>commons-lang3</artifactId>
+ <version>${commons-lang3.version}</version>
+ </dependency>
+ <dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-common</artifactId>
<version>${hadoop.version}</version>
@@ -80,6 +85,12 @@
</dependency>
<dependency>
<groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-distcp</artifactId>
+ <version>${hadoop.version}</version>
+ <scope>provided</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-hdfs</artifactId>
<version>${hadoop.version}</version>
<optional>true</optional>
http://git-wip-us.apache.org/repos/asf/hive/blob/56083008/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/ReplChangeManager.java
----------------------------------------------------------------------
diff --git a/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/ReplChangeManager.java b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/ReplChangeManager.java
new file mode 100644
index 0000000..dd9296a
--- /dev/null
+++ b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/ReplChangeManager.java
@@ -0,0 +1,393 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hive.metastore;
+
+import java.io.IOException;
+import java.util.concurrent.Executors;
+import java.util.concurrent.ScheduledExecutorService;
+import java.util.concurrent.TimeUnit;
+
+import org.apache.commons.lang3.concurrent.BasicThreadFactory;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileChecksum;
+import org.apache.hadoop.fs.FileStatus;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.PathFilter;
+import org.apache.hadoop.fs.Trash;
+import org.apache.hadoop.fs.permission.FsPermission;
+import org.apache.hadoop.hdfs.DFSConfigKeys;
+import org.apache.hadoop.hive.metastore.api.MetaException;
+import org.apache.hadoop.hive.metastore.conf.MetastoreConf;
+import org.apache.hadoop.hive.metastore.conf.MetastoreConf.ConfVars;
+import org.apache.hadoop.hive.metastore.utils.FileUtils;
+import org.apache.hadoop.security.UserGroupInformation;
+import org.apache.hadoop.util.StringUtils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+public class ReplChangeManager {
+ private static final Logger LOG = LoggerFactory.getLogger(ReplChangeManager.class);
+ static private ReplChangeManager instance;
+
+ private static boolean inited = false;
+ private static boolean enabled = false;
+ private static Path cmroot;
+ private static Configuration conf;
+ private String msUser;
+ private String msGroup;
+ private FileSystem fs;
+
+ private static final String ORIG_LOC_TAG = "user.original-loc";
+ static final String REMAIN_IN_TRASH_TAG = "user.remain-in-trash";
+ private static final String URI_FRAGMENT_SEPARATOR = "#";
+
+ public enum RecycleType {
+ MOVE,
+ COPY
+ }
+
+ public static ReplChangeManager getInstance(Configuration conf) throws MetaException {
+ if (instance == null) {
+ instance = new ReplChangeManager(conf);
+ }
+ return instance;
+ }
+
+ private ReplChangeManager(Configuration conf) throws MetaException {
+ try {
+ if (!inited) {
+ if (MetastoreConf.getBoolVar(conf, ConfVars.REPLCMENABLED)) {
+ ReplChangeManager.enabled = true;
+ ReplChangeManager.cmroot = new Path(MetastoreConf.getVar(conf, ConfVars.REPLCMDIR));
+ ReplChangeManager.conf = conf;
+
+ fs = cmroot.getFileSystem(conf);
+ // Create cmroot with permission 700 if not exist
+ if (!fs.exists(cmroot)) {
+ fs.mkdirs(cmroot);
+ fs.setPermission(cmroot, new FsPermission("700"));
+ }
+ UserGroupInformation usergroupInfo = UserGroupInformation.getCurrentUser();
+ msUser = usergroupInfo.getShortUserName();
+ msGroup = usergroupInfo.getPrimaryGroupName();
+ }
+ inited = true;
+ }
+ } catch (IOException e) {
+ throw new MetaException(StringUtils.stringifyException(e));
+ }
+ }
+
+ // Filter files starts with ".". Note Hadoop consider files starts with
+ // "." or "_" as hidden file. However, we need to replicate files starts
+ // with "_". We find at least 2 use cases:
+ // 1. For har files, _index and _masterindex is required files
+ // 2. _success file is required for Oozie to indicate availability of data source
+ private static final PathFilter hiddenFileFilter = new PathFilter(){
+ public boolean accept(Path p){
+ return !p.getName().startsWith(".");
+ }
+ };
+
+ /***
+ * Move a path into cmroot. If the path is a directory (of a partition, or table if nonpartitioned),
+ * recursively move files inside directory to cmroot. Note the table must be managed table
+ * @param path a single file or directory
+ * @param type if the files to be copied or moved to cmpath.
+ * Copy is costly but preserve the source file
+ * @param ifPurge if the file should skip Trash when move/delete source file.
+ * This is referred only if type is MOVE.
+ * @return int
+ * @throws MetaException
+ */
+ int recycle(Path path, RecycleType type, boolean ifPurge) throws MetaException {
+ if (!enabled) {
+ return 0;
+ }
+
+ try {
+ int count = 0;
+
+ if (fs.isDirectory(path)) {
+ FileStatus[] files = fs.listStatus(path, hiddenFileFilter);
+ for (FileStatus file : files) {
+ count += recycle(file.getPath(), type, ifPurge);
+ }
+ } else {
+ String fileCheckSum = checksumFor(path, fs);
+ Path cmPath = getCMPath(conf, path.getName(), fileCheckSum);
+
+ // set timestamp before moving to cmroot, so we can
+ // avoid race condition CM remove the file before setting
+ // timestamp
+ long now = System.currentTimeMillis();
+ fs.setTimes(path, now, -1);
+
+ boolean success = false;
+ if (fs.exists(cmPath) && fileCheckSum.equalsIgnoreCase(checksumFor(cmPath, fs))) {
+ // If already a file with same checksum exists in cmPath, just ignore the copy/move
+ // Also, mark the operation is unsuccessful to notify that file with same name already
+ // exist which will ensure the timestamp of cmPath is updated to avoid clean-up by
+ // CM cleaner.
+ success = false;
+ } else {
+ switch (type) {
+ case MOVE: {
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("Moving {} to {}", path.toString(), cmPath.toString());
+ }
+ // Rename fails if the file with same name already exist.
+ success = fs.rename(path, cmPath);
+ break;
+ }
+ case COPY: {
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("Copying {} to {}", path.toString(), cmPath.toString());
+ }
+ // It is possible to have a file with same checksum in cmPath but the content is
+ // partially copied or corrupted. In this case, just overwrite the existing file with
+ // new one.
+ success = FileUtils.copy(fs, path, fs, cmPath, false, true, conf);
+ break;
+ }
+ default:
+ // Operation fails as invalid input
+ break;
+ }
+ }
+
+ // Ignore if a file with same content already exist in cmroot
+ // We might want to setXAttr for the new location in the future
+ if (success) {
+ // set the file owner to hive (or the id metastore run as)
+ fs.setOwner(cmPath, msUser, msGroup);
+
+ // tag the original file name so we know where the file comes from
+ // Note we currently only track the last known trace as
+ // xattr has limited capacity. We shall revisit and store all original
+ // locations if orig-loc becomes important
+ try {
+ fs.setXAttr(cmPath, ORIG_LOC_TAG, path.toString().getBytes());
+ } catch (UnsupportedOperationException e) {
+ LOG.warn("Error setting xattr for {}", path.toString());
+ }
+
+ count++;
+ } else {
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("A file with the same content of {} already exists, ignore", path.toString());
+ }
+ // Need to extend the tenancy if we saw a newer file with the same content
+ fs.setTimes(cmPath, now, -1);
+ }
+
+ // Tag if we want to remain in trash after deletion.
+ // If multiple files share the same content, then
+ // any file claim remain in trash would be granted
+ if ((type == RecycleType.MOVE) && !ifPurge) {
+ try {
+ fs.setXAttr(cmPath, REMAIN_IN_TRASH_TAG, new byte[]{0});
+ } catch (UnsupportedOperationException e) {
+ LOG.warn("Error setting xattr for {}", cmPath.toString());
+ }
+ }
+ }
+ return count;
+ } catch (IOException e) {
+ throw new MetaException(StringUtils.stringifyException(e));
+ }
+ }
+
+ // Get checksum of a file
+ static public String checksumFor(Path path, FileSystem fs) throws IOException {
+ // TODO: fs checksum only available on hdfs, need to
+ // find a solution for other fs (eg, local fs, s3, etc)
+ String checksumString = null;
+ FileChecksum checksum = fs.getFileChecksum(path);
+ if (checksum != null) {
+ checksumString = StringUtils.byteToHexString(
+ checksum.getBytes(), 0, checksum.getLength());
+ }
+ return checksumString;
+ }
+
+ static public void setCmRoot(Path cmRoot) {
+ ReplChangeManager.cmroot = cmRoot;
+ }
+
+ /***
+ * Convert a path of file inside a partition or table (if non-partitioned)
+ * to a deterministic location of cmroot. So user can retrieve the file back
+ * with the original location plus checksum.
+ * @param conf
+ * @param name original filename
+ * @param checkSum checksum of the file, can be retrieved by {@link #checksumFor(Path, FileSystem)}
+ * @return Path
+ */
+ static Path getCMPath(Configuration conf, String name, String checkSum) throws IOException, MetaException {
+ String newFileName = name + "_" + checkSum;
+ int maxLength = conf.getInt(DFSConfigKeys.DFS_NAMENODE_MAX_COMPONENT_LENGTH_KEY,
+ DFSConfigKeys.DFS_NAMENODE_MAX_COMPONENT_LENGTH_DEFAULT);
+
+ if (newFileName.length() > maxLength) {
+ newFileName = newFileName.substring(0, maxLength-1);
+ }
+
+ return new Path(cmroot, newFileName);
+ }
+
+ /***
+ * Get original file specified by src and chksumString. If the file exists and checksum
+ * matches, return the file; otherwise, use chksumString to retrieve it from cmroot
+ * @param src Original file location
+ * @param checksumString Checksum of the original file
+ * @param conf
+ * @return Corresponding FileStatus object
+ */
+ static public FileStatus getFileStatus(Path src, String checksumString,
+ Configuration conf) throws MetaException {
+ try {
+ FileSystem srcFs = src.getFileSystem(conf);
+ if (checksumString == null) {
+ return srcFs.getFileStatus(src);
+ }
+
+ if (!srcFs.exists(src)) {
+ return srcFs.getFileStatus(getCMPath(conf, src.getName(), checksumString));
+ }
+
+ String currentChecksumString = checksumFor(src, srcFs);
+ if (currentChecksumString == null || checksumString.equals(currentChecksumString)) {
+ return srcFs.getFileStatus(src);
+ } else {
+ return srcFs.getFileStatus(getCMPath(conf, src.getName(), checksumString));
+ }
+ } catch (IOException e) {
+ throw new MetaException(StringUtils.stringifyException(e));
+ }
+ }
+
+ /***
+ * Concatenate filename and checksum with "#"
+ * @param fileUriStr Filename string
+ * @param fileChecksum Checksum string
+ * @return Concatenated Uri string
+ */
+ // TODO: this needs to be enhanced once change management based filesystem is implemented
+ // Currently using fileuri#checksum as the format
+ static public String encodeFileUri(String fileUriStr, String fileChecksum) {
+ if (fileChecksum != null) {
+ return fileUriStr + URI_FRAGMENT_SEPARATOR + fileChecksum;
+ } else {
+ return fileUriStr;
+ }
+ }
+
+ /***
+ * Split uri with fragment into file uri and checksum
+ * @param fileURIStr uri with fragment
+ * @return array of file name and checksum
+ */
+ static public String[] getFileWithChksumFromURI(String fileURIStr) {
+ String[] uriAndFragment = fileURIStr.split(URI_FRAGMENT_SEPARATOR);
+ String[] result = new String[2];
+ result[0] = uriAndFragment[0];
+ if (uriAndFragment.length>1) {
+ result[1] = uriAndFragment[1];
+ }
+ return result;
+ }
+
+ public static boolean isCMFileUri(Path fromPath, FileSystem srcFs) {
+ String[] result = getFileWithChksumFromURI(fromPath.toString());
+ return result[1] != null;
+ }
+
+ /**
+ * Thread to clear old files of cmroot recursively
+ */
+ static class CMClearer implements Runnable {
+ private Path cmroot;
+ private long secRetain;
+ private Configuration conf;
+
+ CMClearer(String cmrootString, long secRetain, Configuration conf) {
+ this.cmroot = new Path(cmrootString);
+ this.secRetain = secRetain;
+ this.conf = conf;
+ }
+
+ @Override
+ public void run() {
+ try {
+ LOG.info("CMClearer started");
+
+ long now = System.currentTimeMillis();
+ FileSystem fs = cmroot.getFileSystem(conf);
+ FileStatus[] files = fs.listStatus(cmroot);
+
+ for (FileStatus file : files) {
+ long modifiedTime = file.getModificationTime();
+ if (now - modifiedTime > secRetain*1000) {
+ try {
+ if (fs.getXAttrs(file.getPath()).containsKey(REMAIN_IN_TRASH_TAG)) {
+ boolean succ = Trash.moveToAppropriateTrash(fs, file.getPath(), conf);
+ if (succ) {
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("Move " + file.toString() + " to trash");
+ }
+ } else {
+ LOG.warn("Fail to move " + file.toString() + " to trash");
+ }
+ } else {
+ boolean succ = fs.delete(file.getPath(), false);
+ if (succ) {
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("Remove " + file.toString());
+ }
+ } else {
+ LOG.warn("Fail to remove " + file.toString());
+ }
+ }
+ } catch (UnsupportedOperationException e) {
+ LOG.warn("Error getting xattr for " + file.getPath().toString());
+ }
+ }
+ }
+ } catch (IOException e) {
+ LOG.error("Exception when clearing cmroot:" + StringUtils.stringifyException(e));
+ }
+ }
+ }
+
+ // Schedule CMClearer thread. Will be invoked by metastore
+ static void scheduleCMClearer(Configuration conf) {
+ if (MetastoreConf.getBoolVar(conf, ConfVars.REPLCMENABLED)) {
+ ScheduledExecutorService executor = Executors.newSingleThreadScheduledExecutor(
+ new BasicThreadFactory.Builder()
+ .namingPattern("cmclearer-%d")
+ .daemon(true)
+ .build());
+ executor.scheduleAtFixedRate(new CMClearer(MetastoreConf.getVar(conf, ConfVars.REPLCMDIR),
+ MetastoreConf.getTimeVar(conf, ConfVars.REPLCMRETIAN, TimeUnit.SECONDS), conf),
+ 0, MetastoreConf.getTimeVar(conf, ConfVars.REPLCMINTERVAL, TimeUnit.SECONDS), TimeUnit.SECONDS);
+ }
+ }
+}