You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by kg...@apache.org on 2019/07/09 15:47:00 UTC
[hive] 04/04: HIVE-21933: Remove unused methods from Utilities
(Ivan Suller via Laszlo Bodor)
This is an automated email from the ASF dual-hosted git repository.
kgyrtkirk pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hive.git
commit 33d2995bc2612e0d3e9ad0cfc2a88a9a7f5141bc
Author: Ivan Suller <is...@cloudera.com>
AuthorDate: Tue Jul 9 17:43:09 2019 +0200
HIVE-21933: Remove unused methods from Utilities (Ivan Suller via Laszlo Bodor)
Signed-off-by: Zoltan Haindrich <ki...@rxd.hu>
---
.../org/apache/hadoop/hive/ql/exec/Utilities.java | 155 ++-------------------
.../apache/hadoop/hive/ql/exec/TestUtilities.java | 9 +-
2 files changed, 12 insertions(+), 152 deletions(-)
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java
index 68986dd..1346bed 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java
@@ -404,7 +404,7 @@ public final class Utilities {
/**
* Pushes work into the global work map
*/
- public static void setBaseWork(Configuration conf, String name, BaseWork work) {
+ private static void setBaseWork(Configuration conf, String name, BaseWork work) {
Path path = getPlanPath(conf, name);
setHasWork(conf, name);
gWorkMap.get(conf).put(path, work);
@@ -753,14 +753,6 @@ public final class Utilities {
}
}
- public static HashMap makeMap(Object... olist) {
- HashMap ret = new HashMap();
- for (int i = 0; i < olist.length; i += 2) {
- ret.put(olist[i], olist[i + 1]);
- }
- return (ret);
- }
-
public static Properties makeProperties(String... olist) {
Properties ret = new Properties();
for (int i = 0; i < olist.length; i += 2) {
@@ -799,37 +791,11 @@ public final class Utilities {
return new PartitionDesc(part, tableDesc);
}
- public static PartitionDesc getPartitionDesc(Partition part) throws HiveException {
- return new PartitionDesc(part);
- }
-
public static PartitionDesc getPartitionDescFromTableDesc(TableDesc tblDesc, Partition part,
boolean usePartSchemaProperties) throws HiveException {
return new PartitionDesc(part, tblDesc, usePartSchemaProperties);
}
- private static String getOpTreeSkel_helper(Operator<?> op, String indent) {
- if (op == null) {
- return StringUtils.EMPTY;
- }
-
- StringBuilder sb = new StringBuilder();
- sb.append(indent);
- sb.append(op.toString());
- sb.append("\n");
- if (op.getChildOperators() != null) {
- for (Object child : op.getChildOperators()) {
- sb.append(getOpTreeSkel_helper((Operator<?>) child, indent + " "));
- }
- }
-
- return sb.toString();
- }
-
- public static String getOpTreeSkel(Operator<?> op) {
- return getOpTreeSkel_helper(op, StringUtils.EMPTY);
- }
-
private static boolean isWhitespace(int c) {
if (c == -1) {
return false;
@@ -926,22 +892,6 @@ public final class Utilities {
}
/**
- * Convert an output stream to a compressed output stream based on codecs and compression options
- * specified in the Job Configuration.
- *
- * @param jc
- * Job Configuration
- * @param out
- * Output Stream to be converted into compressed output stream
- * @return compressed output stream
- */
- public static OutputStream createCompressedStream(JobConf jc, OutputStream out)
- throws IOException {
- boolean isCompressed = FileOutputFormat.getCompressOutput(jc);
- return createCompressedStream(jc, out, isCompressed);
- }
-
- /**
* Convert an output stream to a compressed output stream based on codecs codecs in the Job
* Configuration. Caller specifies directly whether file is compressed or not
*
@@ -966,22 +916,6 @@ public final class Utilities {
}
/**
- * Based on compression option and configured output codec - get extension for output file. This
- * is only required for text files - not sequencefiles
- *
- * @param jc
- * Job Configuration
- * @param isCompressed
- * Whether the output file is compressed or not
- * @return the required file extension (example: .gz)
- * @deprecated Use {@link #getFileExtension(JobConf, boolean, HiveOutputFormat)}
- */
- @Deprecated
- public static String getFileExtension(JobConf jc, boolean isCompressed) {
- return getFileExtension(jc, isCompressed, new HiveIgnoreKeyTextOutputFormat());
- }
-
- /**
* Based on compression option, output format, and configured output codec -
* get extension for output file. Text files require an extension, whereas
* others, like sequence files, do not.
@@ -1014,27 +948,6 @@ public final class Utilities {
}
/**
- * Create a sequencefile output stream based on job configuration.
- *
- * @param jc
- * Job configuration
- * @param fs
- * File System to create file in
- * @param file
- * Path to be created
- * @param keyClass
- * Java Class for key
- * @param valClass
- * Java Class for value
- * @return output stream over the created sequencefile
- */
- public static SequenceFile.Writer createSequenceWriter(JobConf jc, FileSystem fs, Path file,
- Class<?> keyClass, Class<?> valClass, Progressable progressable) throws IOException {
- boolean isCompressed = FileOutputFormat.getCompressOutput(jc);
- return createSequenceWriter(jc, fs, file, keyClass, valClass, isCompressed, progressable);
- }
-
- /**
* Create a sequencefile output stream based on job configuration Uses user supplied compression
* flag (rather than obtaining it from the Job Configuration).
*
@@ -1055,11 +968,11 @@ public final class Utilities {
throws IOException {
CompressionCodec codec = null;
CompressionType compressionType = CompressionType.NONE;
- Class codecClass = null;
+ Class<? extends CompressionCodec> codecClass = null;
if (isCompressed) {
compressionType = SequenceFileOutputFormat.getOutputCompressionType(jc);
codecClass = FileOutputFormat.getOutputCompressorClass(jc, DefaultCodec.class);
- codec = (CompressionCodec) ReflectionUtil.newInstance(codecClass, jc);
+ codec = ReflectionUtil.newInstance(codecClass, jc);
}
return SequenceFile.createWriter(fs, jc, file, keyClass, valClass, compressionType, codec,
progressable);
@@ -1156,7 +1069,7 @@ public final class Utilities {
/**
* Detect if the supplied file is a temporary path.
*/
- public static boolean isTempPath(FileStatus file) {
+ private static boolean isTempPath(FileStatus file) {
String name = file.getPath().getName();
// in addition to detecting hive temporary files, we also check hadoop
// temporary folders that used to show up in older releases
@@ -1181,46 +1094,6 @@ public final class Utilities {
}
}
- /**
- * Moves files from src to dst if it is within the specified set of paths
- * @param fs
- * @param src
- * @param dst
- * @param filesToMove
- * @throws IOException
- * @throws HiveException
- */
- private static void moveSpecifiedFiles(FileSystem fs, Path src, Path dst, Set<Path> filesToMove)
- throws IOException, HiveException {
- if (!fs.exists(dst)) {
- fs.mkdirs(dst);
- }
-
- FileStatus[] files = fs.listStatus(src);
- for (FileStatus file : files) {
- if (filesToMove.contains(file.getPath())) {
- Utilities.moveFile(fs, file, dst);
- } else if (file.isDir()) {
- // Traverse directory contents.
- // Directory nesting for dst needs to match src.
- Path nestedDstPath = new Path(dst, file.getPath().getName());
- Utilities.moveSpecifiedFiles(fs, file.getPath(), nestedDstPath, filesToMove);
- }
- }
- }
-
- public static void moveSpecifiedFiles(FileSystem fs, Path dst, Set<Path> filesToMove)
- throws IOException, HiveException {
- if (!fs.exists(dst)) {
- fs.mkdirs(dst);
- }
-
- for (Path path: filesToMove) {
- FileStatus fsStatus = fs.getFileStatus(path);
- Utilities.moveFile(fs, fsStatus, dst);
- }
- }
-
private static void moveFile(FileSystem fs, FileStatus file, Path dst) throws IOException,
HiveException {
Path srcFilePath = file.getPath();
@@ -1333,7 +1206,7 @@ public final class Utilities {
* @param filename
* filename to extract taskid from
*/
- public static String getPrefixedTaskIdFromFilename(String filename) {
+ private static String getPrefixedTaskIdFromFilename(String filename) {
return getTaskIdFromFilename(filename, FILE_NAME_PREFIXED_TASK_ID_REGEX);
}
@@ -1341,7 +1214,7 @@ public final class Utilities {
return getIdFromFilename(filename, pattern, 1);
}
- public static int getAttemptIdFromFilename(String filename) {
+ private static int getAttemptIdFromFilename(String filename) {
String attemptStr = getIdFromFilename(filename, FILE_NAME_PREFIXED_TASK_ID_REGEX, 3);
return Integer.parseInt(attemptStr.substring(1));
}
@@ -1364,14 +1237,6 @@ public final class Utilities {
return taskId;
}
- public static String getFileNameFromDirName(String dirName) {
- int dirEnd = dirName.lastIndexOf(Path.SEPARATOR);
- if (dirEnd != -1) {
- return dirName.substring(dirEnd + 1);
- }
- return dirName;
- }
-
/**
* Replace the task id from the filename. It is assumed that the filename is derived from the
* output of getTaskId
@@ -1481,7 +1346,7 @@ public final class Utilities {
}
- public static boolean shouldAvoidRename(FileSinkDesc conf, Configuration hConf) {
+ private static boolean shouldAvoidRename(FileSinkDesc conf, Configuration hConf) {
// we are avoiding rename/move only if following conditions are met
// * execution engine is tez
// * query cache is disabled
@@ -1647,7 +1512,7 @@ public final class Utilities {
return removeTempOrDuplicateFiles(fs, stats, dpCtx, conf, hconf, isBaseDir);
}
- public static List<Path> removeTempOrDuplicateFiles(FileSystem fs, FileStatus[] fileStats,
+ private static List<Path> removeTempOrDuplicateFiles(FileSystem fs, FileStatus[] fileStats,
DynamicPartitionCtx dpCtx, FileSinkDesc conf, Configuration hconf, boolean isBaseDir) throws IOException {
return removeTempOrDuplicateFiles(fs, fileStats, dpCtx, conf, hconf, null, isBaseDir);
}
@@ -1657,7 +1522,7 @@ public final class Utilities {
*
* @return a list of path names corresponding to should-be-created empty buckets.
*/
- public static List<Path> removeTempOrDuplicateFiles(FileSystem fs, FileStatus[] fileStats,
+ private static List<Path> removeTempOrDuplicateFiles(FileSystem fs, FileStatus[] fileStats,
DynamicPartitionCtx dpCtx, FileSinkDesc conf, Configuration hconf, Set<FileStatus> filesKept, boolean isBaseDir)
throws IOException {
int dpLevels = dpCtx == null ? 0 : dpCtx.getNumDPCols(),
@@ -2711,7 +2576,6 @@ public final class Utilities {
}
}
- @SuppressWarnings("unchecked")
private static <T> List<T> getTasks(List<Task<? extends Serializable>> tasks,
TaskFilterFunction<T> function) {
DAGTraversal.traverse(tasks, function);
@@ -3665,7 +3529,6 @@ public final class Utilities {
* @param rWork Used to find FileSinkOperators
* @throws IOException
*/
- @SuppressWarnings("unchecked")
public static void createTmpDirs(Configuration conf, ReduceWork rWork)
throws IOException {
if (rWork == null) {
diff --git a/ql/src/test/org/apache/hadoop/hive/ql/exec/TestUtilities.java b/ql/src/test/org/apache/hadoop/hive/ql/exec/TestUtilities.java
index dc851c0..fbf948c 100644
--- a/ql/src/test/org/apache/hadoop/hive/ql/exec/TestUtilities.java
+++ b/ql/src/test/org/apache/hadoop/hive/ql/exec/TestUtilities.java
@@ -101,10 +101,6 @@ public class TestUtilities {
getFileExtension(jc, false, new HiveIgnoreKeyTextOutputFormat()));
assertEquals("Deflate for uncompressed text format", ".deflate",
getFileExtension(jc, true, new HiveIgnoreKeyTextOutputFormat()));
- assertEquals("No extension for uncompressed default format", "",
- getFileExtension(jc, false));
- assertEquals("Deflate for uncompressed default format", ".deflate",
- getFileExtension(jc, true));
String extension = ".myext";
jc.set("hive.output.file.extension", extension);
@@ -349,7 +345,7 @@ public class TestUtilities {
String testPartitionName = "p=" + i;
testPartitionsPaths[i] = new Path(testTablePath, "p=" + i);
mapWork.getPathToAliases().put(testPartitionsPaths[i], Lists.newArrayList(testPartitionName));
- mapWork.getAliasToWork().put(testPartitionName, (Operator<?>) mock(Operator.class));
+ mapWork.getAliasToWork().put(testPartitionName, mock(Operator.class));
mapWork.getPathToPartitionInfo().put(testPartitionsPaths[i], mockPartitionDesc);
}
@@ -501,7 +497,7 @@ public class TestUtilities {
pathToAliasTable.put(testPartitionsPaths[i], Lists.newArrayList(testPartitionName));
- mapWork.getAliasToWork().put(testPartitionName, (Operator<?>) mock(Operator.class));
+ mapWork.getAliasToWork().put(testPartitionName, mock(Operator.class));
}
mapWork.setPathToAliases(pathToAliasTable);
@@ -638,6 +634,7 @@ public class TestUtilities {
super.addDependentTask(wrappedDep);
}
+ @Override
public boolean addDependentTask(Task<? extends Serializable> dependent) {
return wrappedDep.addDependentTask(dependent);
}