You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by xu...@apache.org on 2015/07/16 06:40:50 UTC
[03/50] [abbrv] hive git commit: HIVE-10795: Remove use of PerfLogger
from Orc (Owen O'Malley reviewed by Prasanth Jayachandran)
HIVE-10795: Remove use of PerfLogger from Orc (Owen O'Malley reviewed by Prasanth Jayachandran)
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/527497cc
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/527497cc
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/527497cc
Branch: refs/heads/beeline-cli
Commit: 527497cc85144013ca0bb44fb3aa1fba9de8e052
Parents: 1280cca
Author: Prasanth Jayachandran <j....@gmail.com>
Authored: Tue Jul 7 23:08:07 2015 -0700
Committer: Prasanth Jayachandran <j....@gmail.com>
Committed: Tue Jul 7 23:08:07 2015 -0700
----------------------------------------------------------------------
.../hadoop/hive/ql/io/orc/OrcInputFormat.java | 20 ++++++++++----------
.../hive/ql/io/orc/OrcNewInputFormat.java | 16 ++++++++++------
.../apache/hadoop/hive/ql/log/PerfLogger.java | 1 -
3 files changed, 20 insertions(+), 17 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/527497cc/ql/src/java/org/apache/hadoop/hive/ql/io/orc/OrcInputFormat.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/io/orc/OrcInputFormat.java b/ql/src/java/org/apache/hadoop/hive/ql/io/orc/OrcInputFormat.java
index 7346bc4..8864013 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/io/orc/OrcInputFormat.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/io/orc/OrcInputFormat.java
@@ -58,7 +58,6 @@ import org.apache.hadoop.hive.ql.io.sarg.PredicateLeaf;
import org.apache.hadoop.hive.ql.io.sarg.SearchArgument;
import org.apache.hadoop.hive.ql.io.sarg.SearchArgument.TruthValue;
import org.apache.hadoop.hive.ql.io.sarg.SearchArgumentFactory;
-import org.apache.hadoop.hive.ql.log.PerfLogger;
import org.apache.hadoop.hive.serde2.ColumnProjectionUtils;
import org.apache.hadoop.hive.serde2.SerDeStats;
import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector;
@@ -122,9 +121,6 @@ public class OrcInputFormat implements InputFormat<NullWritable, OrcStruct>,
private static final long DEFAULT_MIN_SPLIT_SIZE = 16 * 1024 * 1024;
private static final long DEFAULT_MAX_SPLIT_SIZE = 256 * 1024 * 1024;
- private static final PerfLogger perfLogger = PerfLogger.getPerfLogger();
- private static final String CLASS_NAME = ReaderImpl.class.getName();
-
/**
* When picking the hosts for a split that crosses block boundaries,
* any drop any host that has fewer than MIN_INCLUDED_LOCATION of the
@@ -490,7 +486,7 @@ public class OrcInputFormat implements InputFormat<NullWritable, OrcStruct>,
context.numFilesCounter.incrementAndGet();
FileInfo fileInfo = Context.footerCache.getIfPresent(file.getPath());
if (fileInfo != null) {
- if (LOG.isDebugEnabled()) {
+ if (isDebugEnabled) {
LOG.debug("Info cached for path: " + file.getPath());
}
if (fileInfo.modificationTime == file.getModificationTime() &&
@@ -501,7 +497,7 @@ public class OrcInputFormat implements InputFormat<NullWritable, OrcStruct>,
} else {
// Invalidate
Context.footerCache.invalidate(file.getPath());
- if (LOG.isDebugEnabled()) {
+ if (isDebugEnabled) {
LOG.debug("Meta-Info for : " + file.getPath() +
" changed. CachedModificationTime: "
+ fileInfo.modificationTime + ", CurrentModificationTime: "
@@ -511,7 +507,7 @@ public class OrcInputFormat implements InputFormat<NullWritable, OrcStruct>,
}
}
} else {
- if (LOG.isDebugEnabled()) {
+ if (isDebugEnabled) {
LOG.debug("Info not cached for path: " + file.getPath());
}
}
@@ -871,7 +867,7 @@ public class OrcInputFormat implements InputFormat<NullWritable, OrcStruct>,
includeStripe[i] = (i >= stripeStats.size()) ||
isStripeSatisfyPredicate(stripeStats.get(i), sarg,
filterColumns);
- if (LOG.isDebugEnabled() && !includeStripe[i]) {
+ if (isDebugEnabled && !includeStripe[i]) {
LOG.debug("Eliminating ORC stripe-" + i + " of file '" +
file.getPath() + "' as it did not satisfy " +
"predicate condition.");
@@ -1060,9 +1056,13 @@ public class OrcInputFormat implements InputFormat<NullWritable, OrcStruct>,
@Override
public InputSplit[] getSplits(JobConf job,
int numSplits) throws IOException {
- perfLogger.PerfLogBegin(CLASS_NAME, PerfLogger.ORC_GET_SPLITS);
+ if (isDebugEnabled) {
+ LOG.debug("getSplits started");
+ }
List<OrcSplit> result = generateSplitsInfo(job, numSplits);
- perfLogger.PerfLogEnd(CLASS_NAME, PerfLogger.ORC_GET_SPLITS);
+ if (isDebugEnabled) {
+ LOG.debug("getSplits finished");
+ }
return result.toArray(new InputSplit[result.size()]);
}
http://git-wip-us.apache.org/repos/asf/hive/blob/527497cc/ql/src/java/org/apache/hadoop/hive/ql/io/orc/OrcNewInputFormat.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/io/orc/OrcNewInputFormat.java b/ql/src/java/org/apache/hadoop/hive/ql/io/orc/OrcNewInputFormat.java
index b6ad0dc..1833d3d 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/io/orc/OrcNewInputFormat.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/io/orc/OrcNewInputFormat.java
@@ -21,10 +21,10 @@ import java.io.IOException;
import java.util.ArrayList;
import java.util.List;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.hive.ql.log.PerfLogger;
import org.apache.hadoop.hive.shims.ShimLoader;
import org.apache.hadoop.io.NullWritable;
import org.apache.hadoop.mapreduce.InputFormat;
@@ -37,8 +37,8 @@ import org.apache.hadoop.mapreduce.lib.input.FileSplit;
/** An InputFormat for ORC files. Keys are meaningless,
* value is the OrcStruct object */
public class OrcNewInputFormat extends InputFormat<NullWritable, OrcStruct>{
- private static final PerfLogger perfLogger = PerfLogger.getPerfLogger();
- private static final String CLASS_NAME = ReaderImpl.class.getName();
+
+ private static final Log LOG = LogFactory.getLog(OrcNewInputFormat.class);
@Override
public RecordReader<NullWritable, OrcStruct> createRecordReader(
@@ -117,7 +117,9 @@ public class OrcNewInputFormat extends InputFormat<NullWritable, OrcStruct>{
@Override
public List<InputSplit> getSplits(JobContext jobContext)
throws IOException, InterruptedException {
- perfLogger.PerfLogBegin(CLASS_NAME, PerfLogger.ORC_GET_SPLITS);
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("getSplits started");
+ }
List<OrcSplit> splits =
OrcInputFormat.generateSplitsInfo(ShimLoader.getHadoopShims()
.getConfiguration(jobContext));
@@ -125,7 +127,9 @@ public class OrcNewInputFormat extends InputFormat<NullWritable, OrcStruct>{
for(OrcSplit split: splits) {
result.add(new OrcNewSplit(split));
}
- perfLogger.PerfLogEnd(CLASS_NAME, PerfLogger.ORC_GET_SPLITS);
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("getSplits finished");
+ }
return result;
}
http://git-wip-us.apache.org/repos/asf/hive/blob/527497cc/ql/src/java/org/apache/hadoop/hive/ql/log/PerfLogger.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/log/PerfLogger.java b/ql/src/java/org/apache/hadoop/hive/ql/log/PerfLogger.java
index f202991..20ca195 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/log/PerfLogger.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/log/PerfLogger.java
@@ -63,7 +63,6 @@ public class PerfLogger {
public static final String TEZ_RUN_PROCESSOR = "TezRunProcessor";
public static final String TEZ_INIT_OPERATORS = "TezInitializeOperators";
public static final String LOAD_HASHTABLE = "LoadHashtable";
- public static final String ORC_GET_SPLITS = "OrcGetSplits";
public static final String SPARK_SUBMIT_TO_RUNNING = "SparkSubmitToRunning";
public static final String SPARK_BUILD_PLAN = "SparkBuildPlan";