You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by gu...@apache.org on 2013/10/16 00:43:00 UTC
svn commit: r1532579 - in
/hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql:
io/orc/OrcInputFormat.java io/orc/ReaderImpl.java log/PerfLogger.java
Author: gunther
Date: Tue Oct 15 22:42:59 2013
New Revision: 1532579
URL: http://svn.apache.org/r1532579
Log:
HIVE-5551: Create perf logger statements for orc init/split creation (Gunther Hagleitner)
Modified:
hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/io/orc/OrcInputFormat.java
hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/io/orc/ReaderImpl.java
hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/log/PerfLogger.java
Modified: hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/io/orc/OrcInputFormat.java
URL: http://svn.apache.org/viewvc/hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/io/orc/OrcInputFormat.java?rev=1532579&r1=1532578&r2=1532579&view=diff
==============================================================================
--- hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/io/orc/OrcInputFormat.java (original)
+++ hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/io/orc/OrcInputFormat.java Tue Oct 15 22:42:59 2013
@@ -42,6 +42,7 @@ import org.apache.hadoop.hive.ql.exec.ve
import org.apache.hadoop.hive.ql.exec.vector.VectorizedRowBatch;
import org.apache.hadoop.hive.ql.io.InputFormatChecker;
import org.apache.hadoop.hive.ql.io.sarg.SearchArgument;
+import org.apache.hadoop.hive.ql.log.PerfLogger;
import org.apache.hadoop.hive.ql.plan.TableScanDesc;
import org.apache.hadoop.hive.serde2.ColumnProjectionUtils;
import org.apache.hadoop.hive.shims.HadoopShims;
@@ -70,6 +71,9 @@ public class OrcInputFormat implements
private static final long DEFAULT_MIN_SPLIT_SIZE = 16 * 1024 * 1024;
private static final long DEFAULT_MAX_SPLIT_SIZE = 256 * 1024 * 1024;
+ private static final PerfLogger perfLogger = PerfLogger.getPerfLogger();
+ private static final String CLASS_NAME = ReaderImpl.class.getName();
+
/**
* When picking the hosts for a split that crosses block boundaries,
* any drop any host that has fewer than MIN_INCLUDED_LOCATION of the
@@ -531,6 +535,7 @@ public class OrcInputFormat implements
*/
@Override
public void run() {
+ perfLogger.PerfLogBegin(CLASS_NAME, PerfLogger.CREATE_ORC_SPLITS);
try {
Reader orcReader = OrcFile.createReader(fs, file.getPath());
long currentOffset = -1;
@@ -563,6 +568,7 @@ public class OrcInputFormat implements
context.errors.add(th);
}
}
+ perfLogger.PerfLogEnd(CLASS_NAME, PerfLogger.CREATE_ORC_SPLITS);
}
}
Modified: hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/io/orc/ReaderImpl.java
URL: http://svn.apache.org/viewvc/hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/io/orc/ReaderImpl.java?rev=1532579&r1=1532578&r2=1532579&view=diff
==============================================================================
--- hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/io/orc/ReaderImpl.java (original)
+++ hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/io/orc/ReaderImpl.java Tue Oct 15 22:42:59 2013
@@ -34,6 +34,7 @@ import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hive.ql.io.orc.OrcProto.Type;
import org.apache.hadoop.hive.ql.io.sarg.SearchArgument;
+import org.apache.hadoop.hive.ql.log.PerfLogger;
import org.apache.hadoop.hive.ql.util.JavaDataModel;
import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector;
import org.apache.hadoop.io.Text;
@@ -57,6 +58,9 @@ final class ReaderImpl implements Reader
private final ObjectInspector inspector;
private long deserializedSize = -1;
+ private static final PerfLogger perfLogger = PerfLogger.getPerfLogger();
+ private static final String CLASS_NAME = ReaderImpl.class.getName();
+
private static class StripeInformationImpl
implements StripeInformation {
private final OrcProto.StripeInformation stripe;
@@ -273,6 +277,7 @@ final class ReaderImpl implements Reader
}
ReaderImpl(FileSystem fs, Path path) throws IOException {
+ perfLogger.PerfLogBegin(CLASS_NAME, PerfLogger.INIT_ORC_RECORD_READER);
this.fileSystem = fs;
this.path = path;
FSDataInputStream file = fs.open(path);
@@ -328,6 +333,7 @@ final class ReaderImpl implements Reader
footer = OrcProto.Footer.parseFrom(instream);
inspector = OrcStruct.createObjectInspector(0, footer.getTypesList());
file.close();
+ perfLogger.PerfLogEnd(CLASS_NAME, PerfLogger.INIT_ORC_RECORD_READER);
}
@Override
Modified: hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/log/PerfLogger.java
URL: http://svn.apache.org/viewvc/hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/log/PerfLogger.java?rev=1532579&r1=1532578&r2=1532579&view=diff
==============================================================================
--- hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/log/PerfLogger.java (original)
+++ hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/log/PerfLogger.java Tue Oct 15 22:42:59 2013
@@ -63,7 +63,8 @@ public class PerfLogger {
public static final String TEZ_RUN_PROCESSOR = "TezRunProcessor";
public static final String TEZ_INIT_OPERATORS = "TezInitializeOperators";
public static final String LOAD_HASHTABLE = "LoadHashtable";
-
+ public static final String INIT_ORC_RECORD_READER = "OrcRecordReaderInit";
+ public static final String CREATE_ORC_SPLITS = "OrcCreateSplits";
protected static final ThreadLocal<PerfLogger> perfLogger = new ThreadLocal<PerfLogger>();