You are viewing a plain text version of this content. The canonical link for it is here.
Posted to hcatalog-commits@incubator.apache.org by kh...@apache.org on 2012/07/19 20:22:00 UTC
svn commit: r1363523 - in /incubator/hcatalog/trunk: CHANGES.txt
src/java/org/apache/hcatalog/mapreduce/FileRecordWriterContainer.java
Author: khorgath
Date: Thu Jul 19 20:21:59 2012
New Revision: 1363523
URL: http://svn.apache.org/viewvc?rev=1363523&view=rev
Log:
HCATALOG-448 HCatStorer performance is 4x slower in HCat 0.4 than HCat 0.2 (rajesh.balamohan via khorgath)
Modified:
incubator/hcatalog/trunk/CHANGES.txt
incubator/hcatalog/trunk/src/java/org/apache/hcatalog/mapreduce/FileRecordWriterContainer.java
Modified: incubator/hcatalog/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/incubator/hcatalog/trunk/CHANGES.txt?rev=1363523&r1=1363522&r2=1363523&view=diff
==============================================================================
--- incubator/hcatalog/trunk/CHANGES.txt (original)
+++ incubator/hcatalog/trunk/CHANGES.txt Thu Jul 19 20:21:59 2012
@@ -61,6 +61,8 @@ Trunk (unreleased changes)
OPTIMIZATIONS
BUG FIXES
+ HCAT-448 HCatStorer performance is 4x slower in HCat 0.4 than HCat 0.2 (rajesh.balamohan via khorgath)
+
HCAT-447 findbugs target doesn't work in trunk (traviscrawford)
HCAT-437 HCatSchema's equals() implementation uses hashcode obtained from toString() method. (avandana)
Modified: incubator/hcatalog/trunk/src/java/org/apache/hcatalog/mapreduce/FileRecordWriterContainer.java
URL: http://svn.apache.org/viewvc/incubator/hcatalog/trunk/src/java/org/apache/hcatalog/mapreduce/FileRecordWriterContainer.java?rev=1363523&r1=1363522&r2=1363523&view=diff
==============================================================================
--- incubator/hcatalog/trunk/src/java/org/apache/hcatalog/mapreduce/FileRecordWriterContainer.java (original)
+++ incubator/hcatalog/trunk/src/java/org/apache/hcatalog/mapreduce/FileRecordWriterContainer.java Thu Jul 19 20:21:59 2012
@@ -62,6 +62,7 @@ class FileRecordWriterContainer extends
private final Map<String, org.apache.hadoop.mapred.OutputCommitter> baseDynamicCommitters;
private final Map<String, org.apache.hadoop.mapred.TaskAttemptContext> dynamicContexts;
private final Map<String, ObjectInspector> dynamicObjectInspectors;
+ private Map<String, OutputJobInfo> dynamicOutputJobInfo;
private final List<Integer> partColsToDel;
@@ -110,6 +111,7 @@ class FileRecordWriterContainer extends
this.baseDynamicCommitters = null;
this.dynamicContexts = null;
this.dynamicObjectInspectors = null;
+ this.dynamicOutputJobInfo = null;
}
else {
this.baseDynamicSerDe = new HashMap<String,SerDe>();
@@ -117,11 +119,12 @@ class FileRecordWriterContainer extends
this.baseDynamicCommitters = new HashMap<String, org.apache.hadoop.mapred.OutputCommitter>();
this.dynamicContexts = new HashMap<String, org.apache.hadoop.mapred.TaskAttemptContext>();
this.dynamicObjectInspectors = new HashMap<String, ObjectInspector>();
+ this.dynamicOutputJobInfo = new HashMap<String, OutputJobInfo>();
}
}
/**
- * @return the storagehandler
+ * @return the storageDriver
*/
public HCatStorageHandler getStorageHandler() {
return storageHandler;
@@ -158,7 +161,6 @@ class FileRecordWriterContainer extends
SerDe localSerDe;
OutputJobInfo localJobInfo = null;
-
if (dynamicPartitioningUsed){
// calculate which writer to use from the remaining values - this needs to be done before we delete cols
List<String> dynamicPartValues = new ArrayList<String>();
@@ -223,18 +225,19 @@ class FileRecordWriterContainer extends
baseDynamicCommitters.put(dynKey,baseOutputCommitter);
dynamicContexts.put(dynKey,currTaskContext);
dynamicObjectInspectors.put(dynKey,InternalUtil.createStructObjectInspector(jobInfo.getOutputSchema()));
+ dynamicOutputJobInfo.put(dynKey, HCatOutputFormat.getJobInfo(dynamicContexts.get(dynKey)));
}
- localJobInfo = HCatOutputFormat.getJobInfo(dynamicContexts.get(dynKey));
+
+ localJobInfo = dynamicOutputJobInfo.get(dynKey);
localWriter = baseDynamicWriters.get(dynKey);
localSerDe = baseDynamicSerDe.get(dynKey);
localContext = dynamicContexts.get(dynKey);
localObjectInspector = dynamicObjectInspectors.get(dynKey);
}
else{
- localJobInfo = HCatBaseOutputFormat.getJobInfo(context);
+ localJobInfo = jobInfo;
localWriter = getBaseRecordWriter();
localSerDe = serDe;
- localContext = HCatMapRedUtil.createTaskAttemptContext(context);
localObjectInspector = objectInspector;
}