svn commit: r1363543 - in /incubator/hcatalog/branches/branch-0.4: CHANGES.txt src/java/org/apache/hcatalog/mapreduce/FileRecordWriterContainer.java

[kh...@apache.org]

Author: khorgath
Date: Thu Jul 19 21:02:32 2012
New Revision: 1363543

URL: http://svn.apache.org/viewvc?rev=1363543&view=rev
Log:
HCATALOG-448 HCatStorer performance is 4x slower in HCat 0.4 than HCat 0.2 (rajesh.balamohan via khorgath)

Modified:
    incubator/hcatalog/branches/branch-0.4/CHANGES.txt
    incubator/hcatalog/branches/branch-0.4/src/java/org/apache/hcatalog/mapreduce/FileRecordWriterContainer.java

Modified: incubator/hcatalog/branches/branch-0.4/CHANGES.txt
URL: http://svn.apache.org/viewvc/incubator/hcatalog/branches/branch-0.4/CHANGES.txt?rev=1363543&r1=1363542&r2=1363543&view=diff
==============================================================================
--- incubator/hcatalog/branches/branch-0.4/CHANGES.txt (original)
+++ incubator/hcatalog/branches/branch-0.4/CHANGES.txt Thu Jul 19 21:02:32 2012
@@ -49,6 +49,8 @@ Trunk (unreleased changes)
   OPTIMIZATIONS
 
   BUG FIXES
+  HCAT-448 HCatStorer performance is 4x slower in HCat 0.4 than HCat 0.2 (rajesh.balamohan via khorgath)
+
   HCAT-437 HCatSchema's equals() implementation uses hashcode obtained from toString() method. (avandana)
 
   HCAT-438 HCatFieldSchema class does not have equals() method implementation (avandana)

Modified: incubator/hcatalog/branches/branch-0.4/src/java/org/apache/hcatalog/mapreduce/FileRecordWriterContainer.java
URL: http://svn.apache.org/viewvc/incubator/hcatalog/branches/branch-0.4/src/java/org/apache/hcatalog/mapreduce/FileRecordWriterContainer.java?rev=1363543&r1=1363542&r2=1363543&view=diff
==============================================================================
--- incubator/hcatalog/branches/branch-0.4/src/java/org/apache/hcatalog/mapreduce/FileRecordWriterContainer.java (original)
+++ incubator/hcatalog/branches/branch-0.4/src/java/org/apache/hcatalog/mapreduce/FileRecordWriterContainer.java Thu Jul 19 21:02:32 2012
@@ -64,6 +64,7 @@ class FileRecordWriterContainer extends 
     private final Map<String, org.apache.hadoop.mapred.OutputCommitter> baseDynamicCommitters;
     private final Map<String, org.apache.hadoop.mapred.TaskAttemptContext> dynamicContexts;
     private final Map<String, ObjectInspector> dynamicObjectInspectors;
+    private Map<String, OutputJobInfo> dynamicOutputJobInfo;
 
 
     private final List<Integer> partColsToDel;
@@ -112,6 +113,7 @@ class FileRecordWriterContainer extends 
             this.baseDynamicCommitters = null;
             this.dynamicContexts = null;
             this.dynamicObjectInspectors = null;
+            this.dynamicOutputJobInfo = null;
         }
         else {
             this.baseDynamicSerDe = new HashMap<String,SerDe>();
@@ -119,6 +121,7 @@ class FileRecordWriterContainer extends 
             this.baseDynamicCommitters = new HashMap<String, org.apache.hadoop.mapred.OutputCommitter>();
             this.dynamicContexts = new HashMap<String, org.apache.hadoop.mapred.TaskAttemptContext>();
             this.dynamicObjectInspectors = new HashMap<String, ObjectInspector>();
+            this.dynamicOutputJobInfo = new HashMap<String, OutputJobInfo>();
         }
     }
 
@@ -155,12 +158,10 @@ class FileRecordWriterContainer extends 
             InterruptedException {
 
         org.apache.hadoop.mapred.RecordWriter localWriter;
-        org.apache.hadoop.mapred.TaskAttemptContext localContext;
         ObjectInspector localObjectInspector;
         SerDe localSerDe;
         OutputJobInfo localJobInfo = null;
 
-
         if (dynamicPartitioningUsed){
             // calculate which writer to use from the remaining values - this needs to be done before we delete cols
             List<String> dynamicPartValues = new ArrayList<String>();
@@ -225,18 +226,18 @@ class FileRecordWriterContainer extends 
                 baseDynamicCommitters.put(dynKey,baseOutputCommitter);
                 dynamicContexts.put(dynKey,currTaskContext);
                 dynamicObjectInspectors.put(dynKey,InternalUtil.createStructObjectInspector(jobInfo.getOutputSchema()));
+                dynamicOutputJobInfo.put(dynKey, HCatOutputFormat.getJobInfo(dynamicContexts.get(dynKey)));
             }
-            localJobInfo = HCatOutputFormat.getJobInfo(dynamicContexts.get(dynKey));
+            
+            localJobInfo = dynamicOutputJobInfo.get(dynKey);
             localWriter = baseDynamicWriters.get(dynKey);
             localSerDe = baseDynamicSerDe.get(dynKey);
-            localContext = dynamicContexts.get(dynKey);
             localObjectInspector = dynamicObjectInspectors.get(dynKey);
         }
         else{
-            localJobInfo = HCatBaseOutputFormat.getJobInfo(context);
+            localJobInfo = jobInfo;
             localWriter = getBaseRecordWriter();
             localSerDe = serDe;
-            localContext = HCatMapRedUtil.createTaskAttemptContext(context);
             localObjectInspector = objectInspector;
         }