You are viewing a plain text version of this content. The canonical link for it is here.
Posted to hcatalog-commits@incubator.apache.org by av...@apache.org on 2012/05/15 23:26:17 UTC
svn commit: r1338949 - in /incubator/hcatalog/branches/branch-0.4:
CHANGES.txt src/java/org/apache/hcatalog/mapreduce/MultiOutputFormat.java
Author: avandana
Date: Tue May 15 23:26:16 2012
New Revision: 1338949
URL: http://svn.apache.org/viewvc?rev=1338949&view=rev
Log:
HCAT-409:MultiOutputformat should make getJobContext public (merged from trunk)
Modified:
incubator/hcatalog/branches/branch-0.4/CHANGES.txt
incubator/hcatalog/branches/branch-0.4/src/java/org/apache/hcatalog/mapreduce/MultiOutputFormat.java
Modified: incubator/hcatalog/branches/branch-0.4/CHANGES.txt
URL: http://svn.apache.org/viewvc/incubator/hcatalog/branches/branch-0.4/CHANGES.txt?rev=1338949&r1=1338948&r2=1338949&view=diff
==============================================================================
--- incubator/hcatalog/branches/branch-0.4/CHANGES.txt (original)
+++ incubator/hcatalog/branches/branch-0.4/CHANGES.txt Tue May 15 23:26:16 2012
@@ -28,6 +28,8 @@ Release 0.4.1 - Unreleased
HCAT-368 Documentation improvements: doc set & API docs (lefty via gates)
BUG FIXES
+ HCAT-409 MultiOutputformat should make getJobContext public
+
HCAT-403 MultiOutputFormat will call taskCommit on all formats if only one ever needs commit (rohini via avandana)
HCAT-400 HCatalog build fails from git checkout due to missing lib directory (traviscrawford via gates)
Modified: incubator/hcatalog/branches/branch-0.4/src/java/org/apache/hcatalog/mapreduce/MultiOutputFormat.java
URL: http://svn.apache.org/viewvc/incubator/hcatalog/branches/branch-0.4/src/java/org/apache/hcatalog/mapreduce/MultiOutputFormat.java?rev=1338949&r1=1338948&r2=1338949&view=diff
==============================================================================
--- incubator/hcatalog/branches/branch-0.4/src/java/org/apache/hcatalog/mapreduce/MultiOutputFormat.java (original)
+++ incubator/hcatalog/branches/branch-0.4/src/java/org/apache/hcatalog/mapreduce/MultiOutputFormat.java Tue May 15 23:26:16 2012
@@ -156,6 +156,35 @@ public class MultiOutputFormat extends O
}
/**
+ * Get the JobContext with the related OutputFormat configuration populated given the alias
+ * and the actual JobContext
+ * @param alias the name given to the OutputFormat configuration
+ * @param context the JobContext
+ * @return a copy of the JobContext with the alias configuration populated
+ */
+ public static JobContext getJobContext(String alias, JobContext context) {
+ String aliasConf = context.getConfiguration().get(getAliasConfName(alias));
+ JobContext aliasContext = new JobContext(context.getConfiguration(), context.getJobID());
+ addToConfig(aliasConf, aliasContext.getConfiguration());
+ return aliasContext;
+ }
+
+ /**
+ * Get the TaskAttemptContext with the related OutputFormat configuration populated given the alias
+ * and the actual TaskAttemptContext
+ * @param alias the name given to the OutputFormat configuration
+ * @param context the Mapper or Reducer Context
+ * @return a copy of the TaskAttemptContext with the alias configuration populated
+ */
+ public static TaskAttemptContext getTaskAttemptContext(String alias, TaskAttemptContext context) {
+ String aliasConf = context.getConfiguration().get(getAliasConfName(alias));
+ TaskAttemptContext aliasContext = new TaskAttemptContext(context.getConfiguration(),
+ context.getTaskAttemptID());
+ addToConfig(aliasConf, aliasContext.getConfiguration());
+ return aliasContext;
+ }
+
+ /**
* Write the output key and value using the OutputFormat defined by the
* alias.
*
@@ -265,21 +294,6 @@ public class MultiOutputFormat extends O
return builder.substring(0, builder.length() - COMMA_DELIM.length());
}
- private static JobContext getJobContext(String alias, JobContext context) {
- String aliasConf = context.getConfiguration().get(getAliasConfName(alias));
- JobContext aliasContext = new JobContext(context.getConfiguration(), context.getJobID());
- addToConfig(aliasConf, aliasContext.getConfiguration());
- return aliasContext;
- }
-
- private static TaskAttemptContext getTaskContext(String alias, TaskAttemptContext context) {
- String aliasConf = context.getConfiguration().get(getAliasConfName(alias));
- TaskAttemptContext aliasContext = new TaskAttemptContext(context.getConfiguration(),
- context.getTaskAttemptID());
- addToConfig(aliasConf, aliasContext.getConfiguration());
- return aliasContext;
- }
-
private static String getAliasConfName(String alias) {
return MO_ALIAS + "." + alias + ".conf";
}
@@ -408,7 +422,7 @@ public class MultiOutputFormat extends O
baseRecordWriters = new LinkedHashMap<String, BaseRecordWriterContainer>();
String[] aliases = getOutputFormatAliases(context);
for (String alias : aliases) {
- TaskAttemptContext aliasContext = getTaskContext(alias, context);
+ TaskAttemptContext aliasContext = getTaskAttemptContext(alias, context);
Configuration aliasConf = aliasContext.getConfiguration();
// Create output directory if not already created.
String outDir = aliasConf.get("mapred.output.dir");
@@ -476,7 +490,7 @@ public class MultiOutputFormat extends O
outputCommitters = new LinkedHashMap<String, MultiOutputFormat.BaseOutputCommitterContainer>();
String[] aliases = getOutputFormatAliases(context);
for (String alias : aliases) {
- TaskAttemptContext aliasContext = getTaskContext(alias, context);
+ TaskAttemptContext aliasContext = getTaskAttemptContext(alias, context);
OutputCommitter baseCommitter = getOutputFormatInstance(aliasContext)
.getOutputCommitter(aliasContext);
outputCommitters.put(alias,