You are viewing a plain text version of this content. The canonical link for it is here.
Posted to hcatalog-commits@incubator.apache.org by ga...@apache.org on 2012/03/07 18:58:07 UTC
svn commit: r1298040 - in /incubator/hcatalog/branches/branch-0.4: ./
src/java/org/apache/hcatalog/common/ src/java/org/apache/hcatalog/mapreduce/
Author: gates
Date: Wed Mar 7 18:58:06 2012
New Revision: 1298040
URL: http://svn.apache.org/viewvc?rev=1298040&view=rev
Log:
HCATALOG-282 HCatInputFormat shouldn't expect storageHandler to be serializable
Modified:
incubator/hcatalog/branches/branch-0.4/CHANGES.txt
incubator/hcatalog/branches/branch-0.4/src/java/org/apache/hcatalog/common/HCatUtil.java
incubator/hcatalog/branches/branch-0.4/src/java/org/apache/hcatalog/mapreduce/FosterStorageHandler.java
incubator/hcatalog/branches/branch-0.4/src/java/org/apache/hcatalog/mapreduce/HCatBaseInputFormat.java
incubator/hcatalog/branches/branch-0.4/src/java/org/apache/hcatalog/mapreduce/PartInfo.java
Modified: incubator/hcatalog/branches/branch-0.4/CHANGES.txt
URL: http://svn.apache.org/viewvc/incubator/hcatalog/branches/branch-0.4/CHANGES.txt?rev=1298040&r1=1298039&r2=1298040&view=diff
==============================================================================
--- incubator/hcatalog/branches/branch-0.4/CHANGES.txt (original)
+++ incubator/hcatalog/branches/branch-0.4/CHANGES.txt Wed Mar 7 18:58:06 2012
@@ -64,6 +64,8 @@ Release 0.4.0 - Unreleased
OPTIMIZATIONS
BUG FIXES
+ HCAT-282 HCatInputFormat shouldn't expect storageHandler to be serializable (khorgath via gates)
+
HCAT-281 HCat use table schema instead of partition schema to read partition data (daijy and khorgath via gates)
HCAT 286 NPE in HdfsAuthorizationProvider (enis via gates)
Modified: incubator/hcatalog/branches/branch-0.4/src/java/org/apache/hcatalog/common/HCatUtil.java
URL: http://svn.apache.org/viewvc/incubator/hcatalog/branches/branch-0.4/src/java/org/apache/hcatalog/common/HCatUtil.java?rev=1298040&r1=1298039&r2=1298040&view=diff
==============================================================================
--- incubator/hcatalog/branches/branch-0.4/src/java/org/apache/hcatalog/common/HCatUtil.java (original)
+++ incubator/hcatalog/branches/branch-0.4/src/java/org/apache/hcatalog/common/HCatUtil.java Wed Mar 7 18:58:06 2012
@@ -74,6 +74,7 @@ import org.apache.hcatalog.mapreduce.HCa
import org.apache.hcatalog.mapreduce.HCatStorageHandler;
import org.apache.hcatalog.mapreduce.InputJobInfo;
import org.apache.hcatalog.mapreduce.OutputJobInfo;
+import org.apache.hcatalog.mapreduce.PartInfo;
import org.apache.hcatalog.mapreduce.StorerInfo;
import org.apache.thrift.TException;
@@ -477,6 +478,15 @@ public class HCatUtil {
storerInfo.getIfClass(),
storerInfo.getOfClass());
}
+
+ public static HCatStorageHandler getStorageHandler(Configuration conf, PartInfo partitionInfo) throws IOException {
+ return HCatUtil.getStorageHandler(
+ conf,
+ partitionInfo.getStorageHandlerClassName(),
+ partitionInfo.getSerdeClassName(),
+ partitionInfo.getInputFormatClassName(),
+ partitionInfo.getOutputFormatClassName());
+ }
/**
* Create an instance of a storage handler. If storageHandler == null,
@@ -497,7 +507,7 @@ public class HCatUtil {
String outputFormat)
throws IOException {
- if (storageHandler == null) {
+ if ((storageHandler == null) || (storageHandler.equals(FosterStorageHandler.class.getName()))){
try {
return new FosterStorageHandler(inputFormat,
outputFormat,
Modified: incubator/hcatalog/branches/branch-0.4/src/java/org/apache/hcatalog/mapreduce/FosterStorageHandler.java
URL: http://svn.apache.org/viewvc/incubator/hcatalog/branches/branch-0.4/src/java/org/apache/hcatalog/mapreduce/FosterStorageHandler.java?rev=1298040&r1=1298039&r2=1298040&view=diff
==============================================================================
--- incubator/hcatalog/branches/branch-0.4/src/java/org/apache/hcatalog/mapreduce/FosterStorageHandler.java (original)
+++ incubator/hcatalog/branches/branch-0.4/src/java/org/apache/hcatalog/mapreduce/FosterStorageHandler.java Wed Mar 7 18:58:06 2012
@@ -30,11 +30,9 @@ import java.util.Map;
* artifacts of tables which don't define a SerDe. This StorageHandler assumes
* the supplied storage artifacts are for a file-based storage system.
*/
-public class FosterStorageHandler extends HCatStorageHandler
- implements Serializable {
+public class FosterStorageHandler extends HCatStorageHandler {
- public Configuration conf
- ;
+ public Configuration conf;
/** The directory under which data is initially written for a partitioned table */
protected static final String DYNTEMP_DIR_NAME = "_DYN";
Modified: incubator/hcatalog/branches/branch-0.4/src/java/org/apache/hcatalog/mapreduce/HCatBaseInputFormat.java
URL: http://svn.apache.org/viewvc/incubator/hcatalog/branches/branch-0.4/src/java/org/apache/hcatalog/mapreduce/HCatBaseInputFormat.java?rev=1298040&r1=1298039&r2=1298040&view=diff
==============================================================================
--- incubator/hcatalog/branches/branch-0.4/src/java/org/apache/hcatalog/mapreduce/HCatBaseInputFormat.java (original)
+++ incubator/hcatalog/branches/branch-0.4/src/java/org/apache/hcatalog/mapreduce/HCatBaseInputFormat.java Wed Mar 7 18:58:06 2012
@@ -151,7 +151,8 @@ public abstract class HCatBaseInputForma
HCatUtil.copyJobPropertiesToJobConf(jobProperties, jobConf);
- storageHandler = partitionInfo.getStorageHandler();
+ storageHandler = HCatUtil.getStorageHandler(
+ jobConf, partitionInfo);
//Get the input format
Class inputFormatClass = storageHandler.getInputFormatClass();
@@ -194,7 +195,9 @@ public abstract class HCatBaseInputForma
PartInfo partitionInfo = hcatSplit.getPartitionInfo();
JobContext jobContext = taskContext;
- HCatStorageHandler storageHandler = partitionInfo.getStorageHandler();
+ HCatStorageHandler storageHandler = HCatUtil.getStorageHandler(
+ jobContext.getConfiguration(), partitionInfo);
+
JobConf jobConf = HCatUtil.getJobConfFromContext(jobContext);
Class inputFormatClass = storageHandler.getInputFormatClass();
Modified: incubator/hcatalog/branches/branch-0.4/src/java/org/apache/hcatalog/mapreduce/PartInfo.java
URL: http://svn.apache.org/viewvc/incubator/hcatalog/branches/branch-0.4/src/java/org/apache/hcatalog/mapreduce/PartInfo.java?rev=1298040&r1=1298039&r2=1298040&view=diff
==============================================================================
--- incubator/hcatalog/branches/branch-0.4/src/java/org/apache/hcatalog/mapreduce/PartInfo.java (original)
+++ incubator/hcatalog/branches/branch-0.4/src/java/org/apache/hcatalog/mapreduce/PartInfo.java Wed Mar 7 18:58:06 2012
@@ -37,7 +37,10 @@ public class PartInfo implements Seriali
private final HCatSchema partitionSchema;
/** The information about which input storage handler to use */
- private final HCatStorageHandler storageHandler;
+ private final String storageHandlerClassName;
+ private final String inputFormatClassName;
+ private final String outputFormatClassName;
+ private final String serdeClassName;
/** HCat-specific properties set at the partition */
private final Properties hcatProperties;
@@ -65,12 +68,16 @@ public class PartInfo implements Seriali
String location, Properties hcatProperties,
Map<String,String> jobProperties, HCatTableInfo tableInfo){
this.partitionSchema = partitionSchema;
- this.storageHandler = storageHandler;
this.location = location;
this.hcatProperties = hcatProperties;
this.jobProperties = jobProperties;
this.tableInfo = tableInfo;
- }
+
+ this.storageHandlerClassName = storageHandler.getClass().getName();
+ this.inputFormatClassName = storageHandler.getInputFormatClass().getName();
+ this.serdeClassName = storageHandler.getSerDeClass().getName();
+ this.outputFormatClassName = storageHandler.getOutputFormatClass().getName();
+}
/**
* Gets the value of partitionSchema.
@@ -80,15 +87,33 @@ public class PartInfo implements Seriali
return partitionSchema;
}
+ /**
+ * @return the storage handler class name
+ */
+ public String getStorageHandlerClassName() {
+ return storageHandlerClassName;
+ }
/**
- * Gets the value of input storage driver class name.
- * @return the input storage driver class name
+ * @return the inputFormatClassName
*/
- public HCatStorageHandler getStorageHandler() {
- return storageHandler;
+ public String getInputFormatClassName() {
+ return inputFormatClassName;
}
+ /**
+ * @return the outputFormatClassName
+ */
+ public String getOutputFormatClassName() {
+ return outputFormatClassName;
+ }
+
+ /**
+ * @return the serdeClassName
+ */
+ public String getSerdeClassName() {
+ return serdeClassName;
+ }
/**
* Gets the value of hcatProperties.