You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@pig.apache.org by dv...@apache.org on 2011/09/05 02:15:17 UTC
svn commit: r1165138 - in /pig/trunk: CHANGES.txt
src/org/apache/pig/backend/hadoop/hbase/HBaseStorage.java
Author: dvryaboy
Date: Mon Sep 5 00:15:16 2011
New Revision: 1165138
URL: http://svn.apache.org/viewvc?rev=1165138&view=rev
Log:
PIG-2115: Fix Pig HBaseStorage configuration and setup issues
Modified:
pig/trunk/CHANGES.txt
pig/trunk/src/org/apache/pig/backend/hadoop/hbase/HBaseStorage.java
Modified: pig/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/pig/trunk/CHANGES.txt?rev=1165138&r1=1165137&r2=1165138&view=diff
==============================================================================
--- pig/trunk/CHANGES.txt (original)
+++ pig/trunk/CHANGES.txt Mon Sep 5 00:15:16 2011
@@ -132,6 +132,8 @@ PIG-2011: Speed up TestTypedMap.java (dv
BUG FIXES
+PIG-2115: Fix Pig HBaseStorage configuration and setup issues (gbowyer@fastmail.co.uk via dvryaboy)
+
PIG-2193: Using HBaseStorage to scan 2 tables in the same Map job produces bad data (rangadi via dvryaboy)
PIG-2232: "declare" document contains a typo (daijy)
Modified: pig/trunk/src/org/apache/pig/backend/hadoop/hbase/HBaseStorage.java
URL: http://svn.apache.org/viewvc/pig/trunk/src/org/apache/pig/backend/hadoop/hbase/HBaseStorage.java?rev=1165138&r1=1165137&r2=1165138&view=diff
==============================================================================
--- pig/trunk/src/org/apache/pig/backend/hadoop/hbase/HBaseStorage.java (original)
+++ pig/trunk/src/org/apache/pig/backend/hadoop/hbase/HBaseStorage.java Mon Sep 5 00:15:16 2011
@@ -77,6 +77,7 @@ import org.apache.pig.ResourceSchema.Res
import org.apache.pig.StoreFuncInterface;
import org.apache.pig.backend.hadoop.executionengine.mapReduceLayer.PigSplit;
import org.apache.pig.backend.hadoop.hbase.HBaseTableInputFormat.HBaseTableIFBuilder;
+import org.apache.pig.backend.hadoop.datastorage.ConfigurationUtil;
import org.apache.pig.builtin.Utf8StorageConverter;
import org.apache.pig.data.DataBag;
import org.apache.pig.data.DataByteArray;
@@ -530,14 +531,7 @@ public class HBaseStorage extends LoadFu
@Override
public void setLocation(String location, Job job) throws IOException {
job.getConfiguration().setBoolean("pig.noSplitCombination", true);
- m_conf = job.getConfiguration();
- HBaseConfiguration.addHbaseResources(m_conf);
-
- // Make sure the HBase, ZooKeeper, and Guava jars get shipped.
- TableMapReduceUtil.addDependencyJars(job.getConfiguration(),
- org.apache.hadoop.hbase.client.HTable.class,
- com.google.common.collect.Lists.class,
- org.apache.zookeeper.ZooKeeper.class);
+ m_conf = initialiseHBaseClassLoaderResources(job);
String tablename = location;
if (location.startsWith("hbase://")){
@@ -574,6 +568,24 @@ public class HBaseStorage extends LoadFu
m_conf.set(TableInputFormat.SCAN, convertScanToString(scan));
}
+ private Configuration initialiseHBaseClassLoaderResources(Job job) throws IOException {
+ Configuration hbaseConfig = initialiseHBaseConfig(job.getConfiguration());
+
+ // Make sure the HBase, ZooKeeper, and Guava jars get shipped.
+ TableMapReduceUtil.addDependencyJars(job.getConfiguration(),
+ org.apache.hadoop.hbase.client.HTable.class,
+ com.google.common.collect.Lists.class,
+ org.apache.zookeeper.ZooKeeper.class);
+
+ return hbaseConfig;
+ }
+
+ private Configuration initialiseHBaseConfig(Configuration conf) {
+ Configuration hbaseConfig = HBaseConfiguration.create();
+ ConfigurationUtil.mergeConf(hbaseConfig, conf);
+ return hbaseConfig;
+ }
+
@Override
public String relativeToAbsolutePath(String location, Path curDir)
throws IOException {
@@ -610,7 +622,7 @@ public class HBaseStorage extends LoadFu
public OutputFormat getOutputFormat() throws IOException {
if (outputFormat == null) {
this.outputFormat = new TableOutputFormat();
- HBaseConfiguration.addHbaseResources(m_conf);
+ m_conf = initialiseHBaseConfig(m_conf);
this.outputFormat.setConf(m_conf);
}
return outputFormat;
@@ -751,7 +763,7 @@ public class HBaseStorage extends LoadFu
schema_ = (ResourceSchema) ObjectSerializer.deserialize(serializedSchema);
}
- m_conf = HBaseConfiguration.addHbaseResources(job.getConfiguration());
+ m_conf = initialiseHBaseClassLoaderResources(job);
}
@Override