You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@pig.apache.org by pr...@apache.org on 2009/08/05 01:56:26 UTC
svn commit: r801031 - in /hadoop/pig/branches/branch-0.3: ./ bin/
src/org/apache/pig/backend/hadoop/datastorage/
src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/
src/org/apache/pig/impl/io/ src/org/apache/pig/tools/pigstats/ test/
Author: pradeepkth
Date: Tue Aug 4 23:56:25 2009
New Revision: 801031
URL: http://svn.apache.org/viewvc?rev=801031&view=rev
Log:
rolling back to revision 800877 to revert accidental changes
Modified:
hadoop/pig/branches/branch-0.3/bin/pig
hadoop/pig/branches/branch-0.3/build.xml
hadoop/pig/branches/branch-0.3/src/org/apache/pig/backend/hadoop/datastorage/HConfiguration.java
hadoop/pig/branches/branch-0.3/src/org/apache/pig/backend/hadoop/datastorage/HDataStorage.java
hadoop/pig/branches/branch-0.3/src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/MRCompiler.java
hadoop/pig/branches/branch-0.3/src/org/apache/pig/impl/io/NullableBytesWritable.java
hadoop/pig/branches/branch-0.3/src/org/apache/pig/tools/pigstats/PigStats.java
hadoop/pig/branches/branch-0.3/test/ (props changed)
Modified: hadoop/pig/branches/branch-0.3/bin/pig
URL: http://svn.apache.org/viewvc/hadoop/pig/branches/branch-0.3/bin/pig?rev=801031&r1=801030&r2=801031&view=diff
==============================================================================
--- hadoop/pig/branches/branch-0.3/bin/pig (original)
+++ hadoop/pig/branches/branch-0.3/bin/pig Tue Aug 4 23:56:25 2009
@@ -34,7 +34,7 @@
#
# PIG_ROOT_LOGGER The root appender. Default is INFO,console
#
-# PIG_HADOOP_VERSION Version of hadoop to run with. Default is 20 (0.20).
+# PIG_HADOOP_VERSION Version of hadoop to run with. Default is 18 (0.18).
cygwin=false
case "`uname`" in
@@ -136,7 +136,7 @@
done
# Set the version for Hadoop, default to 17
-PIG_HADOOP_VERSION="${PIG_HADOOP_VERSION:-20}"
+PIG_HADOOP_VERSION="${PIG_HADOOP_VERSION:-18}"
# add libs to CLASSPATH. There can be more than one version of the hadoop
# libraries in the lib dir, so don't blindly add them all. Only add the one
# that matche PIG_HADOOP_VERSION.
Modified: hadoop/pig/branches/branch-0.3/build.xml
URL: http://svn.apache.org/viewvc/hadoop/pig/branches/branch-0.3/build.xml?rev=801031&r1=801030&r2=801031&view=diff
==============================================================================
--- hadoop/pig/branches/branch-0.3/build.xml (original)
+++ hadoop/pig/branches/branch-0.3/build.xml Tue Aug 4 23:56:25 2009
@@ -47,14 +47,14 @@
<!-- property name="build.encoding" value="ISO-8859-1" / -->
<property name="build.encoding" value="UTF8" />
<!-- TODO with only one version of hadoop in the lib folder we do not need that anymore -->
- <property name="hadoop.jarfile" value="hadoop20.jar" />
+ <property name="hadoop.jarfile" value="hadoop18.jar" />
<property name="hbase.jarfile" value="hbase-0.18.1.jar" />
<property name="hbase.test.jarfile" value="hbase-0.18.1-test.jar" />
<!-- javac properties -->
<property name="javac.debug" value="on" />
<property name="javac.optimize" value="on" />
- <property name="javac.deprecation" value="off" />
+ <property name="javac.deprecation" value="on" />
<property name="javac.version" value="1.5" />
<property name="javac.args" value="" />
<!-- default warnings option -->
@@ -446,7 +446,6 @@
<!-- Excluded under Windows.-->
<exclude name="**/TestHBaseStorage.java" if="isWindows" />
<!-- Excluced because we don't want to run them -->
- <exclude name="**/TestHBaseStorage.java" />
<exclude name="**/PigExecTestCase.java" />
<exclude name="**/TypeCheckingTestUtil.java" />
<exclude name="**/TypeGraphPrinter.java" />
Modified: hadoop/pig/branches/branch-0.3/src/org/apache/pig/backend/hadoop/datastorage/HConfiguration.java
URL: http://svn.apache.org/viewvc/hadoop/pig/branches/branch-0.3/src/org/apache/pig/backend/hadoop/datastorage/HConfiguration.java?rev=801031&r1=801030&r2=801031&view=diff
==============================================================================
--- hadoop/pig/branches/branch-0.3/src/org/apache/pig/backend/hadoop/datastorage/HConfiguration.java (original)
+++ hadoop/pig/branches/branch-0.3/src/org/apache/pig/backend/hadoop/datastorage/HConfiguration.java Tue Aug 4 23:56:25 2009
@@ -23,6 +23,7 @@
import java.util.Enumeration;
import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.mapred.JobConf;
import java.util.Properties;
Modified: hadoop/pig/branches/branch-0.3/src/org/apache/pig/backend/hadoop/datastorage/HDataStorage.java
URL: http://svn.apache.org/viewvc/hadoop/pig/branches/branch-0.3/src/org/apache/pig/backend/hadoop/datastorage/HDataStorage.java?rev=801031&r1=801030&r2=801031&view=diff
==============================================================================
--- hadoop/pig/branches/branch-0.3/src/org/apache/pig/backend/hadoop/datastorage/HDataStorage.java (original)
+++ hadoop/pig/branches/branch-0.3/src/org/apache/pig/backend/hadoop/datastorage/HDataStorage.java Tue Aug 4 23:56:25 2009
@@ -30,7 +30,7 @@
import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.hdfs.DistributedFileSystem;
+import org.apache.hadoop.dfs.DistributedFileSystem;
import org.apache.hadoop.conf.Configuration;
import org.apache.pig.PigException;
import org.apache.pig.backend.datastorage.ContainerDescriptor;
Modified: hadoop/pig/branches/branch-0.3/src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/MRCompiler.java
URL: http://svn.apache.org/viewvc/hadoop/pig/branches/branch-0.3/src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/MRCompiler.java?rev=801031&r1=801030&r2=801031&view=diff
==============================================================================
--- hadoop/pig/branches/branch-0.3/src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/MRCompiler.java (original)
+++ hadoop/pig/branches/branch-0.3/src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/MRCompiler.java Tue Aug 4 23:56:25 2009
@@ -30,7 +30,6 @@
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
-import org.apache.hadoop.mapred.JobConf;
import org.apache.pig.FuncSpec;
import org.apache.pig.PigException;
import org.apache.pig.PigWarning;
@@ -1324,10 +1323,10 @@
ExecutionEngine eng = pigContext.getExecutionEngine();
if(eng instanceof HExecutionEngine){
try {
- val = ((JobConf)((HExecutionEngine)eng).getJobClient().getConf()).getNumReduceTasks();
+ val = Math.round(0.9f * ((HExecutionEngine)eng).getJobClient().getDefaultReduces());
if(val<=0)
val = 1;
- } catch (Exception e) {
+ } catch (IOException e) {
int errCode = 6015;
String msg = "Problem getting the default number of reduces from the Job Client.";
throw new MRCompilerException(msg, errCode, PigException.REMOTE_ENVIRONMENT, e);
Modified: hadoop/pig/branches/branch-0.3/src/org/apache/pig/impl/io/NullableBytesWritable.java
URL: http://svn.apache.org/viewvc/hadoop/pig/branches/branch-0.3/src/org/apache/pig/impl/io/NullableBytesWritable.java?rev=801031&r1=801030&r2=801031&view=diff
==============================================================================
--- hadoop/pig/branches/branch-0.3/src/org/apache/pig/impl/io/NullableBytesWritable.java (original)
+++ hadoop/pig/branches/branch-0.3/src/org/apache/pig/impl/io/NullableBytesWritable.java Tue Aug 4 23:56:25 2009
@@ -39,6 +39,6 @@
public Object getValueAsPigType() {
BytesWritable bw = (BytesWritable)mValue;
- return isNull() ? null : new DataByteArray(bw.getBytes(), 0, bw.getLength());
+ return isNull() ? null : new DataByteArray(bw.get(), 0, bw.getSize());
}
}
Modified: hadoop/pig/branches/branch-0.3/src/org/apache/pig/tools/pigstats/PigStats.java
URL: http://svn.apache.org/viewvc/hadoop/pig/branches/branch-0.3/src/org/apache/pig/tools/pigstats/PigStats.java?rev=801031&r1=801030&r2=801031&view=diff
==============================================================================
--- hadoop/pig/branches/branch-0.3/src/org/apache/pig/tools/pigstats/PigStats.java (original)
+++ hadoop/pig/branches/branch-0.3/src/org/apache/pig/tools/pigstats/PigStats.java Tue Aug 4 23:56:25 2009
@@ -148,14 +148,13 @@
try {
counters = rj.getCounters();
Counters.Group taskgroup = counters.getGroup("org.apache.hadoop.mapred.Task$Counter");
- Counters.Group hdfsgroup = counters.getGroup("FileSystemCounters");
+ Counters.Group hdfsgroup = counters.getGroup("org.apache.hadoop.mapred.Task$FileSystemCounter");
jobStats.put("PIG_STATS_MAP_INPUT_RECORDS", (Long.valueOf(taskgroup.getCounterForName("MAP_INPUT_RECORDS").getCounter())).toString());
jobStats.put("PIG_STATS_MAP_OUTPUT_RECORDS", (Long.valueOf(taskgroup.getCounterForName("MAP_OUTPUT_RECORDS").getCounter())).toString());
jobStats.put("PIG_STATS_REDUCE_INPUT_RECORDS", (Long.valueOf(taskgroup.getCounterForName("REDUCE_INPUT_RECORDS").getCounter())).toString());
jobStats.put("PIG_STATS_REDUCE_OUTPUT_RECORDS", (Long.valueOf(taskgroup.getCounterForName("REDUCE_OUTPUT_RECORDS").getCounter())).toString());
- jobStats.put("PIG_STATS_BYTES_WRITTEN", (Long.valueOf(hdfsgroup.getCounterForName("HDFS_BYTES_WRITTEN").getCounter())).toString());
-
+ jobStats.put("PIG_STATS_BYTES_WRITTEN", (Long.valueOf(hdfsgroup.getCounterForName("HDFS_WRITE").getCounter())).toString());
} catch (IOException e) {
// TODO Auto-generated catch block
String error = "Unable to get the counters.";
Propchange: hadoop/pig/branches/branch-0.3/test/
------------------------------------------------------------------------------
--- svn:externals (original)
+++ svn:externals Tue Aug 4 23:56:25 2009
@@ -1 +1 @@
-bin http://svn.apache.org/repos/asf/hadoop/nightly/test-patch
+bin http://svn.apache.org/repos/asf/hadoop/common/nightly/test-patch