You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@pig.apache.org by dv...@apache.org on 2012/08/24 05:37:50 UTC
svn commit: r1376819 - in /pig/trunk: ./
shims/src/hadoop20/org/apache/pig/backend/hadoop/executionengine/shims/
shims/src/hadoop23/org/apache/pig/backend/hadoop/executionengine/shims/
src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/ t...
Author: dvryaboy
Date: Fri Aug 24 03:37:50 2012
New Revision: 1376819
URL: http://svn.apache.org/viewvc?rev=1376819&view=rev
Log:
PIG-2890: Revert PIG-2578
Modified:
pig/trunk/CHANGES.txt
pig/trunk/shims/src/hadoop20/org/apache/pig/backend/hadoop/executionengine/shims/HadoopShims.java
pig/trunk/shims/src/hadoop23/org/apache/pig/backend/hadoop/executionengine/shims/HadoopShims.java
pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/JobControlCompiler.java
pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/JobControlCompiler.java.orig
pig/trunk/test/org/apache/pig/test/TestMultiQueryLocal.java
Modified: pig/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/pig/trunk/CHANGES.txt?rev=1376819&r1=1376818&r2=1376819&view=diff
==============================================================================
--- pig/trunk/CHANGES.txt (original)
+++ pig/trunk/CHANGES.txt Fri Aug 24 03:37:50 2012
@@ -24,6 +24,8 @@ INCOMPATIBLE CHANGES
IMPROVEMENTS
+PIG-2890: Revert PIG-2578 (dvryaboy)
+
PIG-2850: Pig should support loading macro files as resources stored in JAR files (matterhayes via dvryaboy)
PIG-1314: Add DateTime Support to Pig (zjshen via thejas)
Modified: pig/trunk/shims/src/hadoop20/org/apache/pig/backend/hadoop/executionengine/shims/HadoopShims.java
URL: http://svn.apache.org/viewvc/pig/trunk/shims/src/hadoop20/org/apache/pig/backend/hadoop/executionengine/shims/HadoopShims.java?rev=1376819&r1=1376818&r2=1376819&view=diff
==============================================================================
--- pig/trunk/shims/src/hadoop20/org/apache/pig/backend/hadoop/executionengine/shims/HadoopShims.java (original)
+++ pig/trunk/shims/src/hadoop20/org/apache/pig/backend/hadoop/executionengine/shims/HadoopShims.java Fri Aug 24 03:37:50 2012
@@ -50,7 +50,7 @@ public class HadoopShims {
static public TaskAttemptContext createTaskAttemptContext(Configuration conf,
TaskAttemptID taskId) {
- TaskAttemptContext newContext = new TaskAttemptContext(new Configuration(conf),
+ TaskAttemptContext newContext = new TaskAttemptContext(conf,
taskId);
return newContext;
}
@@ -58,7 +58,7 @@ public class HadoopShims {
static public JobContext createJobContext(Configuration conf,
JobID jobId) {
JobContext newJobContext = new JobContext(
- new Configuration(conf), jobId);
+ conf, jobId);
return newJobContext;
}
Modified: pig/trunk/shims/src/hadoop23/org/apache/pig/backend/hadoop/executionengine/shims/HadoopShims.java
URL: http://svn.apache.org/viewvc/pig/trunk/shims/src/hadoop23/org/apache/pig/backend/hadoop/executionengine/shims/HadoopShims.java?rev=1376819&r1=1376818&r2=1376819&view=diff
==============================================================================
--- pig/trunk/shims/src/hadoop23/org/apache/pig/backend/hadoop/executionengine/shims/HadoopShims.java (original)
+++ pig/trunk/shims/src/hadoop23/org/apache/pig/backend/hadoop/executionengine/shims/HadoopShims.java Fri Aug 24 03:37:50 2012
@@ -43,13 +43,13 @@ public class HadoopShims {
static public TaskAttemptContext createTaskAttemptContext(Configuration conf,
TaskAttemptID taskId) {
- TaskAttemptContext newContext = new TaskAttemptContextImpl(new Configuration(conf), taskId);
+ TaskAttemptContext newContext = new TaskAttemptContextImpl(conf, taskId);
return newContext;
}
static public JobContext createJobContext(Configuration conf,
JobID jobId) {
- JobContext newContext = new JobContextImpl(new Configuration(conf), jobId);
+ JobContext newContext = new JobContextImpl(conf, jobId);
return newContext;
}
Modified: pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/JobControlCompiler.java
URL: http://svn.apache.org/viewvc/pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/JobControlCompiler.java?rev=1376819&r1=1376818&r2=1376819&view=diff
==============================================================================
--- pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/JobControlCompiler.java (original)
+++ pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/JobControlCompiler.java Fri Aug 24 03:37:50 2012
@@ -498,13 +498,13 @@ public class JobControlCompiler{
for (POStore st: mapStores) {
storeLocations.add(st);
StoreFuncInterface sFunc = st.getStoreFunc();
- sFunc.setStoreLocation(st.getSFile().getFileName(), new org.apache.hadoop.mapreduce.Job(nwJob.getConfiguration()));
+ sFunc.setStoreLocation(st.getSFile().getFileName(), nwJob);
}
for (POStore st: reduceStores) {
storeLocations.add(st);
StoreFuncInterface sFunc = st.getStoreFunc();
- sFunc.setStoreLocation(st.getSFile().getFileName(), new org.apache.hadoop.mapreduce.Job(nwJob.getConfiguration()));
+ sFunc.setStoreLocation(st.getSFile().getFileName(), nwJob);
}
// the OutputFormat we report to Hadoop is always PigOutputFormat
Modified: pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/JobControlCompiler.java.orig
URL: http://svn.apache.org/viewvc/pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/JobControlCompiler.java.orig?rev=1376819&r1=1376818&r2=1376819&view=diff
==============================================================================
--- pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/JobControlCompiler.java.orig (original)
+++ pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/JobControlCompiler.java.orig Fri Aug 24 03:37:50 2012
@@ -84,6 +84,7 @@ import org.apache.pig.impl.io.NullableDo
import org.apache.pig.impl.io.NullableFloatWritable;
import org.apache.pig.impl.io.NullableIntWritable;
import org.apache.pig.impl.io.NullableLongWritable;
+import org.apache.pig.impl.io.NullableDateTimeWritable;
import org.apache.pig.impl.io.NullablePartitionWritable;
import org.apache.pig.impl.io.NullableText;
import org.apache.pig.impl.io.NullableTuple;
@@ -956,6 +957,12 @@ public class JobControlCompiler{
}
}
+ public static class PigDateTimeWritableComparator extends PigWritableComparator {
+ public PigDateTimeWritableComparator() {
+ super(NullableDateTimeWritable.class);
+ }
+ }
+
public static class PigCharArrayWritableComparator extends PigWritableComparator {
public PigCharArrayWritableComparator() {
super(NullableText.class);
@@ -1012,6 +1019,12 @@ public class JobControlCompiler{
}
}
+ public static class PigGroupingDateTimeWritableComparator extends WritableComparator {
+ public PigGroupingDateTimeWritableComparator() {
+ super(NullableDateTimeWritable.class, true);
+ }
+ }
+
public static class PigGroupingCharArrayWritableComparator extends WritableComparator {
public PigGroupingCharArrayWritableComparator() {
super(NullableText.class, true);
@@ -1087,6 +1100,10 @@ public class JobControlCompiler{
job.setSortComparatorClass(PigDoubleRawComparator.class);
break;
+ case DataType.DATETIME:
+ job.setSortComparatorClass(PigDateTimeRawComparator.class);
+ break;
+
case DataType.CHARARRAY:
job.setSortComparatorClass(PigTextRawComparator.class);
break;
@@ -1141,6 +1158,11 @@ public class JobControlCompiler{
job.setGroupingComparatorClass(PigGroupingDoubleWritableComparator.class);
break;
+ case DataType.DATETIME:
+ job.setSortComparatorClass(PigDateTimeWritableComparator.class);
+ job.setGroupingComparatorClass(PigGroupingDateTimeWritableComparator.class);
+ break;
+
case DataType.CHARARRAY:
job.setSortComparatorClass(PigCharArrayWritableComparator.class);
job.setGroupingComparatorClass(PigGroupingCharArrayWritableComparator.class);
Modified: pig/trunk/test/org/apache/pig/test/TestMultiQueryLocal.java
URL: http://svn.apache.org/viewvc/pig/trunk/test/org/apache/pig/test/TestMultiQueryLocal.java?rev=1376819&r1=1376818&r2=1376819&view=diff
==============================================================================
--- pig/trunk/test/org/apache/pig/test/TestMultiQueryLocal.java (original)
+++ pig/trunk/test/org/apache/pig/test/TestMultiQueryLocal.java Fri Aug 24 03:37:50 2012
@@ -17,8 +17,6 @@
*/
package org.apache.pig.test;
-import java.io.BufferedReader;
-import java.io.FileReader;
import java.io.StringReader;
import java.io.IOException;
import java.io.File;
@@ -31,16 +29,12 @@ import java.util.Properties;
import junit.framework.Assert;
import junit.framework.TestCase;
-import org.apache.hadoop.mapreduce.Job;
import org.apache.pig.ExecType;
import org.apache.pig.PigException;
import org.apache.pig.PigServer;
import org.apache.pig.backend.executionengine.ExecJob;
import org.apache.pig.backend.hadoop.executionengine.mapReduceLayer.MapReduceLauncher;
import org.apache.pig.backend.hadoop.executionengine.physicalLayer.plans.PhysicalPlan;
-import org.apache.pig.builtin.PigStorage;
-import org.apache.pig.data.Tuple;
-import org.apache.pig.data.TupleFactory;
import org.apache.pig.impl.io.FileLocalizer;
import org.apache.pig.impl.plan.Operator;
import org.apache.pig.impl.plan.OperatorPlan;
@@ -341,69 +335,6 @@ public class TestMultiQueryLocal {
Assert.fail();
}
}
-
- public static class PigStorageWithSuffix extends PigStorage {
-
- private String suffix;
- public PigStorageWithSuffix(String s) {
- this.suffix = s;
- }
- static private final String key="test.key";
- @Override
- public void setStoreLocation(String location, Job job) throws IOException {
- super.setStoreLocation(location, job);
- if (job.getConfiguration().get(key)==null) {
- job.getConfiguration().set(key, suffix);
- }
- suffix = job.getConfiguration().get(key);
- }
-
- @Override
- public void putNext(Tuple f) throws IOException {
- try {
- Tuple t = TupleFactory.getInstance().newTuple();
- for (Object obj : f.getAll()) {
- t.append(obj);
- }
- t.append(suffix);
- writer.write(null, t);
- } catch (InterruptedException e) {
- throw new IOException(e);
- }
- }
- }
-
- // See PIG-2578
- @Test
- public void testMultiStoreWithConfig() {
-
- System.out.println("===== test multi-query with competing config =====");
-
- try {
- myPig.setBatchOn();
-
- myPig.registerQuery("a = load 'test/org/apache/pig/test/data/passwd' " +
- "using PigStorage(':') as (uname:chararray, passwd:chararray, uid:int,gid:int);");
- myPig.registerQuery("store a into '/tmp/Pig-TestMultiQueryLocal1' using " + PigStorageWithSuffix.class.getName() + "('a');");
- myPig.registerQuery("store a into '/tmp/Pig-TestMultiQueryLocal2' using " + PigStorageWithSuffix.class.getName() + "('b');");
-
- myPig.executeBatch();
- myPig.discardBatch();
- BufferedReader reader = new BufferedReader(new FileReader("/tmp/Pig-TestMultiQueryLocal1/part-m-00000"));
- String line;
- while ((line = reader.readLine())!=null) {
- Assert.assertTrue(line.endsWith("a"));
- }
- reader = new BufferedReader(new FileReader("/tmp/Pig-TestMultiQueryLocal2/part-m-00000"));
- while ((line = reader.readLine())!=null) {
- Assert.assertTrue(line.endsWith("b"));
- }
-
- } catch (Exception e) {
- e.printStackTrace();
- Assert.fail();
- }
- }
@Test
public void testMultiQueryWithExplain() {