You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@pig.apache.org by ch...@apache.org on 2013/11/29 00:58:01 UTC
svn commit: r1546477 [2/2] - in /pig/branches/tez:
src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/
src/org/apache/pig/backend/hadoop/executionengine/physicalLayer/
src/org/apache/pig/backend/hadoop/executionengine/physicalLayer/relati...
Added: pig/branches/tez/src/org/apache/pig/backend/hadoop/executionengine/tez/TezLoad.java
URL: http://svn.apache.org/viewvc/pig/branches/tez/src/org/apache/pig/backend/hadoop/executionengine/tez/TezLoad.java?rev=1546477&view=auto
==============================================================================
--- pig/branches/tez/src/org/apache/pig/backend/hadoop/executionengine/tez/TezLoad.java (added)
+++ pig/branches/tez/src/org/apache/pig/backend/hadoop/executionengine/tez/TezLoad.java Thu Nov 28 23:58:00 2013
@@ -0,0 +1,35 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.pig.backend.hadoop.executionengine.tez;
+
+import java.util.Map;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.pig.backend.executionengine.ExecException;
+import org.apache.tez.runtime.api.LogicalInput;
+
+/**
+ * This interface is implemented by PhysicalOperators that can have Tez inputs
+ * attached directly to the operator.
+ */
+
+public interface TezLoad {
+ public void attachInputs(Map<String, LogicalInput> inputs,
+ Configuration conf) throws ExecException;
+}
Modified: pig/branches/tez/src/org/apache/pig/backend/hadoop/executionengine/util/CombinerOptimizerUtil.java
URL: http://svn.apache.org/viewvc/pig/branches/tez/src/org/apache/pig/backend/hadoop/executionengine/util/CombinerOptimizerUtil.java?rev=1546477&r1=1546476&r2=1546477&view=diff
==============================================================================
--- pig/branches/tez/src/org/apache/pig/backend/hadoop/executionengine/util/CombinerOptimizerUtil.java (original)
+++ pig/branches/tez/src/org/apache/pig/backend/hadoop/executionengine/util/CombinerOptimizerUtil.java Thu Nov 28 23:58:00 2013
@@ -213,10 +213,10 @@ public class CombinerOptimizerUtil {
// Use the POCombiner package in the combine plan
// as it needs to act differently than the regular
// package operator.
+ CombinerPackager pkgr = new CombinerPackager(pack.getPkgr(), bags);
POPackage combinePack = pack.clone();
- CombinerPackager combinePkgr = new CombinerPackager(
- pack.getPkgr(), bags);
- combinePack.setPkgr(combinePkgr);
+ combinePack.setPkgr(pkgr);
+
combinePlan.add(combinePack);
combinePlan.add(cfe);
combinePlan.connect(combinePack, cfe);
@@ -253,7 +253,7 @@ public class CombinerOptimizerUtil {
// Change the package operator in the reduce plan to
// be the POCombiner package, as it needs to act
// differently than the regular package operator.
- pack.setPkgr(combinePkgr.clone());
+ pack.setPkgr(pkgr.clone());
} catch (Exception e) {
int errCode = 2018;
String msg = "Internal error. Unable to introduce the combiner for optimization.";
Modified: pig/branches/tez/src/org/apache/pig/data/BinInterSedes.java
URL: http://svn.apache.org/viewvc/pig/branches/tez/src/org/apache/pig/data/BinInterSedes.java?rev=1546477&r1=1546476&r2=1546477&view=diff
==============================================================================
--- pig/branches/tez/src/org/apache/pig/data/BinInterSedes.java (original)
+++ pig/branches/tez/src/org/apache/pig/data/BinInterSedes.java Thu Nov 28 23:58:00 2013
@@ -688,10 +688,6 @@ public class BinInterSedes implements In
@Override
public void setConf(Configuration conf) {
- if (!(conf instanceof JobConf)) {
- mLog.warn("Expected jobconf in setConf, got " + conf.getClass().getName());
- return;
- }
try {
mAsc = (boolean[]) ObjectSerializer.deserialize(conf.get("pig.sortOrder"));
mSecondaryAsc = (boolean[]) ObjectSerializer.deserialize(conf.get("pig.secondarySortOrder"));
Modified: pig/branches/tez/src/org/apache/pig/data/DefaultTuple.java
URL: http://svn.apache.org/viewvc/pig/branches/tez/src/org/apache/pig/data/DefaultTuple.java?rev=1546477&r1=1546476&r2=1546477&view=diff
==============================================================================
--- pig/branches/tez/src/org/apache/pig/data/DefaultTuple.java (original)
+++ pig/branches/tez/src/org/apache/pig/data/DefaultTuple.java Thu Nov 28 23:58:00 2013
@@ -230,13 +230,8 @@ public class DefaultTuple extends Abstra
@Override
public void setConf(Configuration conf) {
- if (!(conf instanceof JobConf)) {
- mLog.warn("Expected jobconf in setConf, got " + conf.getClass().getName());
- return;
- }
- JobConf jconf = (JobConf) conf;
try {
- mAsc = (boolean[]) ObjectSerializer.deserialize(jconf.get("pig.sortOrder"));
+ mAsc = (boolean[]) ObjectSerializer.deserialize(conf.get("pig.sortOrder"));
} catch (IOException ioe) {
mLog.error("Unable to deserialize pig.sortOrder " + ioe.getMessage());
throw new RuntimeException(ioe);
Modified: pig/branches/tez/test/e2e/pig/tests/tez.conf
URL: http://svn.apache.org/viewvc/pig/branches/tez/test/e2e/pig/tests/tez.conf?rev=1546477&r1=1546476&r2=1546477&view=diff
==============================================================================
--- pig/branches/tez/test/e2e/pig/tests/tez.conf (original)
+++ pig/branches/tez/test/e2e/pig/tests/tez.conf Thu Nov 28 23:58:00 2013
@@ -128,6 +128,20 @@ store d into ':OUTPATH:';\,
'sortArgs' => ['-t', ' ', '-k', '2,2'],
},
]
+ },
+ {
+ 'name' => 'Join',
+ 'tests' => [
+ {
+ 'num' => 1,
+ 'pig' => q\a = load ':INPATH:/singlefile/studenttab10k' using PigStorage() as (name, age, gpa);
+b = load ':INPATH:/singlefile/votertab10k' as (name, age, registration, contributions);
+c = filter a by age < 20;
+d = filter b by age < 20;
+e = join c by name, d by name;
+store e into ':OUTPATH:';\,
+ }
+ ]
}
]
}