You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@pig.apache.org by da...@apache.org on 2014/09/17 01:49:48 UTC
svn commit: r1625434 - in /pig/trunk/test: excluded-tests-20
org/apache/pig/test/TestLoaderStorerShipCacheFiles.java
org/apache/pig/test/TestLoaderStorerShipCacheFilesMR.java
org/apache/pig/tez/TestLoaderStorerShipCacheFilesTez.java tez-tests
Author: daijy
Date: Tue Sep 16 23:49:47 2014
New Revision: 1625434
URL: http://svn.apache.org/r1625434
Log:
PIG-4141: Ship UDF/LoadFunc/StoreFunc dependent jar automatically (PIG-4141-6.patch)
Added:
pig/trunk/test/org/apache/pig/test/TestLoaderStorerShipCacheFilesMR.java
pig/trunk/test/org/apache/pig/tez/TestLoaderStorerShipCacheFilesTez.java
Modified:
pig/trunk/test/excluded-tests-20
pig/trunk/test/org/apache/pig/test/TestLoaderStorerShipCacheFiles.java
pig/trunk/test/tez-tests
Modified: pig/trunk/test/excluded-tests-20
URL: http://svn.apache.org/viewvc/pig/trunk/test/excluded-tests-20?rev=1625434&r1=1625433&r2=1625434&view=diff
==============================================================================
--- pig/trunk/test/excluded-tests-20 (original)
+++ pig/trunk/test/excluded-tests-20 Tue Sep 16 23:49:47 2014
@@ -5,3 +5,4 @@
**/TestTezAutoParallelism.java
**/TestJobSubmissionTez.java
**/TestGroupConstParallelTez.java
+**/TestLoaderStorerShipCacheFilesTez.java
Modified: pig/trunk/test/org/apache/pig/test/TestLoaderStorerShipCacheFiles.java
URL: http://svn.apache.org/viewvc/pig/trunk/test/org/apache/pig/test/TestLoaderStorerShipCacheFiles.java?rev=1625434&r1=1625433&r2=1625434&view=diff
==============================================================================
--- pig/trunk/test/org/apache/pig/test/TestLoaderStorerShipCacheFiles.java (original)
+++ pig/trunk/test/org/apache/pig/test/TestLoaderStorerShipCacheFiles.java Tue Sep 16 23:49:47 2014
@@ -17,26 +17,18 @@
*/
package org.apache.pig.test;
-import java.util.ArrayList;
import java.util.List;
import org.apache.pig.ExecType;
import org.apache.pig.PigServer;
-import org.apache.pig.backend.hadoop.executionengine.mapReduceLayer.MapReduceOper;
-import org.apache.pig.backend.hadoop.executionengine.mapReduceLayer.plans.MROperPlan;
import org.apache.pig.backend.hadoop.executionengine.physicalLayer.plans.PhysicalPlan;
-import org.apache.pig.backend.hadoop.executionengine.physicalLayer.plans.UdfCacheShipFilesVisitor;
-import org.apache.pig.backend.hadoop.executionengine.tez.TezLauncher;
-import org.apache.pig.backend.hadoop.executionengine.tez.TezOperPlan;
-import org.apache.pig.backend.hadoop.executionengine.tez.TezPOUserFuncVisitor;
-import org.apache.pig.backend.hadoop.executionengine.tez.TezPlanContainer;
-import org.apache.pig.impl.plan.VisitorException;
+import org.apache.pig.impl.PigContext;
import org.apache.pig.impl.util.Utils;
import org.junit.Assert;
import org.junit.Before;
import org.junit.Test;
-public class TestLoaderStorerShipCacheFiles {
+public abstract class TestLoaderStorerShipCacheFiles {
private static PigServer pigServer;
@Before
@@ -58,12 +50,7 @@ public class TestLoaderStorerShipCacheFi
String[] expectedJars = new String[] {"hive-common", "hive-exec", "hive-serde",
"hive-shims-0." + hadoopVersion, "hive-shims-common-0", "hive-shims-common-secure"};
- MROperPlan mrPlan = Util.buildMRPlan(pp, pigServer.getPigContext());
- assertMRPlanContains(mrPlan.getRoots().get(0), expectedJars, 6);
-
- TezLauncher launcher = new TezLauncher();
- TezPlanContainer tezPlanContainer = launcher.compile(pp, pigServer.getPigContext());
- assertTezPlanContains(tezPlanContainer.getRoots().get(0).getNode(), expectedJars, 6);
+ checkPlan(pp, expectedJars, 6, pigServer.getPigContext());
}
@Test
@@ -79,12 +66,7 @@ public class TestLoaderStorerShipCacheFi
String[] expectedJars = new String[] {"hive-common", "hive-exec", "hive-serde",
"hive-shims-0." + hadoopVersion, "hive-shims-common-0", "hive-shims-common-secure"};
- MROperPlan mrPlan = Util.buildMRPlan(pp, pigServer.getPigContext());
- assertMRPlanContains(mrPlan.getRoots().get(0), expectedJars, 6);
-
- TezLauncher launcher = new TezLauncher();
- TezPlanContainer tezPlanContainer = launcher.compile(pp, pigServer.getPigContext());
- assertTezPlanContains(tezPlanContainer.getRoots().get(0).getNode(), expectedJars, 6);
+ checkPlan(pp, expectedJars, 6, pigServer.getPigContext());
}
@Test
@@ -95,12 +77,7 @@ public class TestLoaderStorerShipCacheFi
String[] expectedJars = new String[] {"avro-1", "avro-mapred-"};
- MROperPlan mrPlan = Util.buildMRPlan(pp, pigServer.getPigContext());
- assertMRPlanContains(mrPlan.getRoots().get(0), expectedJars, 2);
-
- TezLauncher launcher = new TezLauncher();
- TezPlanContainer tezPlanContainer = launcher.compile(pp, pigServer.getPigContext());
- assertTezPlanContains(tezPlanContainer.getRoots().get(0).getNode(), expectedJars, 2);
+ checkPlan(pp, expectedJars, 2, pigServer.getPigContext());
}
@Test
@@ -112,43 +89,12 @@ public class TestLoaderStorerShipCacheFi
String[] expectedJars = new String[] {"jackson-core-"};
- MROperPlan mrPlan = Util.buildMRPlan(pp, pigServer.getPigContext());
- assertMRPlanContains(mrPlan.getLeaves().get(0), expectedJars, 1);
-
- TezLauncher launcher = new TezLauncher();
- TezPlanContainer tezPlanContainer = launcher.compile(pp, pigServer.getPigContext());
- assertTezPlanContains(tezPlanContainer.getRoots().get(0).getNode(), expectedJars, 1);
+ checkPlan(pp, expectedJars, 1, pigServer.getPigContext());
}
- private void assertMRPlanContains(MapReduceOper mro, String[] expectedFiles, int size) throws VisitorException {
- List<String> cacheFiles = new ArrayList<String>();
- List<String> shipFiles = new ArrayList<String>();
- UdfCacheShipFilesVisitor mapUdfCacheFileVisitor = new UdfCacheShipFilesVisitor(mro.mapPlan);
- mapUdfCacheFileVisitor.visit();
- cacheFiles.addAll(mapUdfCacheFileVisitor.getCacheFiles());
- shipFiles.addAll(mapUdfCacheFileVisitor.getShipFiles());
-
- UdfCacheShipFilesVisitor reduceUdfCacheFileVisitor = new UdfCacheShipFilesVisitor(mro.reducePlan);
- reduceUdfCacheFileVisitor.visit();
- cacheFiles.addAll(reduceUdfCacheFileVisitor.getCacheFiles());
- shipFiles.addAll(reduceUdfCacheFileVisitor.getShipFiles());
-
- Assert.assertEquals(shipFiles.size(), size);
- assertContains(shipFiles, expectedFiles);
- }
-
- private void assertTezPlanContains(TezOperPlan plan, String[] expectedFiles, int size) throws VisitorException {
- TezPOUserFuncVisitor udfVisitor = new TezPOUserFuncVisitor(plan);
- udfVisitor.visit();
-
- List<String> shipFiles = new ArrayList<String>();
- shipFiles.addAll(udfVisitor.getShipFiles());
-
- Assert.assertEquals(shipFiles.size(), size);
- assertContains(shipFiles, expectedFiles);
- }
+ abstract protected void checkPlan(PhysicalPlan pp, String[] expectedJars, int size, PigContext pigContext) throws Exception;
- private void assertContains(List<String> collectedFiles, String[] expectedFiles) {
+ protected void assertContains(List<String> collectedFiles, String[] expectedFiles) {
for (String expectedFile : expectedFiles) {
boolean found = false;
for (String collectedFile : collectedFiles) {
Added: pig/trunk/test/org/apache/pig/test/TestLoaderStorerShipCacheFilesMR.java
URL: http://svn.apache.org/viewvc/pig/trunk/test/org/apache/pig/test/TestLoaderStorerShipCacheFilesMR.java?rev=1625434&view=auto
==============================================================================
--- pig/trunk/test/org/apache/pig/test/TestLoaderStorerShipCacheFilesMR.java (added)
+++ pig/trunk/test/org/apache/pig/test/TestLoaderStorerShipCacheFilesMR.java Tue Sep 16 23:49:47 2014
@@ -0,0 +1,55 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.pig.test;
+
+import java.util.ArrayList;
+import java.util.List;
+
+import org.apache.pig.backend.hadoop.executionengine.mapReduceLayer.MapReduceOper;
+import org.apache.pig.backend.hadoop.executionengine.mapReduceLayer.plans.MROperPlan;
+import org.apache.pig.backend.hadoop.executionengine.physicalLayer.plans.PhysicalPlan;
+import org.apache.pig.backend.hadoop.executionengine.physicalLayer.plans.UdfCacheShipFilesVisitor;
+import org.apache.pig.impl.PigContext;
+import org.apache.pig.impl.plan.VisitorException;
+import org.junit.Assert;
+
+public class TestLoaderStorerShipCacheFilesMR extends TestLoaderStorerShipCacheFiles {
+
+ @Override
+ protected void checkPlan(PhysicalPlan pp, String[] expectedFiles, int size, PigContext pigContext) throws Exception {
+ MROperPlan mrPlan = Util.buildMRPlan(pp, pigContext);
+ assertPlanContains(mrPlan.getLeaves().get(0), expectedFiles, size);
+ }
+
+ private void assertPlanContains(MapReduceOper mro, String[] expectedFiles, int size) throws VisitorException {
+ List<String> cacheFiles = new ArrayList<String>();
+ List<String> shipFiles = new ArrayList<String>();
+ UdfCacheShipFilesVisitor mapUdfCacheFileVisitor = new UdfCacheShipFilesVisitor(mro.mapPlan);
+ mapUdfCacheFileVisitor.visit();
+ cacheFiles.addAll(mapUdfCacheFileVisitor.getCacheFiles());
+ shipFiles.addAll(mapUdfCacheFileVisitor.getShipFiles());
+
+ UdfCacheShipFilesVisitor reduceUdfCacheFileVisitor = new UdfCacheShipFilesVisitor(mro.reducePlan);
+ reduceUdfCacheFileVisitor.visit();
+ cacheFiles.addAll(reduceUdfCacheFileVisitor.getCacheFiles());
+ shipFiles.addAll(reduceUdfCacheFileVisitor.getShipFiles());
+
+ Assert.assertEquals(shipFiles.size(), size);
+ assertContains(shipFiles, expectedFiles);
+ }
+}
Added: pig/trunk/test/org/apache/pig/tez/TestLoaderStorerShipCacheFilesTez.java
URL: http://svn.apache.org/viewvc/pig/trunk/test/org/apache/pig/tez/TestLoaderStorerShipCacheFilesTez.java?rev=1625434&view=auto
==============================================================================
--- pig/trunk/test/org/apache/pig/tez/TestLoaderStorerShipCacheFilesTez.java (added)
+++ pig/trunk/test/org/apache/pig/tez/TestLoaderStorerShipCacheFilesTez.java Tue Sep 16 23:49:47 2014
@@ -0,0 +1,54 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.pig.tez;
+
+import java.util.ArrayList;
+import java.util.List;
+
+import org.apache.pig.backend.hadoop.executionengine.physicalLayer.plans.PhysicalPlan;
+import org.apache.pig.backend.hadoop.executionengine.tez.TezLauncher;
+import org.apache.pig.backend.hadoop.executionengine.tez.TezOperPlan;
+import org.apache.pig.backend.hadoop.executionengine.tez.TezPOUserFuncVisitor;
+import org.apache.pig.backend.hadoop.executionengine.tez.TezPlanContainer;
+import org.apache.pig.impl.PigContext;
+import org.apache.pig.impl.plan.VisitorException;
+import org.apache.pig.test.TestLoaderStorerShipCacheFiles;
+import org.junit.Assert;
+
+public class TestLoaderStorerShipCacheFilesTez extends TestLoaderStorerShipCacheFiles {
+
+ @Override
+ protected void checkPlan(PhysicalPlan pp, String[] expectedFiles, int size, PigContext pigContext) throws Exception {
+ TezLauncher launcher = new TezLauncher();
+ TezPlanContainer tezPlanContainer = launcher.compile(pp, pigContext);
+ assertPlanContains(tezPlanContainer.getRoots().get(0).getNode(), expectedFiles, size);
+ }
+
+ private void assertPlanContains(TezOperPlan plan, String[] expectedFiles, int size) throws VisitorException {
+ TezPOUserFuncVisitor udfVisitor = new TezPOUserFuncVisitor(plan);
+ udfVisitor.visit();
+
+ List<String> shipFiles = new ArrayList<String>();
+ shipFiles.addAll(udfVisitor.getShipFiles());
+
+ Assert.assertEquals(shipFiles.size(), size);
+ assertContains(shipFiles, expectedFiles);
+ }
+
+
+}
Modified: pig/trunk/test/tez-tests
URL: http://svn.apache.org/viewvc/pig/trunk/test/tez-tests?rev=1625434&r1=1625433&r2=1625434&view=diff
==============================================================================
--- pig/trunk/test/tez-tests (original)
+++ pig/trunk/test/tez-tests Tue Sep 16 23:49:47 2014
@@ -70,3 +70,4 @@
**/TestMergeJoin.java
**/TestNativeMapReduce.java
**/TestPigProgressReporting.java
+**/TestLoaderStorerShipCacheFilesTez.java