You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@pig.apache.org by ol...@apache.org on 2008/10/22 20:37:14 UTC
svn commit: r707158 [2/2] - in /incubator/pig/branches/types: ./
src/org/apache/pig/builtin/ src/org/apache/pig/data/
src/org/apache/pig/impl/io/ src/org/apache/pig/impl/logicalLayer/schema/
src/org/apache/pig/impl/streaming/ test/org/apache/pig/test/
Modified: incubator/pig/branches/types/test/org/apache/pig/test/TestPigServer.java
URL: http://svn.apache.org/viewvc/incubator/pig/branches/types/test/org/apache/pig/test/TestPigServer.java?rev=707158&r1=707157&r2=707158&view=diff
==============================================================================
--- incubator/pig/branches/types/test/org/apache/pig/test/TestPigServer.java (original)
+++ incubator/pig/branches/types/test/org/apache/pig/test/TestPigServer.java Wed Oct 22 11:37:13 2008
@@ -15,254 +15,544 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-
-package org.apache.pig.test;
-
-import java.io.File;
-import java.io.IOException;
-import java.io.FileOutputStream;
-import java.io.OutputStreamWriter;
-import java.util.List;
-import java.util.Iterator;
-import java.net.URL;
-import java.net.URLClassLoader;
-import java.lang.reflect.Method;
-
+
+package org.apache.pig.test;
+
+import java.io.File;
+import java.io.IOException;
+import java.io.FileOutputStream;
+import java.io.OutputStreamWriter;
+import java.io.BufferedInputStream;
+import java.io.BufferedOutputStream;
+import java.io.BufferedReader;
+import java.io.DataInputStream;
+import java.io.FileInputStream;
+import java.io.InputStream;
+import java.io.InputStreamReader;
+import java.io.PrintStream;
+import java.util.List;
+import java.util.Iterator;
+import java.net.URL;
+import java.net.URLClassLoader;
+import java.lang.reflect.Method;
+
import org.apache.pig.ExecType;
-import org.apache.pig.PigServer;
-
-import org.junit.Test;
-import junit.framework.TestCase;
-
-
-public class TestPigServer extends TestCase {
- private PigServer pig = null;
- MiniCluster cluster = MiniCluster.buildCluster();
-
- private void initPigServer() throws Throwable {
- if (pig == null) {
- pig = new PigServer(ExecType.MAPREDUCE, cluster.getProperties());
- }
- }
-
-
- private final static String FILE_SEPARATOR = System.getProperty("file.separator");
-
- // make sure that name is included or not (depending on flag "included")
- // in the given list of stings
- private static void verifyStringContained(List<URL> list, String name, boolean included) {
- Iterator<URL> iter = list.iterator();
- boolean nameIsSubstring = false;
- int count = 0;
-
- while (iter.hasNext()) {
- if (iter.next().toString().contains(name)) {
- nameIsSubstring = true;
- ++count;
- }
- }
-
- if (included) {
- assertTrue(nameIsSubstring);
- assertTrue(count == 1);
- }
- else {
- assertFalse(nameIsSubstring);
- }
- }
-
- // creates an empty jar file
- private static void createFakeJarFile(String location, String name)
- throws IOException {
- assertFalse((new File(name)).canRead());
-
- assertTrue((new File(location)).mkdirs());
-
- assertTrue((new File(location + FILE_SEPARATOR + name)).
- createNewFile());
- }
-
- // dynamically add more resources to the system class loader
- private static void registerNewResource(String file) throws Exception {
- URL urlToAdd = new File(file).toURI().toURL();
- URLClassLoader sysLoader = (URLClassLoader) ClassLoader.getSystemClassLoader();
- Method addMethod = URLClassLoader.class.
- getDeclaredMethod("addURL",
- new Class[]{URL.class});
- addMethod.setAccessible(true);
- addMethod.invoke(sysLoader, new Object[]{urlToAdd});
- }
-
- private static void executeShellCommand(String cmd) throws Exception {
- Process cmdProc = Runtime.getRuntime().exec(cmd);
-
- cmdProc.waitFor();
-
- assertTrue(cmdProc.exitValue() == 0);
- }
-
- /**
- * The jar file to register is not present
- */
- @Test
- public void testRegisterJarFileNotPresent() throws Throwable {
- // resister a jar file that does not exist
-
- String jarName = "BadFileNameTestJarNotPresent.jar";
-
- // jar name is not present to start with
- initPigServer();
- verifyStringContained(pig.getPigContext().extraJars, jarName, false);
-
- boolean exceptionRaised = false;
- try {
- pig.registerJar(jarName);
- }
- catch (IOException e) {
- exceptionRaised = true;
- }
- assertTrue(exceptionRaised);
- verifyStringContained(pig.getPigContext().extraJars, jarName, false);
- }
-
- /**
- * Jar file to register is not present in the system resources
- * in this case name of jar file is relative to current working dir
- */
- @Test
- public void testRegisterJarLocalDir() throws Throwable {
- String dir1 = "test1_register_jar_local";
- String dir2 = "test2_register_jar_local";
- String jarLocation = dir1 + FILE_SEPARATOR +
- dir2 + FILE_SEPARATOR;
- String jarName = "TestRegisterJarLocal.jar";
-
- initPigServer();
-
- createFakeJarFile(jarLocation, jarName);
-
- verifyStringContained(pig.getPigContext().extraJars, jarName, false);
-
- boolean exceptionRaised = false;
- try {
- pig.registerJar(jarLocation + jarName);
- }
- catch (IOException e) {
- exceptionRaised = true;
- }
- assertFalse(exceptionRaised);
- verifyStringContained(pig.getPigContext().extraJars, jarName, true);
-
- // clean-up
- assertTrue((new File(jarLocation + jarName)).delete());
- (new File(dir1 + FILE_SEPARATOR + dir2)).delete();
- (new File(dir1)).delete();
- }
-
- /**
- * Jar file is located via system resources
- * Test verifies that even with multiple resources matching,
- * only one of them is registered.
- */
- @Test
- public void testRegisterJarFromResources () throws Throwable {
- String dir = "test_register_jar_res_dir";
- String subDir1 = "test_register_jar_res_sub_dir1";
- String subDir2 = "test_register_jar_res_sub_dir2";
- String jarName = "TestRegisterJarFromRes.jar";
- String jarLocation1 = dir + FILE_SEPARATOR + subDir1 + FILE_SEPARATOR;
- String jarLocation2 = dir + FILE_SEPARATOR + subDir2 + FILE_SEPARATOR;
-
- initPigServer();
-
- createFakeJarFile(jarLocation1, jarName);
- createFakeJarFile(jarLocation2, jarName);
-
- verifyStringContained(pig.getPigContext().extraJars, jarName, false);
-
- registerNewResource(jarLocation1);
- registerNewResource(jarLocation2);
-
- boolean exceptionRaised = false;
- try {
- pig.registerJar(jarName);
- }
- catch (IOException e) {
- exceptionRaised = true;
- }
- assertFalse(exceptionRaised);
- verifyStringContained(pig.getPigContext().extraJars, jarName, true);
-
- // clean-up
- assertTrue((new File(jarLocation1 + jarName)).delete());
- assertTrue((new File(jarLocation2 + jarName)).delete());
- (new File(jarLocation1)).delete();
- (new File(jarLocation2)).delete();
- (new File(dir)).delete();
- }
-
- /**
- * Use a resource inside a jar file.
- * Verify that the containing jar file is registered correctly.
- * @throws Exception
- */
- @Test
- public void testRegisterJarResourceInJar() throws Throwable {
- String dir = "test_register_jar_res_in_jar";
- String subDir = "sub_dir";
- String jarName = "TestRegisterJarNonEmpty.jar";
- String className = "TestRegisterJar";
- String javaSrc = "package " + subDir + "; class " + className + " { }";
-
- initPigServer();
-
- // create dirs
- (new File(dir + FILE_SEPARATOR + subDir)).mkdirs();
-
- // generate java file
- FileOutputStream outStream =
- new FileOutputStream(new File(dir + FILE_SEPARATOR + subDir +
- FILE_SEPARATOR + className + ".java"));
-
- OutputStreamWriter outWriter = new OutputStreamWriter(outStream);
- outWriter.write(javaSrc);
- outWriter.close();
-
- // compile
- executeShellCommand("javac " + dir + FILE_SEPARATOR + subDir +
- FILE_SEPARATOR + className + ".java");
-
- // remove src file
- (new File(dir + FILE_SEPARATOR + subDir +
- FILE_SEPARATOR + className + ".java")).delete();
-
- // generate jar file
- executeShellCommand("jar -cf " + dir + FILE_SEPARATOR + jarName + " " +
- "-C " + dir + " " + subDir);
-
- // remove class file and sub_dir
- (new File(dir + FILE_SEPARATOR + subDir +
- FILE_SEPARATOR + className + ".class")).delete();
- (new File(dir + FILE_SEPARATOR + subDir)).delete();
-
- // register resource
- registerNewResource(dir + FILE_SEPARATOR + jarName);
-
- // load the specific resource
- boolean exceptionRaised = false;
- try {
- pig.registerJar("sub_dir/TestRegisterJar.class");
- }
- catch (IOException e) {
- exceptionRaised = true;
- }
-
- // verify proper jar file is located
- assertFalse(exceptionRaised);
- verifyStringContained(pig.getPigContext().extraJars, jarName, true);
-
- // clean up Jar file and test dir
- (new File(dir + FILE_SEPARATOR + jarName)).delete();
- (new File(dir)).delete();
- }
+import org.apache.pig.PigServer;
+
+import org.junit.Before;
+import org.junit.After;
+import org.junit.Test;
+import junit.framework.TestCase;
+
+
+public class TestPigServer extends TestCase {
+ private PigServer pig = null;
+ MiniCluster cluster = MiniCluster.buildCluster();
+ private File stdOutRedirectedFile;
+
+ @Before
+ @Override
+ public void setUp() throws Exception{
+ pig = new PigServer(ExecType.MAPREDUCE, cluster.getProperties());
+ stdOutRedirectedFile = new File("stdout.redirected");
+ // Create file if it does not exist
+ try {
+ if(!stdOutRedirectedFile.createNewFile())
+ fail("Unable to create input files");
+ } catch (IOException e) {
+ fail("Unable to create input files:" + e.getMessage());
+ }
+ }
+
+ @After
+ @Override
+ public void tearDown() throws Exception{
+ pig = null;
+ stdOutRedirectedFile.delete();
+ }
+
+
+ private final static String FILE_SEPARATOR = System.getProperty("file.separator");
+
+ // make sure that name is included or not (depending on flag "included")
+ // in the given list of stings
+ private static void verifyStringContained(List<URL> list, String name, boolean included) {
+ Iterator<URL> iter = list.iterator();
+ boolean nameIsSubstring = false;
+ int count = 0;
+
+ while (iter.hasNext()) {
+ if (iter.next().toString().contains(name)) {
+ nameIsSubstring = true;
+ ++count;
+ }
+ }
+
+ if (included) {
+ assertTrue(nameIsSubstring);
+ assertTrue(count == 1);
+ }
+ else {
+ assertFalse(nameIsSubstring);
+ }
+ }
+
+ // creates an empty jar file
+ private static void createFakeJarFile(String location, String name)
+ throws IOException {
+ assertFalse((new File(name)).canRead());
+
+ assertTrue((new File(location)).mkdirs());
+
+ assertTrue((new File(location + FILE_SEPARATOR + name)).
+ createNewFile());
+ }
+
+ // dynamically add more resources to the system class loader
+ private static void registerNewResource(String file) throws Exception {
+ URL urlToAdd = new File(file).toURI().toURL();
+ URLClassLoader sysLoader = (URLClassLoader) ClassLoader.getSystemClassLoader();
+ Method addMethod = URLClassLoader.class.
+ getDeclaredMethod("addURL",
+ new Class[]{URL.class});
+ addMethod.setAccessible(true);
+ addMethod.invoke(sysLoader, new Object[]{urlToAdd});
+ }
+
+ private static void executeShellCommand(String cmd) throws Exception {
+ Process cmdProc = Runtime.getRuntime().exec(cmd);
+
+ cmdProc.waitFor();
+
+ assertTrue(cmdProc.exitValue() == 0);
+ }
+
+ /**
+ * The jar file to register is not present
+ */
+ @Test
+ public void testRegisterJarFileNotPresent() throws Throwable {
+ // resister a jar file that does not exist
+
+ String jarName = "BadFileNameTestJarNotPresent.jar";
+
+ // jar name is not present to start with
+ verifyStringContained(pig.getPigContext().extraJars, jarName, false);
+
+ boolean exceptionRaised = false;
+ try {
+ pig.registerJar(jarName);
+ }
+ catch (IOException e) {
+ exceptionRaised = true;
+ }
+ assertTrue(exceptionRaised);
+ verifyStringContained(pig.getPigContext().extraJars, jarName, false);
+ }
+
+ /**
+ * Jar file to register is not present in the system resources
+ * in this case name of jar file is relative to current working dir
+ */
+ @Test
+ public void testRegisterJarLocalDir() throws Throwable {
+ String dir1 = "test1_register_jar_local";
+ String dir2 = "test2_register_jar_local";
+ String jarLocation = dir1 + FILE_SEPARATOR +
+ dir2 + FILE_SEPARATOR;
+ String jarName = "TestRegisterJarLocal.jar";
+
+
+ createFakeJarFile(jarLocation, jarName);
+
+ verifyStringContained(pig.getPigContext().extraJars, jarName, false);
+
+ boolean exceptionRaised = false;
+ try {
+ pig.registerJar(jarLocation + jarName);
+ }
+ catch (IOException e) {
+ exceptionRaised = true;
+ }
+ assertFalse(exceptionRaised);
+ verifyStringContained(pig.getPigContext().extraJars, jarName, true);
+
+ // clean-up
+ assertTrue((new File(jarLocation + jarName)).delete());
+ (new File(dir1 + FILE_SEPARATOR + dir2)).delete();
+ (new File(dir1)).delete();
+ }
+
+ /**
+ * Jar file is located via system resources
+ * Test verifies that even with multiple resources matching,
+ * only one of them is registered.
+ */
+ @Test
+ public void testRegisterJarFromResources () throws Throwable {
+ String dir = "test_register_jar_res_dir";
+ String subDir1 = "test_register_jar_res_sub_dir1";
+ String subDir2 = "test_register_jar_res_sub_dir2";
+ String jarName = "TestRegisterJarFromRes.jar";
+ String jarLocation1 = dir + FILE_SEPARATOR + subDir1 + FILE_SEPARATOR;
+ String jarLocation2 = dir + FILE_SEPARATOR + subDir2 + FILE_SEPARATOR;
+
+
+ createFakeJarFile(jarLocation1, jarName);
+ createFakeJarFile(jarLocation2, jarName);
+
+ verifyStringContained(pig.getPigContext().extraJars, jarName, false);
+
+ registerNewResource(jarLocation1);
+ registerNewResource(jarLocation2);
+
+ boolean exceptionRaised = false;
+ try {
+ pig.registerJar(jarName);
+ }
+ catch (IOException e) {
+ exceptionRaised = true;
+ }
+ assertFalse(exceptionRaised);
+ verifyStringContained(pig.getPigContext().extraJars, jarName, true);
+
+ // clean-up
+ assertTrue((new File(jarLocation1 + jarName)).delete());
+ assertTrue((new File(jarLocation2 + jarName)).delete());
+ (new File(jarLocation1)).delete();
+ (new File(jarLocation2)).delete();
+ (new File(dir)).delete();
+ }
+
+ /**
+ * Use a resource inside a jar file.
+ * Verify that the containing jar file is registered correctly.
+ * @throws Exception
+ */
+ @Test
+ public void testRegisterJarResourceInJar() throws Throwable {
+ String dir = "test_register_jar_res_in_jar";
+ String subDir = "sub_dir";
+ String jarName = "TestRegisterJarNonEmpty.jar";
+ String className = "TestRegisterJar";
+ String javaSrc = "package " + subDir + "; class " + className + " { }";
+
+
+ // create dirs
+ (new File(dir + FILE_SEPARATOR + subDir)).mkdirs();
+
+ // generate java file
+ FileOutputStream outStream =
+ new FileOutputStream(new File(dir + FILE_SEPARATOR + subDir +
+ FILE_SEPARATOR + className + ".java"));
+
+ OutputStreamWriter outWriter = new OutputStreamWriter(outStream);
+ outWriter.write(javaSrc);
+ outWriter.close();
+
+ // compile
+ executeShellCommand("javac " + dir + FILE_SEPARATOR + subDir +
+ FILE_SEPARATOR + className + ".java");
+
+ // remove src file
+ (new File(dir + FILE_SEPARATOR + subDir +
+ FILE_SEPARATOR + className + ".java")).delete();
+
+ // generate jar file
+ executeShellCommand("jar -cf " + dir + FILE_SEPARATOR + jarName + " " +
+ "-C " + dir + " " + subDir);
+
+ // remove class file and sub_dir
+ (new File(dir + FILE_SEPARATOR + subDir +
+ FILE_SEPARATOR + className + ".class")).delete();
+ (new File(dir + FILE_SEPARATOR + subDir)).delete();
+
+ // register resource
+ registerNewResource(dir + FILE_SEPARATOR + jarName);
+
+ // load the specific resource
+ boolean exceptionRaised = false;
+ try {
+ pig.registerJar("sub_dir/TestRegisterJar.class");
+ }
+ catch (IOException e) {
+ exceptionRaised = true;
+ }
+
+ // verify proper jar file is located
+ assertFalse(exceptionRaised);
+ verifyStringContained(pig.getPigContext().extraJars, jarName, true);
+
+ // clean up Jar file and test dir
+ (new File(dir + FILE_SEPARATOR + jarName)).delete();
+ (new File(dir)).delete();
+ }
+
+ @Test
+ public void testDescribeLoad() throws Throwable {
+ PrintStream console = System.out;
+ PrintStream out = new PrintStream(new BufferedOutputStream(new FileOutputStream(stdOutRedirectedFile)));
+
+ pig.registerQuery("a = load 'a' as (field1: int, field2: float, field3: chararray );") ;
+ System.setOut(out);
+ pig.dumpSchema("a") ;
+ out.close(); // Remember this!
+ System.setOut(console);
+
+ String s;
+ InputStream fileWithStdOutContents = new DataInputStream( new BufferedInputStream( new FileInputStream(stdOutRedirectedFile)));
+ BufferedReader reader = new BufferedReader(new InputStreamReader(fileWithStdOutContents));
+ while ((s = reader.readLine()) != null) {
+ assertTrue(s.equals("a: {field1: int,field2: float,field3: chararray}") == true);
+ }
+
+ }
+
+ @Test
+ public void testDescribeFilter() throws Throwable {
+ PrintStream console = System.out;
+ PrintStream out = new PrintStream(new BufferedOutputStream(new FileOutputStream(stdOutRedirectedFile)));
+
+ pig.registerQuery("a = load 'a' as (field1: int, field2: float, field3: chararray );") ;
+ pig.registerQuery("b = filter a by field1 > 10;") ;
+ System.setOut(out);
+ pig.dumpSchema("b") ;
+ out.close(); // Remember this!
+ System.setOut(console);
+
+ String s;
+ InputStream fileWithStdOutContents = new DataInputStream( new BufferedInputStream( new FileInputStream(stdOutRedirectedFile)));
+ BufferedReader reader = new BufferedReader(new InputStreamReader(fileWithStdOutContents));
+ while ((s = reader.readLine()) != null) {
+ assertTrue(s.equals("b: {field1: int,field2: float,field3: chararray}") == true);
+ }
+
+ }
+
+ @Test
+ public void testDescribeDistinct() throws Throwable {
+ PrintStream console = System.out;
+ PrintStream out = new PrintStream(new BufferedOutputStream(new FileOutputStream(stdOutRedirectedFile)));
+
+ pig.registerQuery("a = load 'a' as (field1: int, field2: float, field3: chararray );") ;
+ pig.registerQuery("b = distinct a ;") ;
+ System.setOut(out);
+ pig.dumpSchema("b") ;
+ out.close(); // Remember this!
+ System.setOut(console);
+
+ String s;
+ InputStream fileWithStdOutContents = new DataInputStream( new BufferedInputStream( new FileInputStream(stdOutRedirectedFile)));
+ BufferedReader reader = new BufferedReader(new InputStreamReader(fileWithStdOutContents));
+ while ((s = reader.readLine()) != null) {
+ assertTrue(s.equals("b: {field1: int,field2: float,field3: chararray}") == true);
+ }
+
+ }
+
+ @Test
+ public void testDescribeSort() throws Throwable {
+ PrintStream console = System.out;
+ PrintStream out = new PrintStream(new BufferedOutputStream(new FileOutputStream(stdOutRedirectedFile)));
+
+ pig.registerQuery("a = load 'a' as (field1: int, field2: float, field3: chararray );") ;
+ pig.registerQuery("b = order a by * desc;") ;
+ System.setOut(out);
+ pig.dumpSchema("b") ;
+ out.close(); // Remember this!
+ System.setOut(console);
+
+ String s;
+ InputStream fileWithStdOutContents = new DataInputStream( new BufferedInputStream( new FileInputStream(stdOutRedirectedFile)));
+ BufferedReader reader = new BufferedReader(new InputStreamReader(fileWithStdOutContents));
+ while ((s = reader.readLine()) != null) {
+ assertTrue(s.equals("b: {field1: int,field2: float,field3: chararray}") == true);
+ }
+
+ }
+
+ @Test
+ public void testDescribeLimit() throws Throwable {
+ PrintStream console = System.out;
+ PrintStream out = new PrintStream(new BufferedOutputStream(new FileOutputStream(stdOutRedirectedFile)));
+
+ pig.registerQuery("a = load 'a' as (field1: int, field2: float, field3: chararray );") ;
+ pig.registerQuery("b = limit a 10;") ;
+ System.setOut(out);
+ pig.dumpSchema("b") ;
+ out.close(); // Remember this!
+ System.setOut(console);
+
+ String s;
+ InputStream fileWithStdOutContents = new DataInputStream( new BufferedInputStream( new FileInputStream(stdOutRedirectedFile)));
+ BufferedReader reader = new BufferedReader(new InputStreamReader(fileWithStdOutContents));
+ while ((s = reader.readLine()) != null) {
+ assertTrue(s.equals("b: {field1: int,field2: float,field3: chararray}") == true);
+ }
+
+ }
+
+ @Test
+ public void testDescribeForeach() throws Throwable {
+ PrintStream console = System.out;
+ PrintStream out = new PrintStream(new BufferedOutputStream(new FileOutputStream(stdOutRedirectedFile)));
+
+ pig.registerQuery("a = load 'a' as (field1: int, field2: float, field3: chararray );") ;
+ pig.registerQuery("b = foreach a generate field1 + 10;") ;
+ System.setOut(out);
+ pig.dumpSchema("b") ;
+ out.close();
+ System.setOut(console);
+
+ String s;
+ InputStream fileWithStdOutContents = new DataInputStream( new BufferedInputStream( new FileInputStream(stdOutRedirectedFile)));
+ BufferedReader reader = new BufferedReader(new InputStreamReader(fileWithStdOutContents));
+ while ((s = reader.readLine()) != null) {
+ assertTrue(s.equals("b: {int}") == true);
+ }
+
+ }
+
+ @Test
+ public void testDescribeForeachFail() throws Throwable {
+
+ pig.registerQuery("a = load 'a' as (field1: int, field2: float, field3: chararray );") ;
+ pig.registerQuery("b = foreach a generate field1 + 10;") ;
+ try {
+ pig.dumpSchema("c") ;
+ fail("Error expected");
+ } catch (Exception e) {
+ assertTrue(e.getMessage().contains("Unable to describe schema for alias c"));
+ }
+ }
+
+ @Test
+ public void testDescribeForeachNoSchema() throws Throwable {
+ PrintStream console = System.out;
+ PrintStream out = new PrintStream(new BufferedOutputStream(new FileOutputStream(stdOutRedirectedFile)));
+
+ pig.registerQuery("a = load 'a' ;") ;
+ pig.registerQuery("b = foreach a generate *;") ;
+ System.setOut(out);
+ pig.dumpSchema("b") ;
+ out.close();
+ System.setOut(console);
+
+ String s;
+ InputStream fileWithStdOutContents = new DataInputStream( new BufferedInputStream( new FileInputStream(stdOutRedirectedFile)));
+ BufferedReader reader = new BufferedReader(new InputStreamReader(fileWithStdOutContents));
+ while ((s = reader.readLine()) != null) {
+ assertTrue(s.equals("Schema for b unknown.") == true);
+ }
+
+ }
+
+ @Test
+ public void testDescribeCogroup() throws Throwable {
+ PrintStream console = System.out;
+ PrintStream out = new PrintStream(new BufferedOutputStream(new FileOutputStream(stdOutRedirectedFile)));
+
+ pig.registerQuery("a = load 'a' as (field1: int, field2: float, field3: chararray );") ;
+ pig.registerQuery("b = load 'b' as (field4, field5: double, field6: chararray );") ;
+ pig.registerQuery("c = cogroup a by field1, b by field4;") ;
+ System.setOut(out);
+ pig.dumpSchema("c") ;
+ out.close();
+ System.setOut(console);
+
+ String s;
+ InputStream fileWithStdOutContents = new DataInputStream( new BufferedInputStream( new FileInputStream(stdOutRedirectedFile)));
+ BufferedReader reader = new BufferedReader(new InputStreamReader(fileWithStdOutContents));
+ while ((s = reader.readLine()) != null) {
+ assertTrue(s.equals("c: {group: int,a: {field1: int,field2: float,field3: chararray},b: {field4: bytearray,field5: double,field6: chararray}}") == true);
+ }
+
+ }
+
+ @Test
+ public void testDescribeCross() throws Throwable {
+ PrintStream console = System.out;
+ PrintStream out = new PrintStream(new BufferedOutputStream(new FileOutputStream(stdOutRedirectedFile)));
+
+ pig.registerQuery("a = load 'a' as (field1: int, field2: float, field3: chararray );") ;
+ pig.registerQuery("b = load 'b' as (field4, field5: double, field6: chararray );") ;
+ pig.registerQuery("c = cross a, b;") ;
+ System.setOut(out);
+ pig.dumpSchema("c") ;
+ out.close();
+ System.setOut(console);
+
+ String s;
+ InputStream fileWithStdOutContents = new DataInputStream( new BufferedInputStream( new FileInputStream(stdOutRedirectedFile)));
+ BufferedReader reader = new BufferedReader(new InputStreamReader(fileWithStdOutContents));
+ while ((s = reader.readLine()) != null) {
+ assertTrue(s.equals("c: {a::field1: int,a::field2: float,a::field3: chararray,b::field4: bytearray,b::field5: double,b::field6: chararray}") == true);
+ }
+
+ }
+
+ @Test
+ public void testDescribeJoin() throws Throwable {
+ PrintStream console = System.out;
+ PrintStream out = new PrintStream(new BufferedOutputStream(new FileOutputStream(stdOutRedirectedFile)));
+
+ pig.registerQuery("a = load 'a' as (field1: int, field2: float, field3: chararray );") ;
+ pig.registerQuery("b = load 'b' as (field4, field5: double, field6: chararray );") ;
+ pig.registerQuery("c = join a by field1, b by field4;") ;
+ System.setOut(out);
+ pig.dumpSchema("c") ;
+ out.close();
+ System.setOut(console);
+
+ String s;
+ InputStream fileWithStdOutContents = new DataInputStream( new BufferedInputStream( new FileInputStream(stdOutRedirectedFile)));
+ BufferedReader reader = new BufferedReader(new InputStreamReader(fileWithStdOutContents));
+ while ((s = reader.readLine()) != null) {
+ assertTrue(s.equals("c: {a::field1: int,a::field2: float,a::field3: chararray,b::field4: bytearray,b::field5: double,b::field6: chararray}") == true);
+ }
+
+ }
+
+ @Test
+ public void testDescribeUnion() throws Throwable {
+ PrintStream console = System.out;
+ PrintStream out = new PrintStream(new BufferedOutputStream(new FileOutputStream(stdOutRedirectedFile)));
+
+ pig.registerQuery("a = load 'a' as (field1: int, field2: float, field3: chararray );") ;
+ pig.registerQuery("b = load 'b' as (field4, field5: double, field6: chararray );") ;
+ pig.registerQuery("c = union a, b;") ;
+ System.setOut(out);
+ pig.dumpSchema("c") ;
+ out.close();
+ System.setOut(console);
+
+ String s;
+ InputStream fileWithStdOutContents = new DataInputStream( new BufferedInputStream( new FileInputStream(stdOutRedirectedFile)));
+ BufferedReader reader = new BufferedReader(new InputStreamReader(fileWithStdOutContents));
+ while ((s = reader.readLine()) != null) {
+ assertTrue(s.equals("c: {field1: int,field2: double,field3: chararray}") == true);
+ }
+
+ }
+
+ @Test
+ public void testDescribeComplex() throws Throwable {
+ PrintStream console = System.out;
+ PrintStream out = new PrintStream(new BufferedOutputStream(new FileOutputStream(stdOutRedirectedFile)));
+
+ pig.registerQuery("a = load 'a' as (site: chararray, count: int, itemCounts: bag { itemCountsTuple: tuple (type: chararray, typeCount: int, f: float, m: map[]) } ) ;") ;
+ pig.registerQuery("b = foreach a generate site, count, FLATTEN(itemCounts);") ;
+ System.setOut(out);
+ pig.dumpSchema("b") ;
+ out.close();
+ System.setOut(console);
+
+ String s;
+ InputStream fileWithStdOutContents = new DataInputStream( new BufferedInputStream( new FileInputStream(stdOutRedirectedFile)));
+ BufferedReader reader = new BufferedReader(new InputStreamReader(fileWithStdOutContents));
+ while ((s = reader.readLine()) != null) {
+ assertTrue(s.equals("b: {site: chararray,count: int,itemCounts::itemCountsTuple: (type: chararray,typeCount: int,f: float,m: map[ ])}") == true);
+ }
+
+ }
}
Modified: incubator/pig/branches/types/test/org/apache/pig/test/TestPigSplit.java
URL: http://svn.apache.org/viewvc/incubator/pig/branches/types/test/org/apache/pig/test/TestPigSplit.java?rev=707158&r1=707157&r2=707158&view=diff
==============================================================================
--- incubator/pig/branches/types/test/org/apache/pig/test/TestPigSplit.java (original)
+++ incubator/pig/branches/types/test/org/apache/pig/test/TestPigSplit.java Wed Oct 22 11:37:13 2008
@@ -90,7 +90,7 @@
}
pw.close();
- pigServer.registerQuery("a = load 'file:" + f + "';");
+ pigServer.registerQuery("a = load '" + Util.generateURI(f.toString()) + "';");
pigServer.registerQuery("a = filter a by $0 == '1';");
Iterator<Tuple> iter = pigServer.openIterator("a");
Modified: incubator/pig/branches/types/test/org/apache/pig/test/TestSplitStore.java
URL: http://svn.apache.org/viewvc/incubator/pig/branches/types/test/org/apache/pig/test/TestSplitStore.java?rev=707158&r1=707157&r2=707158&view=diff
==============================================================================
--- incubator/pig/branches/types/test/org/apache/pig/test/TestSplitStore.java (original)
+++ incubator/pig/branches/types/test/org/apache/pig/test/TestSplitStore.java Wed Oct 22 11:37:13 2008
@@ -45,7 +45,7 @@
@Test
public void test1() throws Exception{
- pig.registerQuery("A = LOAD 'file:" + tmpFile + "';");
+ pig.registerQuery("A = LOAD '" + Util.generateURI(tmpFile.toString()) + "';");
pig.registerQuery("Split A into A1 if $0<=10, A2 if $0>10;");
pig.store("A1", "'" + FileLocalizer.getTemporaryPath(null, pigContext) + "'");
pig.store("A2", "'" + FileLocalizer.getTemporaryPath(null, pigContext) + "'");
@@ -53,7 +53,7 @@
@Test
public void test2() throws Exception{
- pig.registerQuery("A = LOAD 'file:" + tmpFile + "';");
+ pig.registerQuery("A = LOAD '" + Util.generateURI(tmpFile.toString()) + "';");
pig.registerQuery("Split A into A1 if $0<=10, A2 if $0>10;");
pig.openIterator("A1");
pig.store("A2", "'" + FileLocalizer.getTemporaryPath(null, pigContext) + "'");
@@ -61,7 +61,7 @@
@Test
public void test3() throws Exception{
- pig.registerQuery("A = LOAD 'file:" + tmpFile + "';");
+ pig.registerQuery("A = LOAD '" + Util.generateURI(tmpFile.toString()) + "';");
pig.registerQuery("Split A into A1 if $0<=10, A2 if $0>10;");
pig.openIterator("A2");
pig.store("A1", "'" + FileLocalizer.getTemporaryPath(null, pigContext) + "'");
@@ -69,7 +69,7 @@
@Test
public void test4() throws Exception{
- pig.registerQuery("A = LOAD 'file:" + tmpFile + "';");
+ pig.registerQuery("A = LOAD '" + Util.generateURI(tmpFile.toString()) + "';");
pig.registerQuery("Split A into A1 if $0<=10, A2 if $0>10;");
pig.store("A1", "'" + FileLocalizer.getTemporaryPath(null, pigContext) + "'");
pig.openIterator("A2");
@@ -77,7 +77,7 @@
@Test
public void test5() throws Exception{
- pig.registerQuery("A = LOAD 'file:" + tmpFile + "';");
+ pig.registerQuery("A = LOAD '" + Util.generateURI(tmpFile.toString()) + "';");
pig.registerQuery("Split A into A1 if $0<=10, A2 if $0>10;");
pig.store("A2", "'" + FileLocalizer.getTemporaryPath(null, pigContext) + "'");
pig.openIterator("A1");
@@ -85,7 +85,7 @@
@Test
public void test6() throws Exception{
- pig.registerQuery("A = LOAD 'file:" + tmpFile + "';");
+ pig.registerQuery("A = LOAD '" + Util.generateURI(tmpFile.toString()) + "';");
pig.registerQuery("Split A into A1 if $0<=10, A2 if $0>10;");
pig.openIterator("A1");
pig.registerQuery("Store A2 into '" + FileLocalizer.getTemporaryPath(null, pigContext) + "';");
@@ -93,7 +93,7 @@
@Test
public void test7() throws Exception{
- pig.registerQuery("A = LOAD 'file:" + tmpFile + "';");
+ pig.registerQuery("A = LOAD '" + Util.generateURI(tmpFile.toString()) + "';");
pig.registerQuery("Split A into A1 if $0<=10, A2 if $0>10;");
pig.openIterator("A2");
pig.registerQuery("Store A1 into '" + FileLocalizer.getTemporaryPath(null, pigContext) + "';");
@@ -101,7 +101,7 @@
@Test
public void test8() throws Exception{
- pig.registerQuery("A = LOAD 'file:" + tmpFile + "';");
+ pig.registerQuery("A = LOAD '" + Util.generateURI(tmpFile.toString()) + "';");
pig.registerQuery("Split A into A1 if $0<=10, A2 if $0>10;");
pig.registerQuery("Store A1 into '" + FileLocalizer.getTemporaryPath(null, pigContext) + "';");
pig.openIterator("A2");
@@ -109,7 +109,7 @@
@Test
public void test9() throws Exception{
- pig.registerQuery("A = LOAD 'file:" + tmpFile + "';");
+ pig.registerQuery("A = LOAD '" + Util.generateURI(tmpFile.toString()) + "';");
pig.registerQuery("Split A into A1 if $0<=10, A2 if $0>10;");
pig.registerQuery("Store A2 into '" + FileLocalizer.getTemporaryPath(null, pigContext) + "';");
pig.openIterator("A1");
Modified: incubator/pig/branches/types/test/org/apache/pig/test/TestStore.java
URL: http://svn.apache.org/viewvc/incubator/pig/branches/types/test/org/apache/pig/test/TestStore.java?rev=707158&r1=707157&r2=707158&view=diff
==============================================================================
--- incubator/pig/branches/types/test/org/apache/pig/test/TestStore.java (original)
+++ incubator/pig/branches/types/test/org/apache/pig/test/TestStore.java Wed Oct 22 11:37:13 2008
@@ -58,7 +58,7 @@
@Before
public void setUp() throws Exception {
st = GenPhyOp.topStoreOp();
- fSpec = new FileSpec("file:////tmp/storeTest.txt",
+ fSpec = new FileSpec("file:/tmp/storeTest.txt",
new FuncSpec(PigStorage.class.getName(), new String[]{":"}));
st.setSFile(fSpec);
pc = new PigContext();
Modified: incubator/pig/branches/types/test/org/apache/pig/test/Util.java
URL: http://svn.apache.org/viewvc/incubator/pig/branches/types/test/org/apache/pig/test/Util.java?rev=707158&r1=707157&r2=707158&view=diff
==============================================================================
--- incubator/pig/branches/types/test/org/apache/pig/test/Util.java (original)
+++ incubator/pig/branches/types/test/org/apache/pig/test/Util.java Wed Oct 22 11:37:13 2008
@@ -219,4 +219,19 @@
String replacement = quoteReplacement("\\\\");
return str.replaceAll(regex, replacement);
}
+
+ /**
+ * Helper method to construct URI for local file system. For unix, it will
+ * put "file:" in the front of the path; For Windows, it will put "file:/" in
+ * front of the path, and also call encodeEscape to replace "\" with "\\"
+ *
+ * @param str absolute path (under cygwin, should be a windows style path)
+ * @return The resulting string
+ */
+ public static String generateURI(String path)
+ {
+ if (System.getProperty("os.name").toUpperCase().startsWith("WINDOWS"))
+ return "file:/"+encodeEscape(path);
+ return "file:"+path;
+ }
}