You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@pig.apache.org by ga...@apache.org on 2009/11/18 22:12:24 UTC
svn commit: r881937 [7/7] - in /hadoop/pig/trunk/contrib/zebra: ./
src/java/org/apache/hadoop/zebra/io/ src/java/org/apache/hadoop/zebra/mapred/
src/java/org/apache/hadoop/zebra/schema/
src/java/org/apache/hadoop/zebra/tfile/ src/java/org/apache/hadoop...
Added: hadoop/pig/trunk/contrib/zebra/src/test/org/apache/hadoop/zebra/types/TestStorageGrammar.java
URL: http://svn.apache.org/viewvc/hadoop/pig/trunk/contrib/zebra/src/test/org/apache/hadoop/zebra/types/TestStorageGrammar.java?rev=881937&view=auto
==============================================================================
--- hadoop/pig/trunk/contrib/zebra/src/test/org/apache/hadoop/zebra/types/TestStorageGrammar.java (added)
+++ hadoop/pig/trunk/contrib/zebra/src/test/org/apache/hadoop/zebra/types/TestStorageGrammar.java Wed Nov 18 21:12:22 2009
@@ -0,0 +1,833 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership. The ASF
+ * licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations under
+ * the License.
+ */
+package org.apache.hadoop.zebra.types;
+
+import java.io.BufferedReader;
+import java.io.ByteArrayOutputStream;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.InputStreamReader;
+import java.io.PrintStream;
+import java.util.HashMap;
+import java.util.Map;
+import javax.security.auth.login.LoginException;
+
+import junit.framework.Assert;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.LocalFileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.RawLocalFileSystem;
+import org.apache.hadoop.io.BytesWritable;
+import org.apache.hadoop.zebra.io.BasicTable;
+import org.apache.hadoop.zebra.io.TableInserter;
+import org.apache.hadoop.zebra.parser.ParseException;
+import org.apache.hadoop.zebra.schema.Schema;
+import org.apache.hadoop.zebra.types.TypesUtils;
+import org.apache.pig.data.DataBag;
+import org.apache.pig.data.DataByteArray;
+import org.apache.pig.data.Tuple;
+import org.junit.AfterClass;
+import org.junit.BeforeClass;
+import org.junit.Test;
+
+/**
+ *
+ * Test projections on complicated column types.
+ *
+ */
+public class TestStorageGrammar {
+
+ final static String STR_SCHEMA = "s1:bool, s2:int, s3:long, s4:float, s5:string, s6:bytes, r1:record(f1:int, f2:long), r2:record(r3:record(f3:float, f4)), m1:map(string),m2:map(map(int)), c:collection(f13:double, f14:float, f15:bytes),s7:string, s8:string, s9:string, s10:string, s11:string, s12:string, s13:string, s14:string, s15:string, s16:string, s17:string, s18:string, s19:string, s20:string, s21:string, s22:string, s23:string";
+ static String STR_STORAGE = null;
+ final private static Configuration conf = new Configuration();
+ private static Path path;
+ private static FileSystem fs;
+ private static String user;
+ private static String group;
+ private static String defaultUser;
+
+ static{
+ try{
+ String command = "whoami";
+ Process process = new ProcessBuilder(command).start();
+ InputStream is = process.getInputStream();
+ InputStreamReader isr = new InputStreamReader(is);
+ BufferedReader br = new BufferedReader(isr);
+ System.out.printf("Output of running %s is:",
+ command);
+ String line;
+ while ((line = br.readLine()) != null) {
+ System.out.println("default user0:" +line);
+ defaultUser = line;
+
+ }
+ }catch(Exception e){
+ e.printStackTrace();
+ }
+ }
+
+ @BeforeClass
+ public static void setUpOnce() throws IOException, LoginException {
+ if (System.getProperty("user") == null) {
+ System.setProperty("user", defaultUser);
+ }
+ user = System.getProperty("user");
+ if (System.getProperty("group") == null) {
+ System.setProperty("group", "users");
+ }
+ group = System.getProperty("group");
+ System.out.println("user:" + user + " group: " + group);
+ STR_STORAGE = "[s1, s2] COMPRESS BY gz SECURE BY user:"
+ + user
+ + " group:"
+ + group
+ + " perm:777 SERIALIZE BY pig; [m1#{a}] SERIALIZE BY pig COMPRESS BY gz SECURE BY user:"
+ + user
+ + " group:"
+ + group
+ + " perm:777 ; [r1.f1] SECURE BY user:"
+ + user
+ + " group:"
+ + group
+ + " perm:777 SERIALIZE BY pig COMPRESS BY gz ; [s3, s4, r2.r3.f3] SERIALIZE BY pig SECURE BY user:"
+ + user
+ + " group: "
+ + group
+ + " perm:777 COMPRESS BY gz ; [s5, s6, m2#{x|y}] compREss by gz secURe by user:"
+ + user
+ + " group:"
+ + group
+ + " perm:777 SerialIZE BY pig; [r1.f2, m1#{b}]; [r2.r3.f4, m2#{z}] SERIALIZE BY avro;[s7,s8] SERIALIZE BY AVRO;[s9,s10] COMPRESS BY gz ";
+ System.out.println("storage: " + STR_STORAGE);
+ conf.setInt("table.output.tfile.minBlock.size", 64 * 1024);
+ conf.setInt("table.input.split.minSize", 64 * 1024);
+ conf.set("table.output.tfile.compression", "none");
+ // String STR_STORAGE1 =
+ // "[s1, s2] COMPRESS BY gz SECURE BY user:"+USER1.getUserName()+" group:"+USER1.getGroupNames()[0]+" perm:777 SERIALIZE BY pig";
+ RawLocalFileSystem rawLFS = new RawLocalFileSystem();
+ fs = new LocalFileSystem(rawLFS);
+ path = new Path(fs.getWorkingDirectory(), "TestStorageGrammar");
+ System.out.println("path: " + path.toString());
+ // Process p = Runtime.getRuntime().exec("rm -rf "+path.toString()+"*");
+
+ fs = path.getFileSystem(conf);
+ // drop any previous tables
+ BasicTable.drop(path, conf);
+
+ BasicTable.Writer writer = new BasicTable.Writer(path, STR_SCHEMA,
+ STR_STORAGE, conf);
+
+ writer.finish();
+
+ Schema schema = writer.getSchema();
+ Tuple tuple = TypesUtils.createTuple(schema);
+ BasicTable.Writer writer1 = new BasicTable.Writer(path, conf);
+ int part = 0;
+ TableInserter inserter = null;
+ try {
+ inserter = writer1.getInserter("part" + part, true);
+ } catch (Exception e) {
+ e.printStackTrace();
+ }
+ TypesUtils.resetTuple(tuple);
+ Tuple tupRecord1 = null;
+ try {
+ tupRecord1 = TypesUtils.createTuple(schema.getColumnSchema("r1")
+ .getSchema());
+ } catch (ParseException e) {
+ e.printStackTrace();
+ }
+
+ Tuple tupRecord2 = null;
+ try {
+ tupRecord2 = TypesUtils.createTuple(schema.getColumnSchema("r2")
+ .getSchema());
+ } catch (ParseException e) {
+ e.printStackTrace();
+ }
+
+ Tuple tupRecord3 = null;
+ try {
+ tupRecord3 = TypesUtils.createTuple(new Schema("f3:float, f4"));
+ } catch (ParseException e) {
+ e.printStackTrace();
+ }
+
+ // row 1
+ tuple.set(0, true); // bool
+ tuple.set(1, 1); // int
+ tuple.set(2, 1001L); // long
+ tuple.set(3, 1.1); // float
+ tuple.set(4, "hello world 1"); // string
+ tuple.set(5, new DataByteArray("hello byte 1")); // byte
+
+ // r1:record(f1:int, f2:long
+ tupRecord1.set(0, 1);
+ tupRecord1.set(1, 1001L);
+ tuple.set(6, tupRecord1);
+
+ // r2:record(r3:record(f3:float, f4))
+ tupRecord2.set(0, tupRecord3);
+ tupRecord3.set(0, 1.3);
+ tupRecord3.set(1, new DataByteArray("r3 row 1 byte array "));
+ tuple.set(7, tupRecord2);
+
+ // m1:map(string)
+ Map<String, String> m1 = new HashMap<String, String>();
+ m1.put("a", "A");
+ m1.put("b", "B");
+ m1.put("c", "C");
+ tuple.set(8, m1);
+
+ // m2:map(map(int))
+ HashMap<String, Map> m2 = new HashMap<String, Map>();
+ Map<String, Integer> m3 = new HashMap<String, Integer>();
+ m3.put("m311", 311);
+ m3.put("m321", 321);
+ m3.put("m331", 331);
+ Map<String, Integer> m4 = new HashMap<String, Integer>();
+ m4.put("m411", 411);
+ m4.put("m421", 421);
+ m4.put("m431", 431);
+ m2.put("x", m3);
+ m2.put("y", m4);
+ tuple.set(9, m2);
+
+ // c:collection(f13:double, f14:float, f15:bytes)
+ DataBag bagColl = TypesUtils.createBag();
+ Schema schColl = schema.getColumn(10).getSchema();
+ Tuple tupColl1 = TypesUtils.createTuple(schColl);
+ Tuple tupColl2 = TypesUtils.createTuple(schColl);
+ byte[] abs1 = new byte[3];
+ byte[] abs2 = new byte[4];
+ tupColl1.set(0, 3.1415926);
+ tupColl1.set(1, 1.6);
+ abs1[0] = 11;
+ abs1[1] = 12;
+ abs1[2] = 13;
+ tupColl1.set(2, new DataByteArray(abs1));
+ bagColl.add(tupColl1);
+ tupColl2.set(0, 123.456789);
+ tupColl2.set(1, 100);
+ abs2[0] = 21;
+ abs2[1] = 22;
+ abs2[2] = 23;
+ abs2[3] = 24;
+ tupColl2.set(2, new DataByteArray(abs2));
+ bagColl.add(tupColl2);
+ tuple.set(10, bagColl);
+
+ // set s7 to s23
+ for (int i = 7; i <= 23; i++) {
+ tuple.set(i + 4, "s" + "i" + ", line1");
+ }
+
+ int row = 0;
+ inserter.insert(new BytesWritable(String.format("k%d%d", part + 1, row + 1)
+ .getBytes()), tuple);
+
+ // row 2
+ row++;
+ TypesUtils.resetTuple(tuple);
+ TypesUtils.resetTuple(tupRecord1);
+ TypesUtils.resetTuple(tupRecord2);
+ TypesUtils.resetTuple(tupRecord3);
+ m1.clear();
+ m2.clear();
+ m3.clear();
+ m4.clear();
+ tuple.set(0, false);
+ tuple.set(1, 2); // int
+ tuple.set(2, 1002L); // long
+ tuple.set(3, 3.1); // float
+ tuple.set(4, "hello world 2"); // string
+ tuple.set(5, new DataByteArray("hello byte 2")); // byte
+
+ // r1:record(f1:int, f2:long
+ tupRecord1.set(0, 2);
+ tupRecord1.set(1, 1002L);
+ tuple.set(6, tupRecord1);
+
+ // r2:record(r3:record(f3:float, f4))
+ tupRecord2.set(0, tupRecord3);
+ tupRecord3.set(0, 2.3);
+ tupRecord3.set(1, new DataByteArray("r3 row2 byte array"));
+ tuple.set(7, tupRecord2);
+
+ // m1:map(string)
+ m1.put("a2", "A2");
+ m1.put("b2", "B2");
+ m1.put("c2", "C2");
+ tuple.set(8, m1);
+
+ // m2:map(map(int))
+ m3.put("m321", 321);
+ m3.put("m322", 322);
+ m3.put("m323", 323);
+ m2.put("z", m3);
+ tuple.set(9, m2);
+
+ // c:collection(f13:double, f14:float, f15:bytes)
+ bagColl.clear();
+ TypesUtils.resetTuple(tupColl1);
+ TypesUtils.resetTuple(tupColl2);
+ tupColl1.set(0, 7654.321);
+ tupColl1.set(1, 0.0001);
+ abs1[0] = 31;
+ abs1[1] = 32;
+ abs1[2] = 33;
+ tupColl1.set(2, new DataByteArray(abs1));
+ bagColl.add(tupColl1);
+ tupColl2.set(0, 0.123456789);
+ tupColl2.set(1, 0.3333);
+ abs2[0] = 41;
+ abs2[1] = 42;
+ abs2[2] = 43;
+ abs2[3] = 44;
+ tupColl2.set(2, new DataByteArray(abs2));
+ bagColl.add(tupColl2);
+ tuple.set(10, bagColl);
+ // set s7 to s23
+ for (int i = 7; i <= 23; i++) {
+ tuple.set(i + 4, "s" + "i" + ", line2");
+ }
+ inserter.insert(new BytesWritable(String.format("k%d%d", part + 1, row + 1)
+ .getBytes()), tuple);
+
+ inserter.close();
+ writer1.finish();
+
+ writer.close();
+ }
+
+ @AfterClass
+ public static void tearDownOnce() throws IOException {
+ }
+
+ //@Test
+ /*
+ * group1 is user specific
+ */
+ public void test1() throws IOException, ParseException {
+
+ String schema = "s1:string, s2:string";
+ String storage = "[s1, s2]COMPRESS BY gz SECURE BY user:user1 group:users perm:744 SERIALIZE BY pig";
+ RawLocalFileSystem rawLFS = new RawLocalFileSystem();
+ fs = new LocalFileSystem(rawLFS);
+ Path path1 = new Path(path.toString() + "1");
+ Runtime.getRuntime().exec("rm -rf " + path1.toString());
+
+ fs = path.getFileSystem(conf);
+ BasicTable.Writer writer = new BasicTable.Writer(path1, schema, storage,
+ conf);
+ writer.finish();
+ ByteArrayOutputStream bos = new ByteArrayOutputStream();
+ PrintStream ps = new PrintStream(bos);
+ System.out.println("start dumpinfo ===========");
+ BasicTable.dumpInfo(path1.toString(), ps, conf);
+
+ Assert.assertEquals(true, bos.toString().contains("Serializer: pig"));
+ Assert.assertEquals(true, bos.toString().contains("Compressor: gz"));
+ Assert.assertEquals(true, bos.toString().contains(
+ "Schema : s1:string,s2:string"));
+ }
+
+ @Test
+ public void test2() throws IOException, ParseException {
+ String schema = "m1:map(string),m2:map(map(int))";
+ String storage = "[m1#{a}] SERIALIZE BY pig COMPRESS BY gz SECURE BY user:"
+ + user + " group:" + group + " perm:770 ";
+ Path path1 = new Path(path.toString() + "2");
+ BasicTable.Writer writer = null;
+ try {
+ writer = new BasicTable.Writer(path1, schema, storage, conf);
+ } catch (Exception e) {
+ e.printStackTrace();
+ }
+ writer.finish();
+ ByteArrayOutputStream bos = new ByteArrayOutputStream();
+ PrintStream ps = new PrintStream(bos);
+ BasicTable.dumpInfo(path1.toString(), ps, conf);
+ System.out.println("start dumpinfo ===========\n" + bos.toString());
+ Assert.assertEquals(true, bos.toString().contains("Serializer: pig"));
+ Assert.assertEquals(true, bos.toString().contains("Compressor: gz"));
+ Assert.assertEquals(true, bos.toString().contains(
+ "Schema : m1:map(string),m2:map(map(int))"));
+ }
+
+ @Test
+ public void test3() throws IOException, ParseException {
+ String schema = "r1:record(f1:int, f2:long)";
+ String storage = "[r1.f1] SECURE BY user:" + user + " group:" + group
+ + " perm:777 SERIALIZE BY pig COMPRESS BY gz";
+ Path path1 = new Path(path.toString() + "3");
+ BasicTable.Writer writer = null;
+ try {
+ writer = new BasicTable.Writer(path1, schema, storage, conf);
+ } catch (Exception e) {
+ e.printStackTrace();
+ }
+ writer.finish();
+ ByteArrayOutputStream bos = new ByteArrayOutputStream();
+ PrintStream ps = new PrintStream(bos);
+ BasicTable.dumpInfo(path1.toString(), ps, conf);
+ System.out.println("start dumpinfo ===========\n" + bos.toString());
+ Assert.assertEquals(true, bos.toString().contains("Serializer: pig"));
+ Assert.assertEquals(true, bos.toString().contains("Compressor: gz"));
+ // Assert.assertEquals(true,bos.toString().contains("Schema : r1:record(f1:int, f2:long)"));
+ }
+
+ @Test
+ public void test4() throws IOException, ParseException {
+ String schema = "s3:string, s4:string, r2:record(r3:record(f3:float, f4)), c:collection(f13:double, f14:float, f15:bytes)";
+ String storage = "[s3, s4, r2.r3.f3, c] SERIALIZE BY pig SECURE BY user:"
+ + user + " group:" + group + " perm:777 COMPRESS BY gz";
+ Path path1 = new Path(path.toString() + "4");
+ BasicTable.Writer writer = null;
+ try {
+ writer = new BasicTable.Writer(path1, schema, storage, conf);
+ } catch (Exception e) {
+ e.printStackTrace();
+ }
+ writer.finish();
+ ByteArrayOutputStream bos = new ByteArrayOutputStream();
+ PrintStream ps = new PrintStream(bos);
+ BasicTable.dumpInfo(path1.toString(), ps, conf);
+ System.out.println("start dumpinfo ===========\n" + bos.toString());
+ Assert.assertEquals(true, bos.toString().contains("Serializer: pig"));
+ Assert.assertEquals(true, bos.toString().contains("Compressor: gz"));
+ // Assert.assertEquals(true,bos.toString().contains("s3:string,s4:string,r2:record(r3:record(f3:float, f4)),c:collection(f13:double,f14:float,f15:bytes)"));
+ }
+
+ @Test
+ public void test5() throws IOException, ParseException {
+ String schema = "s5:string, s6:string, m2:map(map(int))";
+ String storage = "[s5, s6, m2#{x|y}] compREss by gz secURe by user:" + user
+ + " group:" + group + " perm:777 SerialIZE BY pig";
+ Path path1 = new Path(path.toString() + "5");
+ BasicTable.Writer writer = null;
+ try {
+ writer = new BasicTable.Writer(path1, schema, storage, conf);
+ } catch (Exception e) {
+ e.printStackTrace();
+ }
+ writer.finish();
+ ByteArrayOutputStream bos = new ByteArrayOutputStream();
+ PrintStream ps = new PrintStream(bos);
+ BasicTable.dumpInfo(path1.toString(), ps, conf);
+ System.out.println("start dumpinfo ===========\n" + bos.toString());
+ Assert.assertEquals(true, bos.toString().contains("Serializer: pig"));
+ // Assert.assertEquals(true,bos.toString().contains("Compressor: gz"));
+ Assert.assertEquals(true, bos.toString().contains(
+ "Schema : s5:string,s6:string,m2:map(map(int))"));
+ }
+
+ @Test
+ public void test6() throws IOException, ParseException {
+ String schema = "r1:record(f1:int, f2:long),m1:map(string)";
+ String storage = "[r1.f2, m1#{b}]";
+ Path path1 = new Path(path.toString() + "6");
+ BasicTable.Writer writer = null;
+ try {
+ writer = new BasicTable.Writer(path1, schema, storage, conf);
+ } catch (Exception e) {
+ e.printStackTrace();
+ }
+ writer.finish();
+ ByteArrayOutputStream bos = new ByteArrayOutputStream();
+ PrintStream ps = new PrintStream(bos);
+ BasicTable.dumpInfo(path1.toString(), ps, conf);
+ System.out.println("start dumpinfo ===========\n" + bos.toString());
+ Assert.assertEquals(true, bos.toString().contains("Serializer: pig"));
+ // Assert.assertEquals(true,bos.toString().contains("Compressor: gz"));
+ // Assert.assertEquals(true,bos.toString().contains("Schema : r1:record(f1:int, f2:long),m1:map(string)"));
+ }
+
+ @Test
+ public void test7() throws IOException, ParseException {
+ System.out.println("========7777");
+ String schema = "r2:record(r3:record(f3:float, f4)),m2:map(map(int))";
+ String storage = "[r2.r3.f4, m2#{z}] SERIALIZE BY avro";
+ Path path1 = new Path(path.toString() + "7");
+ BasicTable.Writer writer = null;
+ try {
+ writer = new BasicTable.Writer(path1, schema, storage, conf);
+ } catch (Exception e) {
+ e.printStackTrace();
+ }
+ writer.finish();
+ ByteArrayOutputStream bos = new ByteArrayOutputStream();
+ PrintStream ps = new PrintStream(bos);
+ BasicTable.dumpInfo(path1.toString(), ps, conf);
+ System.out.println("start dumpinfo ===========\n" + bos.toString());
+ Assert.assertEquals(true, bos.toString().contains("Serializer: avro"));
+ Assert.assertEquals(true, bos.toString().contains("Compressor: gz"));
+ // Assert.assertEquals(true,bos.toString().contains("Schema : r2:record(r3:record(f3:float, f4)),m2:map(map(int))"));
+ }
+
+ @Test
+ public void test8() throws IOException, ParseException {
+ String schema = "s7:string, s8:string";
+ String storage = "[s7,s8] SERIALIZE BY AvRO";
+ Path path1 = new Path(path.toString() + "8");
+ BasicTable.Writer writer = null;
+ try {
+ writer = new BasicTable.Writer(path1, schema, storage, conf);
+ } catch (Exception e) {
+ e.printStackTrace();
+ }
+ writer.finish();
+ ByteArrayOutputStream bos = new ByteArrayOutputStream();
+ PrintStream ps = new PrintStream(bos);
+ BasicTable.dumpInfo(path1.toString(), ps, conf);
+ System.out.println("start dumpinfo ===========\n" + bos.toString());
+ Assert.assertEquals(true, bos.toString().contains("Serializer: AvRO"));
+ Assert.assertEquals(true, bos.toString().contains("Compressor: gz"));
+ Assert.assertEquals(true, bos.toString().contains(
+ "Schema : s7:string,s8:string"));
+ }
+
+ @Test(expected = IOException.class)
+ public void test9() throws IOException, ParseException {
+ String schema = "some1:string, some2:string";
+ String storage = "[some1,some2] SERIALIZE BY something";
+ Path path1 = new Path(path.toString() + "9");
+ BasicTable.Writer writer = null;
+
+ writer = new BasicTable.Writer(path1, schema, storage, conf);
+ Assert.fail("should throw exception, none defined serializer");
+ writer.finish();
+ }
+
+ @Test(expected = IOException.class)
+ public void test10() throws IOException, ParseException {
+ String schema = "some1:string, some2:string";
+ String storage = "[some1,some2] COMPRESS BY gz SERIALLLLIZE BY pig";
+ Path path1 = new Path(path.toString() + "10");
+ BasicTable.Writer writer = null;
+
+ writer = new BasicTable.Writer(path1, schema, storage, conf);
+ Assert.fail("should throw exception, none defined serializer");
+ writer.finish();
+ }
+
+ @Test(expected = IOException.class)
+ public void test11() throws IOException, ParseException {
+ String schema = "some1:string, some2:string";
+ String storage = "[some1,some2] COMPREEEEESS BY gz SERIALIZE BY pig";
+ Path path1 = new Path(path.toString() + "11");
+ BasicTable.Writer writer = null;
+
+ writer = new BasicTable.Writer(path1, schema, storage, conf);
+ Assert.fail("should throw exception, none defined serializer");
+ writer.finish();
+ }
+
+ @Test(expected = IOException.class)
+ public void test12() throws IOException, ParseException {
+ String schema = "some1:string, some2:string";
+ String storage = "[some1,some2] COMPRESS BY gz SECURrrrrrE BY user:user1 group:grop1 perm:760 SERIALIZE BY pig";
+ Path path1 = new Path(path.toString() + "12");
+ BasicTable.Writer writer = null;
+
+ writer = new BasicTable.Writer(path1, schema, storage, conf);
+ Assert.fail("should throw exception, none defined serializer");
+ writer.finish();
+ }
+
+ @Test(expected = IOException.class)
+ public void test13() throws IOException, ParseException {
+ String schema = "some1:string, some2:string";
+ String storage = "[some1,some2] COMPRESS gz SECURE BY user:user1 group:users perm:760 SERIALIZE BY pig";
+ Path path1 = new Path(path.toString() + "13");
+ BasicTable.Writer writer = null;
+
+ writer = new BasicTable.Writer(path1, schema, storage, conf);
+ Assert.fail("should throw exception, none defined serializer");
+ writer.finish();
+ }
+
+ @Test(expected = IOException.class)
+ public void test14() throws IOException, ParseException {
+ String schema = "some1:string, some2:string";
+ String storage = "[some1,some2] COMPRESS BY gz SECURE BY usssser:user1 group:users perm:760 SERIALIZE BY pig";
+ Path path1 = new Path(path.toString() + "14");
+ BasicTable.Writer writer = null;
+ writer = new BasicTable.Writer(path1, schema, storage, conf);
+ Assert.fail("should throw exception, none defined serializer");
+ writer.finish();
+ }
+
+ @Test(expected = IOException.class)
+ public void test15() throws IOException, ParseException {
+ String schema = "some1:string, some2:string";
+ String storage = "[some1,some2] COMPRESS BY gz SECURE BY user:user1 grouuuup:group1 perm:760 SERIALIZE BY pig";
+ Path path1 = new Path(path.toString() + "15");
+ BasicTable.Writer writer = null;
+ writer = new BasicTable.Writer(path1, schema, storage, conf);
+ Assert.fail("should throw exception, none defined serializer");
+ writer.finish();
+ }
+
+ @Test(expected = IOException.class)
+ public void test16() throws IOException, ParseException {
+ String schema = "some1:string, some2:string";
+ String storage = "[some1,some2] COMPRESS BY gz SECURE BY user:user1 group:users perrrrrm:760 SERIALIZE BY pig";
+ Path path1 = new Path(path.toString() + "16");
+ BasicTable.Writer writer = null;
+ writer = new BasicTable.Writer(path1, schema, storage, conf);
+ Assert.fail("should throw exception, none defined serializer");
+ writer.finish();
+ }
+
+ @Test
+ public void test17() throws IOException, ParseException {
+ String schema = "s9:string, s10:string";
+ String storage = "[s9,s10] COMPRESS BY lzo";
+
+ Path path1 = new Path(path.toString() + "17");
+ Runtime.getRuntime().exec("rm -rf " + path1.toString());
+ fs = path.getFileSystem(conf);
+ BasicTable.Writer writer = new BasicTable.Writer(path1, schema, storage,
+ conf);
+ writer.finish();
+ ByteArrayOutputStream bos = new ByteArrayOutputStream();
+ PrintStream ps = new PrintStream(bos);
+ System.out.println("start dumpinfo 17 ===========");
+ BasicTable.dumpInfo(path1.toString(), ps, conf);
+ System.out.println("bos: " + bos.toString());
+ Assert.assertEquals(true, bos.toString().contains("Serializer: pig"));
+ Assert.assertEquals(true, bos.toString().contains("Compressor: lzo"));
+ Assert.assertEquals(true, bos.toString().contains(
+ "Schema : s9:string,s10:string"));
+ }
+
+ //@Test
+ public void test18() throws IOException, ParseException {
+ String schema = "s9:string, s10:string";
+ String storage = "[s9,s10] COMPRESS BY gZ";
+
+ Path path1 = new Path(path.toString() + "18");
+ Runtime.getRuntime().exec("rm -rf " + path1.toString());
+ fs = path.getFileSystem(conf);
+ BasicTable.Writer writer = new BasicTable.Writer(path1, schema, storage,
+ conf);
+ writer.finish();
+ ByteArrayOutputStream bos = new ByteArrayOutputStream();
+ PrintStream ps = new PrintStream(bos);
+ System.out.println("start dumpinfo 18===========" + bos.toString());
+ BasicTable.dumpInfo(path1.toString(), ps, conf);
+ System.out.println("bos: " + bos.toString());
+
+ Assert.assertEquals(true, bos.toString().contains("Serializer: pig"));
+
+ Assert.assertEquals(true, bos.toString().contains("Compressor: gz"));
+ Assert.assertEquals(true, bos.toString().contains(
+ "Schema : s9:string,s10:string"));
+ }
+
+ @Test(expected = IOException.class)
+ public void test19() throws IOException, ParseException {
+ String schema = "some1:string, some2:string";
+ String storage = "[some1,some2] COMPRESS BY something";
+ Path path1 = new Path(path.toString() + "19");
+ BasicTable.Writer writer = null;
+ writer = new BasicTable.Writer(path1, schema, storage, conf);
+ Assert.fail("should throw exception, none defined serializer");
+ writer.finish();
+ }
+
+ //@Test
+ /*
+ * group1 is user specific
+ */
+ public void test20() throws IOException, ParseException {
+ String schema = "s9:string, s10:string, s11:string";
+ String storage = "[s9,s10] COMPRESS BY gZ; secure by user:user1 group:users perm:755";
+
+ Path path1 = new Path(path.toString() + "20");
+ Runtime.getRuntime().exec("rm -rf " + path1.toString());
+ fs = path.getFileSystem(conf);
+ BasicTable.Writer writer = new BasicTable.Writer(path1, schema, storage,
+ conf);
+ writer.finish();
+ ByteArrayOutputStream bos = new ByteArrayOutputStream();
+ PrintStream ps = new PrintStream(bos);
+ System.out.println("start dumpinfo 20===========" + bos.toString());
+ BasicTable.dumpInfo(path1.toString(), ps, conf);
+ System.out.println("bos: " + bos.toString());
+
+ Assert.assertEquals(true, bos.toString().contains("Serializer: pig"));
+ Assert.assertEquals(true, bos.toString().contains("Compressor: gz"));
+ Assert.assertEquals(true, bos.toString().contains(
+ "Schema : s9:string,s10:string"));
+ }
+
+ @Test(expected = IOException.class)
+ public void test21() throws IOException, ParseException {
+ String schema = "s9:string, s10:string, s11:string";
+ String storage = "[s9,s10] COMPRESS BY gZ; secure by user:user1 group:users perm:755;secure by user:user1 group:users perm:755";
+ Path path1 = new Path(path.toString() + "16");
+ BasicTable.Writer writer = null;
+ writer = new BasicTable.Writer(path1, schema, storage, conf);
+ Assert.fail("should throw exception, none defined serializer");
+ writer.finish();
+ }
+
+ @Test
+ public void test22() throws IOException, ParseException {
+ String schema = "s9:string, s10:string";
+ String storage = "[s9,s10] secure by user:user3";
+
+ Path path1 = new Path(path.toString() + "22");
+ Runtime.getRuntime().exec("rm -rf " + path1.toString());
+ fs = path.getFileSystem(conf);
+ BasicTable.Writer writer = new BasicTable.Writer(path1, schema, storage,
+ conf);
+ writer.finish();
+ ByteArrayOutputStream bos = new ByteArrayOutputStream();
+ PrintStream ps = new PrintStream(bos);
+ System.out.println("start dumpinfo 22===========" + bos.toString());
+ BasicTable.dumpInfo(path1.toString(), ps, conf);
+ System.out.println("bos: " + bos.toString());
+
+ Assert.assertEquals(true, bos.toString().contains("Serializer: pig"));
+ Assert.assertEquals(true, bos.toString().contains("Compressor: gz"));
+ Assert.assertEquals(true, bos.toString().contains(
+ "Schema : s9:string,s10:string"));
+ Assert.assertEquals(true, bos.toString().contains("Group: null"));
+ Assert.assertEquals(true, bos.toString().contains("Perm: -1"));
+ }
+
+ //@Test
+ /*
+ * group1 is user specific
+ */
+ public void test23() throws IOException, ParseException {
+ String schema = "s9:string, s10:string";
+ String storage = "[s9,s10] secure by group:users";
+
+ Path path1 = new Path(path.toString() + "23");
+ Runtime.getRuntime().exec("rm -rf " + path1.toString());
+ fs = path.getFileSystem(conf);
+ BasicTable.Writer writer = new BasicTable.Writer(path1, schema, storage,
+ conf);
+ writer.finish();
+ ByteArrayOutputStream bos = new ByteArrayOutputStream();
+ PrintStream ps = new PrintStream(bos);
+ System.out.println("start dumpinfo 23===========" + bos.toString());
+ BasicTable.dumpInfo(path1.toString(), ps, conf);
+ System.out.println("bos: " + bos.toString());
+
+ Assert.assertEquals(true, bos.toString().contains("Serializer: pig"));
+ Assert.assertEquals(true, bos.toString().contains("Compressor: gz"));
+ Assert.assertEquals(true, bos.toString().contains(
+ "Schema : s9:string,s10:string"));
+ Assert.assertEquals(true, bos.toString().contains("Group: users"));
+ Assert.assertEquals(true, bos.toString().contains("Perm: -1"));
+ }
+
+ //@Test
+ public void test24() throws IOException, ParseException {
+ String schema = "s9:string, s10:string";
+ String storage = "[s9,s10] secure by perm:755";
+
+ Path path1 = new Path(path.toString() + "24");
+ Runtime.getRuntime().exec("rm -rf " + path1.toString());
+ fs = path.getFileSystem(conf);
+ BasicTable.Writer writer = new BasicTable.Writer(path1, schema, storage,
+ conf);
+ writer.finish();
+ ByteArrayOutputStream bos = new ByteArrayOutputStream();
+ PrintStream ps = new PrintStream(bos);
+ System.out.println("start dumpinfo 24===========" + bos.toString());
+ BasicTable.dumpInfo(path1.toString(), ps, conf);
+ System.out.println("bos: " + bos.toString());
+
+ Assert.assertEquals(true, bos.toString().contains("Serializer: pig"));
+ Assert.assertEquals(true, bos.toString().contains("Compressor: gz"));
+ Assert.assertEquals(true, bos.toString().contains(
+ "Schema : s9:string,s10:string"));
+ Assert.assertEquals(true, bos.toString().contains("Group:"));
+ Assert.assertEquals(true, bos.toString().contains("Perm: 0755"));
+ }
+
+ //@Test
+ /*
+ * group1 is user specific
+ */
+ public void test25() throws IOException, ParseException {
+ String schema = "s9:string, s10:string";
+ String storage = "[s9,s10] secure by user:user1 group:users";
+
+ Path path1 = new Path(path.toString() + "25");
+ Runtime.getRuntime().exec("rm -rf " + path1.toString());
+ fs = path.getFileSystem(conf);
+ BasicTable.Writer writer = new BasicTable.Writer(path1, schema, storage,
+ conf);
+ writer.finish();
+ ByteArrayOutputStream bos = new ByteArrayOutputStream();
+ PrintStream ps = new PrintStream(bos);
+ System.out.println("start dumpinfo 25===========" + bos.toString());
+ BasicTable.dumpInfo(path1.toString(), ps, conf);
+ System.out.println("bos: " + bos.toString());
+
+ Assert.assertEquals(true, bos.toString().contains("Serializer: pig"));
+ Assert.assertEquals(true, bos.toString().contains("Compressor: gz"));
+ Assert.assertEquals(true, bos.toString().contains(
+ "Schema : s9:string,s10:string"));
+ Assert.assertEquals(true, bos.toString().contains("Group: users"));
+ Assert.assertEquals(true, bos.toString().contains("Perm: -1"));
+ }
+
+ //@Test
+ public void test26() throws IOException, ParseException {
+ String schema = "s9:string, s10:string";
+ String storage = "[s9,s10] secure by perm:755 group:users";
+
+ Path path1 = new Path(path.toString() + "26");
+ Runtime.getRuntime().exec("rm -rf " + path1.toString());
+ fs = path.getFileSystem(conf);
+ BasicTable.Writer writer = new BasicTable.Writer(path1, schema, storage,
+ conf);
+ writer.finish();
+ ByteArrayOutputStream bos = new ByteArrayOutputStream();
+ PrintStream ps = new PrintStream(bos);
+ System.out.println("start dumpinfo 26===========" + bos.toString());
+ BasicTable.dumpInfo(path1.toString(), ps, conf);
+ System.out.println("bos: " + bos.toString());
+
+ Assert.assertEquals(true, bos.toString().contains("Serializer: pig"));
+ Assert.assertEquals(true, bos.toString().contains("Compressor: gz"));
+ Assert.assertEquals(true, bos.toString().contains(
+ "Schema : s9:string,s10:string"));
+ Assert.assertEquals(true, bos.toString().contains("Group: users"));
+ Assert.assertEquals(true, bos.toString().contains("Perm: 755"));
+ }
+
+ @Test(expected = IOException.class)
+ public void test27() throws IOException, ParseException {
+ String schema = "s9:string, s10:string";
+ String storage = "[s9,s10] COMPRESS BY gZ; secure by ";
+ Path path1 = new Path(path.toString() + "27");
+ BasicTable.Writer writer = null;
+ writer = new BasicTable.Writer(path1, schema, storage, conf);
+ Assert.fail("should throw exception, none defined serializer");
+ writer.finish();
+ }
+}