You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@pig.apache.org by ga...@apache.org on 2010/04/16 01:56:50 UTC
svn commit: r934649 [21/24] - in /hadoop/pig/trunk/contrib: ./ owl/ owl/bin/
owl/ci/ owl/ci/test_results/ owl/ci/test_scripts/ owl/docs/ owl/ivy/
owl/java/ owl/java/lib/ owl/java/main/ owl/java/main/META-INF/
owl/java/main/org/ owl/java/main/org/apache...
Added: hadoop/pig/trunk/contrib/owl/java/test/org/apache/hadoop/owl/logical/OwlLogicalLayerTest.java
URL: http://svn.apache.org/viewvc/hadoop/pig/trunk/contrib/owl/java/test/org/apache/hadoop/owl/logical/OwlLogicalLayerTest.java?rev=934649&view=auto
==============================================================================
--- hadoop/pig/trunk/contrib/owl/java/test/org/apache/hadoop/owl/logical/OwlLogicalLayerTest.java (added)
+++ hadoop/pig/trunk/contrib/owl/java/test/org/apache/hadoop/owl/logical/OwlLogicalLayerTest.java Thu Apr 15 23:56:44 2010
@@ -0,0 +1,759 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.owl.logical;
+
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+import org.apache.hadoop.owl.backend.OwlBackend;
+import org.apache.hadoop.owl.client.DatabaseTest;
+import org.apache.hadoop.owl.client.OwlClient;
+import org.apache.hadoop.owl.common.ErrorType;
+import org.apache.hadoop.owl.common.OwlException;
+import org.apache.hadoop.owl.common.OwlUtil;
+import org.apache.hadoop.owl.entity.DatabaseEntity;
+import org.apache.hadoop.owl.entity.DataElementEntity;
+import org.apache.hadoop.owl.entity.GlobalKeyEntity;
+import org.apache.hadoop.owl.entity.KeyValueEntity;
+import org.apache.hadoop.owl.entity.OwlTableEntity;
+import org.apache.hadoop.owl.entity.OwlTableKeyValueEntity;
+import org.apache.hadoop.owl.entity.PartitionEntity;
+import org.apache.hadoop.owl.entity.PartitionKeyEntity;
+import org.apache.hadoop.owl.entity.PropertyKeyEntity;
+import org.apache.hadoop.owl.protocol.OwlDataElement;
+import org.apache.hadoop.owl.protocol.OwlKey;
+import org.apache.hadoop.owl.protocol.OwlObject;
+
+import org.junit.After;
+import org.junit.AfterClass;
+import org.junit.Before;
+import org.junit.BeforeClass;
+import org.junit.Test;
+
+import junit.framework.TestCase;
+
+public class OwlLogicalLayerTest extends TestCase {
+
+ private static final String RETENTION = "retention";
+ private static final String VUTS = "vuts";
+ private static final String EIGHT = "8";
+ private static final String NINE = "9";
+ private static final String ONE = "1";
+ private static final String ZYXW = "zyxw";
+ private static final String ABCD = "ABCD";
+
+ private static final String PROP_B_INT = "prbi";
+ private static final String PTN_B_INT = "pnbi";
+ private static final String PROP_A_STRING = "pras";
+ private static final String PTN_A_STRING = "pnas";
+
+ private static final String TESTDATABASE_LOCN = "/tmp/ll_testcat";
+ private static final String TESTDATABASE_NAME = "ll_testcat";
+
+ private static final String TESTOWLTABLE_NAME = "ll_testowltable";
+ private static final String TESTGKEYOWLTABLE_NAME = "ll_testgkeyowltable";
+
+ private static final String TESTDE_LOCN = TESTDATABASE_LOCN + "/" + TESTOWLTABLE_NAME + "/ll_testde";
+ private static final String TESTDE_LOCN2 = TESTDATABASE_LOCN + "/" + TESTOWLTABLE_NAME + "/ll_testde_2";
+
+ private static final String TEST_GKEY = "ll_gkey";
+
+ private static OwlBackend backend;
+
+ private static int databaseId = 0;
+ private static int otableId = 0;
+ private static int ptnAStringKeyId = 0;
+ private static int ptnBIntegerKeyId = 0;
+ private static int propAStringKeyId = 0;
+ private static int propBIntegerKeyId = 0;
+
+ private static boolean gkeyBasicTested;
+ private static boolean gkeyTested;
+
+ private static boolean databaseCreated;
+ private static boolean owlTableCreated;
+ private static boolean dePublished;
+ private static boolean deSelected;
+
+ private static boolean deAltered1;
+ private static boolean deAltered2;
+ private static boolean deAltered3;
+
+ @BeforeClass
+ public void setUp() throws OwlException {
+ System.out.println("@BeforeClass running");
+
+
+ OwlException connectException = null;
+ //Retry backend connection in case previous test has not closed its derbydb connection yet
+ for(int i = 0; i < DatabaseTest.CONNECTION_RETRY_COUNT;i++) {
+ try {
+ backend = new OwlBackend();
+ cleanup();
+ connectException = null;
+ break;
+ } catch(OwlException e) {
+ if( e.getErrorType() != ErrorType.ERROR_DB_INIT ) {
+ throw e;
+ } else {
+ connectException = e;
+ long retryTime = DatabaseTest.RETRY_SLEEP_MILLIS * (i + 1);
+ System.out.println("Connection failed " + e.toString() + ". Retrying after " + retryTime + " ms");
+ try { Thread.sleep(retryTime); } catch(Exception e2) {}
+ }
+ }
+ }
+
+ if( connectException != null ) {
+ throw connectException;
+ }
+ }
+
+ @AfterClass
+ public void tearDown() throws OwlException {
+ System.out.println("@AfterClass running");
+ cleanup();
+ backend.close();
+ }
+
+ private static void localTestBasicGkey() throws OwlException {
+ if (!gkeyBasicTested){
+
+ // create global key
+ CreateGlobalKeyCommand createGlobalKeyCmd = (CreateGlobalKeyCommand) CommandFactory.getCommand("CREATE GLOBALKEY");
+ createGlobalKeyCmd.addPropertyKey(TEST_GKEY, "STRING");
+ createGlobalKeyCmd.execute(backend);
+
+ List<GlobalKeyEntity> listOfGlobalKeys;
+
+ // get list of global keys to see if it exists
+ listOfGlobalKeys = backend.find(GlobalKeyEntity.class,
+ "name = \""+TEST_GKEY+"\"");
+ assertEquals(1, listOfGlobalKeys.size() );
+ assertEquals(OwlKey.DataType.STRING.getCode(),listOfGlobalKeys.get(0).getDataType());
+
+ // delete global key
+ DropGlobalKeyCommand dropGlobalKeyCmd = (DropGlobalKeyCommand) CommandFactory.getCommand("DROP GLOBALKEY");
+ dropGlobalKeyCmd.addPropertyKey(TEST_GKEY, null);
+ dropGlobalKeyCmd.execute(backend);
+
+ // verify that global key is deleted
+ listOfGlobalKeys = backend.find(GlobalKeyEntity.class,
+ "name = \""+TEST_GKEY+"\"");
+ assertTrue(listOfGlobalKeys.size() == 0);
+
+ gkeyBasicTested = true;
+ }
+ }
+
+ private static void localTestCreateDatabase() throws OwlException {
+ if (!databaseCreated){
+ localTestBasicGkey();
+
+ CreateDatabaseCommand createDatabaseCmd = (CreateDatabaseCommand) CommandFactory.getCommand("CREATE OWLDATABASE");
+ createDatabaseCmd.setName(TESTDATABASE_NAME);
+ createDatabaseCmd.setExternalStorageIdentifier(TESTDATABASE_LOCN);
+ createDatabaseCmd.execute(backend);
+
+ List<DatabaseEntity> listOfDatabases = backend.find(DatabaseEntity.class,
+ "name = \""+TESTDATABASE_NAME+"\"");
+ assertTrue(listOfDatabases.size() == 1);
+ databaseCreated = true;
+
+ databaseId = listOfDatabases.get(0).getId();
+ assertTrue(databaseId > 0);
+ assertTrue(listOfDatabases.get(0).getName().equals(TESTDATABASE_NAME));
+ assertTrue(listOfDatabases.get(0).getLocation().equals(TESTDATABASE_LOCN));
+ }
+ }
+
+ private static void localTestCreateOwlTable() throws OwlException {
+
+ if (!owlTableCreated){
+ localTestCreateDatabase();
+
+ CreateOwlTableCommand createOwlTableCmd = (CreateOwlTableCommand) CommandFactory.getCommand("CREATE OWLTABLE");
+
+ createOwlTableCmd.setBasicDataSetType();
+ createOwlTableCmd.setName(TESTOWLTABLE_NAME);
+ createOwlTableCmd.setParentDatabase(TESTDATABASE_NAME);
+ createOwlTableCmd.addPropertyKey(RETENTION, "INTEGER");
+ createOwlTableCmd.addPartitionKey(PTN_A_STRING, "STRING", "LIST", null, null, null, null);
+ createOwlTableCmd.addPropertyKey(PROP_A_STRING,"STRING");
+ createOwlTableCmd.addPartitionKey(PTN_B_INT, "INTEGER", "LIST", null, null, null, null);
+ createOwlTableCmd.addPropertyKey(PROP_B_INT,"INTEGER");
+ createOwlTableCmd.addPropertyKeyValue(RETENTION, "60");
+ createOwlTableCmd.addSchemaElement("c1:int");
+ createOwlTableCmd.execute(backend);
+
+ String owlTableFilter = "name = \""+TESTOWLTABLE_NAME+"\" and databaseId = "+databaseId;
+ System.out.println("Selecting owltables matching filter : "+ owlTableFilter);
+ List<OwlTableEntity> listOfOwlTables = backend.find(OwlTableEntity.class,
+ owlTableFilter
+ );
+
+ System.out.println("Matches : "+listOfOwlTables.size());
+
+ assertTrue(listOfOwlTables.size() == 1);
+ owlTableCreated = true;
+
+ OwlTableEntity otable = listOfOwlTables.get(0);
+ otableId = otable.getId();
+
+ assertTrue(otable.getName().equals(TESTOWLTABLE_NAME));
+ assertTrue(otable.getDatabaseId() == databaseId);
+ assertTrue(otableId > 0);
+
+ List<OwlTableKeyValueEntity> okvs = otable.getKeyValues();
+ assertTrue (okvs.size() == 1);
+ OwlTableKeyValueEntity retentionKeyValue = okvs.get(0);
+ int retentionKeyId = retentionKeyValue.getPropertyKeyId();
+ assertTrue(retentionKeyValue.getIntValue() == 60);
+
+ Map<String,PropertyKeyEntity> propKeysByName = new HashMap<String,PropertyKeyEntity>();
+ for (PropertyKeyEntity propKey : otable.getPropertyKeys()){
+ propKeysByName.put(propKey.getName(),propKey);
+ }
+
+ assertTrue(propKeysByName.size() == 3);
+
+ assertTrue(propKeysByName.containsKey(RETENTION));
+ PropertyKeyEntity retentionKey = propKeysByName.get(RETENTION);
+ assertTrue(retentionKey.getId() == retentionKeyId);
+
+ assertTrue(propKeysByName.containsKey( PROP_A_STRING));
+ PropertyKeyEntity propAString = propKeysByName.get( PROP_A_STRING);
+ propAStringKeyId = propAString.getId();
+ assertTrue(propAStringKeyId > 0);
+
+ assertTrue(propKeysByName.containsKey(PROP_B_INT));
+ PropertyKeyEntity propBInteger = propKeysByName.get(PROP_B_INT);
+ propBIntegerKeyId = propBInteger.getId();
+ assertTrue(propBIntegerKeyId > 0);
+
+ Map<String,PartitionKeyEntity> ptnKeysByName = new HashMap<String,PartitionKeyEntity>();
+ for (PartitionKeyEntity ptnKey : otable.getPartitionKeys()){
+ ptnKeysByName.put(ptnKey.getName(), ptnKey);
+ }
+
+ assertTrue(ptnKeysByName.size() == 2);
+
+ assertTrue(ptnKeysByName.containsKey(PTN_A_STRING));
+ PartitionKeyEntity ptnAString = ptnKeysByName.get(PTN_A_STRING);
+ assertTrue(ptnAString.getPartitionLevel() == 1);
+ ptnAStringKeyId = ptnAString.getId();
+ assertTrue(ptnAStringKeyId > 0);
+
+ assertTrue(ptnKeysByName.containsKey( PTN_B_INT));
+ PartitionKeyEntity ptnBInteger = ptnKeysByName.get(PTN_B_INT);
+ assertTrue(ptnBInteger.getPartitionLevel() == 2);
+ ptnBIntegerKeyId = ptnBInteger.getId();
+ assertTrue(ptnBIntegerKeyId > 0);
+ }
+ }
+
+ private static void verifyDE(String valPNAS, String valPNBI, String valPRAS, String valPRASAtLowerPtn , String valPRBI) throws OwlException {
+
+ // TODO : modify to allow nulls, and gkey possibilities - be more generic testing for the DE structure
+
+ List<PartitionEntity> listOfPartitionsInOwlTable = backend.find(PartitionEntity.class,
+ "owlTableId = " + otableId
+ );
+ System.out.println("BECK: listOfPartitionsInOwlTable.size : " + listOfPartitionsInOwlTable.size());
+
+ List<PartitionEntity> listOfPartitions = backend.find(PartitionEntity.class,
+ "owlTableId = " + otableId
+ + " and ["+ PTN_A_STRING+"] = \""+valPNAS+"\""
+ + " and ["+ PTN_B_INT+"] = "+valPNBI
+ );
+ System.out.println("BECK: listOfPartitions.size : " + listOfPartitions.size());
+ assertTrue(listOfPartitions.size() == 1);
+
+ List<DataElementEntity> listOfDataElements = backend.find(DataElementEntity.class,
+ "partitionId = "+listOfPartitions.get(0).getId());
+
+ assertTrue(listOfDataElements.size() == 1);
+
+ assertTrue(listOfDataElements.get(0).getLocation().equals(TESTDE_LOCN));
+
+ List<PartitionEntity> listOfTopLevelPartitions = backend.find(PartitionEntity.class,
+ "owlTableId = " + otableId
+ + " and ["+ PTN_A_STRING+"] = \""+valPNAS+"\""
+ // + " and partitionLevel = 1"
+ + " and parentPartitionId is null"
+ );
+
+ assertTrue(listOfTopLevelPartitions.size() == 1);
+
+ PartitionEntity abcdPartition = listOfTopLevelPartitions.get(0);
+ assertTrue(abcdPartition.getId() > 0);
+
+ List<KeyValueEntity> abcdKeyValues = abcdPartition.getKeyValues();
+
+ assertTrue(abcdKeyValues.size() >= 1);
+
+ Map <Integer,KeyValueEntity> abcdPropKeyValuesByKeyId = new HashMap<Integer,KeyValueEntity>();
+ Map <Integer,KeyValueEntity> abcdPtnKeyValuesByKeyId = new HashMap<Integer,KeyValueEntity>();
+ for (KeyValueEntity kve : abcdKeyValues){
+ System.out.println(
+ " otable["+ kve.getOwlTableId()
+ + "] ptn[" + kve.getPartition().getId()
+ + "] ptnKeyId[" + nullGuardedPrint(kve.getPartitionKeyId())
+ + "] propKeyId[" + nullGuardedPrint(kve.getPropertyKeyId())
+ + "] globalKeyId[" + nullGuardedPrint(kve.getGlobalKeyId())
+ + "] stringValue[" + nullGuardedPrint(kve.getStringValue())
+ + "] intValue[" + nullGuardedPrint(kve.getIntValue())
+ + "]"
+ );
+
+ if (kve.getPropertyKeyId() != null){
+ abcdPropKeyValuesByKeyId.put(kve.getPropertyKeyId(),kve);
+ }
+ if (kve.getPartitionKeyId() != null){
+ abcdPtnKeyValuesByKeyId.put(kve.getPartitionKeyId(),kve);
+ }
+ }
+
+
+ assertTrue(abcdPtnKeyValuesByKeyId.containsKey(ptnAStringKeyId));
+ KeyValueEntity abcdPtnKeyValue = abcdPtnKeyValuesByKeyId.get(ptnAStringKeyId);
+ assertTrue(abcdPtnKeyValue.getStringValue().equals(valPNAS));
+
+ if (valPRAS != null){
+ assertTrue(abcdPropKeyValuesByKeyId.containsKey(propAStringKeyId));
+ KeyValueEntity abcdPropKeyValue = abcdPropKeyValuesByKeyId.get(propAStringKeyId);
+ assertTrue(abcdPropKeyValue.getStringValue().equals(valPRAS));
+ }
+
+ List<PartitionEntity> listOfBottomLevelPartitions = backend.find(PartitionEntity.class,
+ "owlTableId = " + otableId
+ + "and ["+ PTN_A_STRING+"] = \""+valPNAS+"\""
+ + " and ["+ PTN_B_INT+"] = "+valPNBI
+ // + " and partitionLevel = 2"
+ + " and parentPartitionId = " + abcdPartition.getId()
+ );
+
+ assertTrue(listOfBottomLevelPartitions.size() == 1);
+
+ PartitionEntity abcd1Partition = listOfBottomLevelPartitions.get(0);
+ assertTrue(abcd1Partition.getId() > 0);
+ List<KeyValueEntity> abcd1KeyValues = abcd1Partition.getKeyValues();
+
+ assertTrue (abcd1KeyValues.size() >= 2);
+
+ Map <Integer,KeyValueEntity> abcd1PropKeyValuesByKeyId = new HashMap<Integer,KeyValueEntity>();
+ Map <Integer,KeyValueEntity> abcd1PtnKeyValuesByKeyId = new HashMap<Integer,KeyValueEntity>();
+
+ for (KeyValueEntity kve : abcd1KeyValues){
+ // System.out.println("JIMI kve id=[" + kve.getId() +"] , otableId = ["+kve.getOwlTableId()+"]");
+ // if (kve.getIntValue() != null){ System.out.println("JIMI kve intval=["+kve.getIntValue()+"]");}
+ // if (kve.getStringValue() != null){ System.out.println("JIMI kve stringval=["+kve.getStringValue()+"]");}
+ // if (kve.getPropertyKeyId() != null){ System.out.println("JIMI kve PropertyKeyId()=["+kve.getPropertyKeyId()+"]");}
+ // if (kve.getPartitionKeyId() != null){ System.out.println("JIMI kve PartitionKeyId()=["+kve.getPartitionKeyId()+"]");}
+ // if (kve.getGlobalKeyId() != null){ System.out.println("JIMI kve GlobalKeyId()=["+kve.getGlobalKeyId()+"]");}
+
+ if (kve.getPropertyKeyId() != null){
+ abcd1PropKeyValuesByKeyId.put(kve.getPropertyKeyId(),kve);
+ }
+ if (kve.getPartitionKeyId() != null){
+ abcd1PtnKeyValuesByKeyId.put(kve.getPartitionKeyId(),kve);
+ }
+
+ }
+
+ assertTrue(abcd1PtnKeyValuesByKeyId.containsKey(ptnBIntegerKeyId));
+ assertTrue(abcd1PtnKeyValuesByKeyId.get(ptnBIntegerKeyId).getIntValue().intValue() == Integer.valueOf(valPNBI).intValue());
+
+ if (valPRBI != null){
+ assertTrue(abcd1PropKeyValuesByKeyId.containsKey(propBIntegerKeyId));
+ assertEquals(abcd1PropKeyValuesByKeyId.get(propBIntegerKeyId).getIntValue(), Integer.valueOf(valPRBI));
+ }
+
+ assertTrue(abcd1PtnKeyValuesByKeyId.containsKey(ptnAStringKeyId));
+ assertTrue(abcd1PtnKeyValuesByKeyId.get(ptnAStringKeyId).getStringValue().equals(valPNAS));
+
+ if (valPRASAtLowerPtn != null){
+ assertTrue(abcd1PropKeyValuesByKeyId.containsKey(propAStringKeyId));
+ assertTrue(abcd1PropKeyValuesByKeyId.get(propAStringKeyId).getStringValue().equals(valPRASAtLowerPtn));
+ }
+
+ assertTrue(listOfDataElements.get(0).getPartitionId() == abcd1Partition.getId());
+
+ }
+
+
+ private static void localPublishDataElement() throws OwlException{
+ if (! dePublished){
+ localTestCreateOwlTable();
+
+ PublishDataElementCommand publishCmd = (PublishDataElementCommand) CommandFactory.getCommand("PUBLISH DATAELEMENT");
+ publishCmd.setName(TESTOWLTABLE_NAME);
+ publishCmd.setParentDatabase(TESTDATABASE_NAME);
+ publishCmd.setLocation(TESTDE_LOCN);
+ publishCmd.addPartitionKeyValue(PTN_A_STRING, ABCD);
+ publishCmd.addPropertyKeyValue(PROP_A_STRING, ZYXW);
+ publishCmd.addPartitionKeyValue(PTN_B_INT, ONE);
+ publishCmd.addPropertyKeyValue(PROP_B_INT, NINE);
+ publishCmd.addSchemaElement("c1:int");
+ publishCmd.execute(backend);
+
+ verifyDE(ABCD,ONE,null,ZYXW,NINE);
+ dePublished = true;
+
+ }
+ }
+
+ private static void localSelectDataElementObjects() throws OwlException {
+ if (! deSelected){
+ localPublishDataElement();
+
+ SelectDataelementObjectsCommand selectDeCmd1 = (SelectDataelementObjectsCommand) CommandFactory.getCommand("SELECT DATAELEMENT OBJECTS");
+ selectDeCmd1.addAdditionalActionInfo(TESTOWLTABLE_NAME);
+ selectDeCmd1.setParentDatabase(TESTDATABASE_NAME);
+ selectDeCmd1.addPartitionFilter(PTN_A_STRING, CommandInfo.Operator.EQUAL.getOp(), ABCD);
+ selectDeCmd1.addPropertyFilter(PROP_A_STRING, CommandInfo.Operator.EQUAL.getOp(), ZYXW);
+ selectDeCmd1.addPartitionFilter(PTN_B_INT, CommandInfo.Operator.EQUAL.getOp(), ONE);
+ selectDeCmd1.addPropertyFilter(PROP_B_INT, CommandInfo.Operator.EQUAL.getOp(), NINE);
+
+ //List<OwlDataElement>
+ List<? extends OwlObject> results = selectDeCmd1.execute(backend);
+
+ System.out.println("testSelectDataelementObjects \n");
+
+ assertTrue(results.size() == 1);
+
+ OwlDataElement de = (OwlDataElement) results.get(0);
+ //TODO: verify de.getKeyValues
+
+ deSelected = true;
+ }
+ }
+
+
+ private static void localAlterOwlTableModifyPartitionProperty() throws OwlException {
+ if (! deAltered1){
+
+ localSelectDataElementObjects();
+
+ AlterOwlTableCommand alterModifyCmd1 = (AlterOwlTableCommand) CommandFactory.getCommand("ALTER OWLTABLE");
+ alterModifyCmd1.setName(TESTOWLTABLE_NAME);
+ alterModifyCmd1.setParentDatabase(TESTDATABASE_NAME);
+ alterModifyCmd1.addAdditionalActionInfo("MODIFY PARTITION PROPERTY");
+ alterModifyCmd1.addPartitionFilter(PTN_A_STRING, CommandInfo.Operator.EQUAL.getOp(), ABCD);
+ alterModifyCmd1.addPartitionFilter(PTN_B_INT, CommandInfo.Operator.EQUAL.getOp(),ONE);
+ alterModifyCmd1.addPropertyKeyValue(PROP_B_INT, EIGHT);
+ alterModifyCmd1.execute(backend);
+
+ verifyDE(ABCD,ONE,null,ZYXW,EIGHT);
+
+ AlterOwlTableCommand alterModifyCmd2 = (AlterOwlTableCommand) CommandFactory.getCommand("ALTER OWLTABLE");
+ alterModifyCmd2.setName(TESTOWLTABLE_NAME);
+ alterModifyCmd2.setParentDatabase(TESTDATABASE_NAME);
+ alterModifyCmd2.addAdditionalActionInfo("MODIFY PARTITION PROPERTY");
+ alterModifyCmd2.addPartitionFilter(PTN_A_STRING, CommandInfo.Operator.EQUAL.getOp(), ABCD);
+ alterModifyCmd2.addPropertyKeyValue(PROP_A_STRING, VUTS);
+ alterModifyCmd2.execute(backend);
+
+ verifyDE(ABCD,ONE,VUTS,VUTS,EIGHT); // change to verifyDE(ABCD,ONE,VUTS,ZYXW,EIGHT) after property pushdown removal
+
+ AlterOwlTableCommand alterModifyCmd3 = (AlterOwlTableCommand) CommandFactory.getCommand("ALTER OWLTABLE");
+ alterModifyCmd3.setName(TESTOWLTABLE_NAME);
+ alterModifyCmd3.setParentDatabase(TESTDATABASE_NAME);
+ alterModifyCmd3.addAdditionalActionInfo("MODIFY PARTITION PROPERTY");
+ alterModifyCmd3.addPartitionFilter(PTN_A_STRING, CommandInfo.Operator.EQUAL.getOp(),ABCD);
+ alterModifyCmd3.addPartitionFilter(PTN_B_INT, CommandInfo.Operator.EQUAL.getOp(),ONE);
+ alterModifyCmd3.addPropertyKeyValue(PROP_A_STRING, ZYXW);
+ alterModifyCmd3.execute(backend);
+
+ verifyDE(ABCD,ONE,VUTS,ZYXW,EIGHT);
+
+ deAltered1 = true;
+ }
+ }
+
+ private static void localTestAlterOwlTableDropPartitionProperty() throws OwlException {
+ if (!deAltered2){
+
+ localAlterOwlTableModifyPartitionProperty();
+
+ AlterOwlTableCommand alterDropCmd1 = (AlterOwlTableCommand) CommandFactory.getCommand("ALTER OWLTABLE");
+ alterDropCmd1.setName(TESTOWLTABLE_NAME);
+ alterDropCmd1.setParentDatabase(TESTDATABASE_NAME);
+ alterDropCmd1.addAdditionalActionInfo("DROP PARTITION PROPERTY");
+ alterDropCmd1.addPartitionFilter(PTN_A_STRING, CommandInfo.Operator.EQUAL.getOp(), ABCD);
+ alterDropCmd1.addPartitionFilter(PTN_B_INT, CommandInfo.Operator.EQUAL.getOp(), ONE);
+ alterDropCmd1.addPropertyKeyValue(PROP_A_STRING, ""); // TODO : Grammar needs to change to let this just be addPropertyKey
+ alterDropCmd1.execute(backend);
+
+ /* TODO note:
+ * Drop Partition Property takes the same command form as Modify Partition Property,
+ * so it uses addPropertyKeyValue, even though it ignores the value - this is not good/clear,
+ * and should be changed from DDL onwards.
+ */
+
+ verifyDE(ABCD,ONE,VUTS,null,EIGHT);
+
+ // AlterOwlTableCommand alterModifyCmd1 = (AlterOwlTableCommand) CommandFactory.getCommand("ALTER OWLTABLE");
+ // alterModifyCmd1.setName(TESTOWLTABLE_NAME);
+ // alterModifyCmd1.setParentDatabase(TESTDATABASE_NAME);
+ // alterModifyCmd1.addAdditionalActionInfo("MODIFY PARTITION PROPERTY");
+ // alterModifyCmd1.addPartitionFilter(PTN_A_STRING, CommandInfo.Operator.EQUAL.getOp(), ABCD);
+ // alterModifyCmd1.addPropertyKeyValue(PROP_A_STRING, ZYXW);
+ // alterModifyCmd1.execute(backend);
+ //
+ // verifyDE(ABCD,ONE,ZYXW,ZYXW,EIGHT);
+ //
+ // AlterOwlTableCommand alterDropCmd2 = (AlterOwlTableCommand) CommandFactory.getCommand("ALTER OWLTABLE");
+ // alterDropCmd2.setName(TESTOWLTABLE_NAME);
+ // alterDropCmd2.setParentDatabase(TESTDATABASE_NAME);
+ // alterDropCmd2.addAdditionalActionInfo("DROP PARTITION PROPERTY");
+ // alterDropCmd2.addPartitionFilter(PTN_A_STRING, CommandInfo.Operator.EQUAL.getOp(), ABCD);
+ // alterDropCmd2.addPropertyKeyValue(PROP_A_STRING, ""); // TODO : Grammar needs to change to let this just be addPropertyKey
+ // alterDropCmd2.execute(backend);
+ //
+ // verifyDE(ABCD,ONE,null,null,EIGHT);
+ //
+ // AlterOwlTableCommand alterModifyCmd2 = (AlterOwlTableCommand) CommandFactory.getCommand("ALTER OWLTABLE");
+ // alterModifyCmd2.setName(TESTOWLTABLE_NAME);
+ // alterModifyCmd2.setParentDatabase(TESTDATABASE_NAME);
+ // alterModifyCmd2.addAdditionalActionInfo("MODIFY PARTITION PROPERTY");
+ // alterModifyCmd2.addPartitionFilter(PTN_A_STRING, CommandInfo.Operator.EQUAL.getOp(), ABCD);
+ // alterModifyCmd2.addPartitionFilter(PTN_B_INT, CommandInfo.Operator.EQUAL.getOp(), ONE);
+ // alterModifyCmd2.addPropertyKeyValue(PROP_A_STRING, VUTS);
+ // alterModifyCmd2.execute(backend);
+ //
+ // verifyDE(ABCD,ONE,null,VUTS,EIGHT);
+
+ deAltered2 = true;
+ }
+ }
+
+ @Test
+ public static void testGkey() throws OwlException {
+ if (!gkeyTested){
+ backend.beginTransaction();
+ localTestAlterOwlTableDropPartitionProperty();
+
+ // create global key
+ CreateGlobalKeyCommand createGlobalKeyCmd = (CreateGlobalKeyCommand) CommandFactory.getCommand("CREATE GLOBALKEY");
+ createGlobalKeyCmd.addPropertyKey(TEST_GKEY, "STRING");
+ createGlobalKeyCmd.execute(backend);
+
+ List<GlobalKeyEntity> listOfGlobalKeys;
+
+ // get list of global keys to see if it exists
+ listOfGlobalKeys = backend.find(GlobalKeyEntity.class,
+ "name = \""+TEST_GKEY+"\"");
+ assertTrue(listOfGlobalKeys.size() == 1);
+ assertEquals(OwlKey.DataType.STRING.getCode(),listOfGlobalKeys.get(0).getDataType());
+ int gKeyId = listOfGlobalKeys.get(0).getId();
+
+ // create a simpletable, set a gkey-value during table create.
+ CreateOwlTableCommand createOwlTableCmd = (CreateOwlTableCommand) CommandFactory.getCommand("CREATE OWLTABLE");
+
+ createOwlTableCmd.setBasicDataSetType();
+ createOwlTableCmd.setName(TESTGKEYOWLTABLE_NAME);
+ createOwlTableCmd.setParentDatabase(TESTDATABASE_NAME);
+ createOwlTableCmd.addPartitionKey(PTN_A_STRING, "STRING", "LIST", null, null, null, null);
+ createOwlTableCmd.addPropertyKeyValue(TEST_GKEY, VUTS);
+ createOwlTableCmd.addSchemaElement("c1:int");
+ createOwlTableCmd.execute(backend);
+
+ // check that the table was created correctly with the gkey-value as an otkv
+
+ String owlTableFilter = "name = \""+TESTGKEYOWLTABLE_NAME+"\" and databaseId = "+databaseId;
+ System.out.println("Selecting owltables matching filter : "+ owlTableFilter);
+ List<OwlTableEntity> listOfOwlTables = backend.find(OwlTableEntity.class,
+ owlTableFilter
+ );
+
+ System.out.println("Matches : "+listOfOwlTables.size());
+
+ assertTrue(listOfOwlTables.size() == 1);
+
+ OwlTableEntity otable = listOfOwlTables.get(0);
+
+ assertTrue(otable.getPartitionKeys().size() == 1);
+ int ptnKeyId = otable.getPartitionKeys().get(0).getId();
+ int gkeyOtableId = otable.getId();
+
+ List<OwlTableKeyValueEntity> okvs = otable.getKeyValues();
+ assertTrue (okvs.size() == 1);
+ OwlTableKeyValueEntity gKeyValue = okvs.get(0);
+ assertTrue(gKeyId == gKeyValue.getGlobalKeyId());
+ assertTrue(gKeyValue.getStringValue().equalsIgnoreCase(VUTS));
+
+ // publish a de to the table, check that there's an appropriate kv at the leaf ptn
+
+ PublishDataElementCommand publishCmd = (PublishDataElementCommand) CommandFactory.getCommand("PUBLISH DATAELEMENT");
+ publishCmd.setName(TESTGKEYOWLTABLE_NAME);
+ publishCmd.setParentDatabase(TESTDATABASE_NAME);
+ publishCmd.setLocation(TESTDE_LOCN2);
+ publishCmd.addPartitionKeyValue(PTN_A_STRING, ABCD);
+ publishCmd.addPropertyKeyValue(TEST_GKEY, ZYXW);
+ publishCmd.addSchemaElement("c2:collection(r1:record(f1:int,f2:int))");
+ publishCmd.execute(backend);
+
+ verifyDECase2(gKeyId, ptnKeyId, gkeyOtableId,ZYXW);
+
+ // alter table to alter the kv at the leaf ptn
+
+ AlterOwlTableCommand alterModifyCmd1 = (AlterOwlTableCommand) CommandFactory.getCommand("ALTER OWLTABLE");
+ alterModifyCmd1.setName(TESTGKEYOWLTABLE_NAME);
+ alterModifyCmd1.setParentDatabase(TESTDATABASE_NAME);
+ alterModifyCmd1.addAdditionalActionInfo("MODIFY PARTITION PROPERTY");
+ alterModifyCmd1.addPartitionFilter(PTN_A_STRING, CommandInfo.Operator.EQUAL.getOp(), ABCD);
+ alterModifyCmd1.addPropertyKeyValue(TEST_GKEY, ABCD);
+ alterModifyCmd1.execute(backend);
+
+ verifyDECase2(gKeyId, ptnKeyId, gkeyOtableId,ABCD);
+
+ // alter table to alter the kv at the root node
+
+ // AlterOwlTableCommand alterModifyCmd2 = (AlterOwlTableCommand) CommandFactory.getCommand("ALTER OWLTABLE");
+ // alterModifyCmd2.setName(TESTGKEYOWLTABLE_NAME);
+ // alterModifyCmd2.setParentDatabase(TESTDATABASE_NAME);
+ // alterModifyCmd2.addAdditionalActionInfo("MODIFY PARTITION PROPERTY");
+ // alterModifyCmd2.addPropertyKeyValue(TEST_GKEY, ZYXW);
+ // alterModifyCmd2.execute(backend);
+ //
+ // verifyDECase2(gKeyId, ptnKeyId, gkeyOtableId,ABCD); // verify that the gkey-value at the leaf ptn did not change
+ //
+ // System.out.println("Selecting owltables matching filter : "+ owlTableFilter);
+ // listOfOwlTables = backend.find(OwlTableEntity.class,
+ // owlTableFilter
+ // );
+ //
+ // System.out.println("Matches : "+listOfOwlTables.size());
+ //
+ // assertTrue(listOfOwlTables.size() == 1);
+ //
+ // otable = listOfOwlTables.get(0);
+ //
+ // assertTrue(otable.getPartitionKeys().size() == 1);
+ // assertTrue(ptnKeyId == otable.getPartitionKeys().get(0).getId());
+ // assertTrue(gkeyOtableId == otable.getId());
+ //
+ // okvs = otable.getKeyValues();
+ // assertTrue (okvs.size() == 1);
+ // gKeyValue = okvs.get(0);
+ // assertTrue(gKeyId == gKeyValue.getGlobalKeyId());
+ // assertTrue(gKeyValue.getStringValue().equalsIgnoreCase(ZYXW)); // verify that the gkey-value at the table level changed
+
+ // nuke table
+ DropOwlTableCommand dropTableCommand = (DropOwlTableCommand) CommandFactory.getCommand("DROP OWLTABLE");
+ dropTableCommand.setName(TESTGKEYOWLTABLE_NAME);
+ dropTableCommand.setParentDatabase(TESTDATABASE_NAME);
+ dropTableCommand.execute(backend);
+
+ // delete global key
+ DropGlobalKeyCommand dropGlobalKeyCmd = (DropGlobalKeyCommand) CommandFactory.getCommand("DROP GLOBALKEY");
+ dropGlobalKeyCmd.addPropertyKey(TEST_GKEY, null);
+ dropGlobalKeyCmd.execute(backend);
+
+ // verify that global key is deleted
+ listOfGlobalKeys = backend.find(GlobalKeyEntity.class,
+ "name = \""+ TEST_GKEY+"\"");
+ assertTrue(listOfGlobalKeys.size() == 0);
+ backend.rollbackTransaction();
+
+ gkeyTested = true;
+ }
+ }
+
+ private static void verifyDECase2(int gKeyId, int ptnKeyId, int gkeyOtableId, String gkeyValue)
+ throws OwlException {
+ List<PartitionEntity> listOfTopLevelPartitions = backend.find(PartitionEntity.class,
+ "owlTableId = " + gkeyOtableId
+ + " and ["+ PTN_A_STRING+"] = \""+ABCD+"\""
+ + " and parentPartitionId is null"
+ );
+
+ assertTrue(listOfTopLevelPartitions.size() == 1);
+ PartitionEntity abcdPartition = listOfTopLevelPartitions.get(0);
+ assertTrue(abcdPartition.getId() > 0);
+
+ List<KeyValueEntity> abcdKeyValues = abcdPartition.getKeyValues();
+ assertTrue(abcdKeyValues.size() == 2); // one ptnkey-value, one gkey-value
+
+ int keyValuesStillToCheck = 2;
+ for (KeyValueEntity kve : abcdKeyValues){
+ System.out.println(
+ " otable["+ kve.getOwlTableId()
+ + "] ptn[" + kve.getPartition().getId()
+ + "] ptnKeyId[" + nullGuardedPrint(kve.getPartitionKeyId())
+ + "] propKeyId[" + nullGuardedPrint(kve.getPropertyKeyId())
+ + "] globalKeyId[" + nullGuardedPrint(kve.getGlobalKeyId())
+ + "] stringValue[" + nullGuardedPrint(kve.getStringValue())
+ + "] intValue[" + nullGuardedPrint(kve.getIntValue())
+ + "]"
+ );
+ if (kve.getGlobalKeyId() != null){
+ assertTrue((kve.getGlobalKeyId() == gKeyId) && kve.getStringValue().equals(gkeyValue));
+ keyValuesStillToCheck--;
+ }
+ if (kve.getPartitionKeyId() != null){
+ assertTrue((kve.getPartitionKeyId() == ptnKeyId) && kve.getStringValue().equals(ABCD));
+ keyValuesStillToCheck--;
+ }
+ }
+ assertTrue(keyValuesStillToCheck == 0);
+ }
+
+ private static void cleanup() throws OwlException {
+
+ try {
+ backend.beginTransaction();
+ List<DatabaseEntity> databases = backend.find(DatabaseEntity.class, "name = \"" + TESTDATABASE_NAME + "\"");
+ if( databases.size() == 1 ) {
+ int catId = databases.get(0).getId();
+ backend.delete(OwlTableEntity.class, "databaseId = " + catId + " and name = \""+TESTOWLTABLE_NAME+"\"");
+ backend.delete(DatabaseEntity.class, "name = \""+TESTDATABASE_NAME+"\"");
+ }
+
+ backend.commitTransaction();
+
+ } catch(Exception e) {
+ e.printStackTrace();
+ assertNull(e); // An error during cleanup is critical, should not be ignored, is a failure.
+ }
+
+ gkeyBasicTested = false;
+
+ databaseCreated = false; databaseId = 0;
+ owlTableCreated = false; otableId = 0;
+ dePublished = false; deSelected = false;
+ deAltered1 = false;
+ deAltered2 = false;
+ deAltered3 = false;
+
+ ptnAStringKeyId = 0; ptnBIntegerKeyId = 0;
+ propAStringKeyId = 0; propBIntegerKeyId = 0;
+ }
+
+ private static String nullGuardedPrint(Object o) {
+ if (o != null) {
+ return o.toString();
+ }else{
+ return "null";
+ }
+ }
+
+}
Added: hadoop/pig/trunk/contrib/owl/java/test/org/apache/hadoop/owl/logical/TestSchemaUtil.java
URL: http://svn.apache.org/viewvc/hadoop/pig/trunk/contrib/owl/java/test/org/apache/hadoop/owl/logical/TestSchemaUtil.java?rev=934649&view=auto
==============================================================================
--- hadoop/pig/trunk/contrib/owl/java/test/org/apache/hadoop/owl/logical/TestSchemaUtil.java (added)
+++ hadoop/pig/trunk/contrib/owl/java/test/org/apache/hadoop/owl/logical/TestSchemaUtil.java Thu Apr 15 23:56:44 2010
@@ -0,0 +1,262 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.owl.logical;
+
+import static org.junit.Assert.*;
+
+import java.util.ArrayList;
+import java.util.List;
+
+import org.apache.hadoop.owl.common.ErrorType;
+import org.apache.hadoop.owl.common.OwlException;
+import org.junit.Test;
+
+public class TestSchemaUtil {
+
+ class ValidityTestCase {
+ String schema;
+ boolean intendedToSucceed;
+
+ public ValidityTestCase(String schema, boolean intendedToSucceed){
+ this.schema = schema;
+ this.intendedToSucceed = intendedToSucceed;
+ }
+
+ public void testValidate() throws OwlException{
+ try {
+ SchemaUtil.validateSchema(schema);
+ if (! intendedToSucceed){
+ // test case was supposed to fail, this is an error.
+ fail("Validate negative testcase ["+schema+"] validated. This is a failure");
+ }else{
+ System.out.println("Validate positive testcase ["+schema+"] validated. This is correct behaviour.");
+ }
+ } catch (OwlException e){
+ if (intendedToSucceed){
+ // test case was supposed to succeed, this is an error.
+ fail("Validate positive testcase ["+schema+"] failed validation. This is a failure");
+ }else{
+ // test case was supposed to fail, this is good.
+ if ((e.getErrorType() != ErrorType.ZEBRA_SCHEMA_EXCEPTION) &&
+ (e.getErrorType() != ErrorType.ZEBRA_TOKENMRGERROR_EXCEPTION)){
+ throw e; // rethrow exception, we don't know what this failure was, we let junit error on it.
+ }else{
+ System.out.println(
+ "Validate negative testcase [" + schema
+ + "] failed throwing [" + e.getMessage()
+ + ". This is correct behaviour."
+ );
+ }
+ }
+ }
+ }
+ };
+
+ class ConsolidationTestCase {
+ String baseSchema;
+ String schemaToAppend;
+ String expectedConsolidatedSchema;
+ boolean intendedToSucceed;
+
+ public ConsolidationTestCase(String baseSchema, String schemaToAppend, boolean intendedToSucceed, String expectedConsolidatedSchema){
+ this.baseSchema = baseSchema;
+ this.schemaToAppend = schemaToAppend;
+ this.expectedConsolidatedSchema = expectedConsolidatedSchema;
+ this.intendedToSucceed = intendedToSucceed;
+ }
+
+ public void testConsolidate() throws OwlException{
+ try {
+ String consolidatedSchema = SchemaUtil.consolidateSchema(baseSchema, schemaToAppend);
+ if (! intendedToSucceed){
+ // test case was supposed to fail, this is an error.
+ fail(
+ "Consolidate negative testcase ["
+ + baseSchema + "] * [" + schemaToAppend
+ + "] did not throw an exception, and consolidated to ["
+ + consolidatedSchema + "]. This is a failure."
+ );
+ }else{
+ if (consolidatedSchema.equals(expectedConsolidatedSchema)){
+ System.out.println(
+ "Consolidate positive testcase ["
+ + baseSchema+"] * ["+schemaToAppend
+ + "] => ["+expectedConsolidatedSchema+"] succeeded."
+ );
+ } else {
+ fail(
+ "Consolidate positive testcase ["
+ + baseSchema + "] * [" + schemaToAppend
+ + "] => [" + expectedConsolidatedSchema
+ + "] resulted in different consolidated result ["+consolidatedSchema+"]"
+ );
+ }
+ }
+ } catch (OwlException e){
+ if (intendedToSucceed){
+ // test case was supposed to succeed, this is an error.
+ fail(
+ "Consolidate positive testcase ["
+ + baseSchema + "] * [" + schemaToAppend
+ + "] => ["+expectedConsolidatedSchema+"] failed consolidation. This is a failure"
+ );
+ }else{
+ // test case was supposed to fail, this is good.
+ if ((e.getErrorType() != ErrorType.ZEBRA_SCHEMA_EXCEPTION) &&
+ (e.getErrorType() != ErrorType.ZEBRA_TOKENMRGERROR_EXCEPTION)){
+ throw e; // rethrow exception, we don't know what this failure was, we let junit error on it.
+ }else{
+ System.out.println(
+ "Consolidate negative testcase ["
+ + baseSchema + "] * [" + schemaToAppend
+ + "] failed throwing [" + e.getMessage()
+ + "]. This is correct behaviour."
+ );
+ }
+ }
+ }
+ }
+ };
+
+
+ /**
+ * Test method for {@link org.apache.hadoop.owl.logical.SchemaUtil#validateSchema(java.lang.String)}.
+ * @throws OwlException
+ */
+ @Test
+ public void testValidateSchema() throws OwlException {
+
+ List<ValidityTestCase> schemaValidityTestCases = new ArrayList<ValidityTestCase>();
+
+ // Trivial cases
+ schemaValidityTestCases.add(new ValidityTestCase(null,true));
+ // In reality, expected false, but understandable
+ schemaValidityTestCases.add(new ValidityTestCase("",true));
+ // In reality, expected false, but understandable
+ schemaValidityTestCases.add(new ValidityTestCase("c1",true));
+ // works out to "c1:bytes" if type is not specified
+
+ // Basic types
+ schemaValidityTestCases.add(new ValidityTestCase("c1:integer",false)); // "integer" is not a recognized type
+ schemaValidityTestCases.add(new ValidityTestCase("c1:int",true));
+ schemaValidityTestCases.add(new ValidityTestCase("c1:INT",true)); // capitalizing still works
+
+ schemaValidityTestCases.add(new ValidityTestCase("c1:bool",true));
+ schemaValidityTestCases.add(new ValidityTestCase("c1:long",true));
+ schemaValidityTestCases.add(new ValidityTestCase("c1:float",true));
+ schemaValidityTestCases.add(new ValidityTestCase("c1:double",true));
+ schemaValidityTestCases.add(new ValidityTestCase("c1:string",true));
+ schemaValidityTestCases.add(new ValidityTestCase("c1:bytes",true));
+
+ // mixed form, specifying type for some columns, not for others(thus making them default to "bytes"
+ schemaValidityTestCases.add(new ValidityTestCase("c1,c2:bool",true));
+
+ // column name clashes
+ schemaValidityTestCases.add(new ValidityTestCase("c1:bool,c1:bool",false));
+ schemaValidityTestCases.add(new ValidityTestCase("c1:bool,c1:int",false));
+
+ // Complex types : record
+ schemaValidityTestCases.add(new ValidityTestCase("c1:record(a:int,b:string)",true));
+
+ // Complex type : collection
+ schemaValidityTestCases.add(new ValidityTestCase("c1:collection(r1:record(a:int,b:string))",true));
+ schemaValidityTestCases.add(new ValidityTestCase("c1:collection(r1:record(a))",true)); // will default to a:bytes inside
+
+ // testing sub-column name clash potential
+ schemaValidityTestCases.add(new ValidityTestCase("c1:collection(r1:record(f1:int,f2:int)),c2:collection(r2:record(f1:int,f2:int))",true));
+
+ // Complex types : map
+ schemaValidityTestCases.add(new ValidityTestCase("c1:map(int)",true));
+ // key type is always string and not specified
+ schemaValidityTestCases.add(new ValidityTestCase("c1:map<int>",false));
+ schemaValidityTestCases.add(new ValidityTestCase("c1:map(string,int)",false));
+
+ // Complex type combinatorics
+ schemaValidityTestCases.add(new ValidityTestCase(
+ "c1:collection(r1:record(a:int,b:int)),b:map(collection(r2:record(a:int,b:int)))",true));
+ schemaValidityTestCases.add(new ValidityTestCase("c1:map(map(map(map(string))))",true));
+ schemaValidityTestCases.add(new ValidityTestCase("c1:record(a:record(a:record(a:record(a:string))))",true));
+ schemaValidityTestCases.add(new ValidityTestCase("c1:record(record(record(record(a:string))))",false));
+ schemaValidityTestCases.add(new ValidityTestCase("c1:collection(r1:record(a:collection(r2:record(a:collection(r3:record(a:collection(r4:record(a:string))))))))",true));
+ // test sub schema as a RECORD type within COLLECTION without a column name
+ schemaValidityTestCases.add(new ValidityTestCase("c1:collection(record(a:collection(record(a:collection(record(a:collection(record(a:string))))))))",true));
+ schemaValidityTestCases.add(new ValidityTestCase("c1:collection(record(record(record(a:string))))",false));
+ schemaValidityTestCases.add(new ValidityTestCase("c1:collection(record(r1:record(r2:record(a:string))))",true));
+
+
+ for (ValidityTestCase testcase : schemaValidityTestCases){
+ testcase.testValidate();
+ }
+ }
+
+ /**
+ * Test method for {@link org.apache.hadoop.owl.logical.SchemaUtil#consolidateSchema(java.lang.String, java.lang.String)}.
+ * @throws OwlException
+ */
+ @Test
+ public void testConsolidateSchema() throws OwlException {
+ List<ConsolidationTestCase> schemaConsolidationTestCases = new ArrayList<ConsolidationTestCase>();
+
+ // Basic trivial cases
+ schemaConsolidationTestCases.add(new ConsolidationTestCase("blah","yikes",true,"blah:bytes,yikes:bytes"));
+ schemaConsolidationTestCases.add(new ConsolidationTestCase("blah",null,true,"blah"));
+ schemaConsolidationTestCases.add(new ConsolidationTestCase("blah","",true,"blah:bytes"));
+ schemaConsolidationTestCases.add(new ConsolidationTestCase("","yikes",true,"yikes:bytes"));
+ schemaConsolidationTestCases.add(new ConsolidationTestCase(null,"yikes",true,"yikes"));
+
+ // Consolidation cases
+ schemaConsolidationTestCases.add(new ConsolidationTestCase("c1:int","c2:string",true,"c1:int,c2:string"));
+
+ // Same column-names
+ schemaConsolidationTestCases.add(new ConsolidationTestCase("c1:int","c1:string",false,null));
+ schemaConsolidationTestCases.add(new ConsolidationTestCase("c1:int,c2:string","c2:int",false,null));
+
+ // Consolidation with subset
+ // User case: suppose partition has fewer columns, no change to table level schema
+ schemaConsolidationTestCases.add(new ConsolidationTestCase("c1:int,c2:string","c2:string",true,"c1:int,c2:string"));
+ schemaConsolidationTestCases.add(new ConsolidationTestCase("c1:int,c2:int","c2:int",true,"c1:int,c2:int"));
+ schemaConsolidationTestCases.add(new ConsolidationTestCase("c1:int,c2:int","",true,"c1:int,c2:int"));
+ schemaConsolidationTestCases.add(new ConsolidationTestCase("c1:int,c2:int",null,true,"c1:int,c2:int"));
+ schemaConsolidationTestCases.add(new ConsolidationTestCase("c1:int,c2:int","c2:int,c3:int",true,"c1:int,c2:int,c3:int"));
+
+ // Consolidation with subset interesting cases because of missing type defaulting to bytes
+ schemaConsolidationTestCases.add(new ConsolidationTestCase("c1,c2:int","c2",false,null));
+ schemaConsolidationTestCases.add(new ConsolidationTestCase("c1,c2:int","c1:int",false,null));
+ schemaConsolidationTestCases.add(new ConsolidationTestCase("c1,c2:int","c2:int",true,"c1:bytes,c2:int"));
+ schemaConsolidationTestCases.add(new ConsolidationTestCase("c1,c2:int","",true,"c1:bytes,c2:int"));
+ schemaConsolidationTestCases.add(new ConsolidationTestCase("c1,c2:int",null,true,"c1,c2:int"));
+
+ // Consolidation inside complex columns
+ schemaConsolidationTestCases.add(new ConsolidationTestCase("c1:collection(r1:record(f1:int,f2:int))","c1:collection(r1:record(f1:int,f2:int,f3:int))",false,null));
+ // Note : This was an interesting case - actually expected "c1:collection(f1:int,f2:int,f3:int)"
+ // but schema consolidation does not happen inside complex nested types. Equality is checked for the purpose
+ // of top-level consolidation, but that's the extent of it.
+
+ // Case-sensitivity
+ schemaConsolidationTestCases.add(new ConsolidationTestCase("c1:int,c2:string","C2:string",true,"c1:int,c2:string,C2:string"));
+ schemaConsolidationTestCases.add(new ConsolidationTestCase("c1:int,c2:string","c2:STRING",true,"c1:int,c2:string"));
+
+
+ for (ConsolidationTestCase testcase : schemaConsolidationTestCases){
+ testcase.testConsolidate();
+ }
+
+ }
+
+}
Added: hadoop/pig/trunk/contrib/owl/java/test/org/apache/hadoop/owl/mapreduce/TestIsFeatureSupported.java
URL: http://svn.apache.org/viewvc/hadoop/pig/trunk/contrib/owl/java/test/org/apache/hadoop/owl/mapreduce/TestIsFeatureSupported.java?rev=934649&view=auto
==============================================================================
--- hadoop/pig/trunk/contrib/owl/java/test/org/apache/hadoop/owl/mapreduce/TestIsFeatureSupported.java (added)
+++ hadoop/pig/trunk/contrib/owl/java/test/org/apache/hadoop/owl/mapreduce/TestIsFeatureSupported.java Thu Apr 15 23:56:44 2010
@@ -0,0 +1,162 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.owl.mapreduce;
+
+import org.apache.hadoop.owl.OwlTestCase;
+import org.apache.hadoop.owl.protocol.OwlLoaderInfo;
+import org.apache.hadoop.owl.mapreduce.OwlPartitionValues;
+import org.apache.hadoop.owl.protocol.OwlSchema;
+import org.apache.hadoop.io.BytesWritable;
+import org.apache.hadoop.mapreduce.InputFormat;
+import org.apache.hadoop.mapreduce.JobContext;
+import org.apache.pig.data.Tuple;
+
+import java.util.Arrays;
+import java.util.List;
+import java.util.ArrayList;
+import java.io.IOException;
+
+import org.junit.Test;
+import static org.mockito.Mockito.*;
+
+/**
+ * Dummy implementation of an abstract storage driver class
+ * that reports that it DOES support all features
+ */
+class OwlInputYesSD extends OwlInputStorageDriver {
+ public InputFormat<BytesWritable, Tuple> getInputFormat(OwlLoaderInfo loaderInfo) { return null; }
+ public boolean isFeatureSupported(OwlInputFormat.OwlOperation operation) throws IOException { return true; }
+ public void setInputPath(JobContext jobContext, String location) throws IOException {}
+ public void setOriginalSchema(JobContext jobContext, OwlSchema schema) throws IOException {}
+ public void setOutputSchema(JobContext jobContext, OwlSchema schema) throws IOException {}
+ public void setPartitionValues(JobContext jobContext, OwlPartitionValues partitionValues) throws IOException {}
+
+}
+
+/**
+ * Dummy implementation of an abstract storage driver class
+ * that reports that it DOES NOT support all features
+ */
+class OwlInputNoSD extends OwlInputStorageDriver {
+ public InputFormat<BytesWritable, Tuple> getInputFormat(OwlLoaderInfo loaderInfo) { return null; }
+ public boolean isFeatureSupported(OwlInputFormat.OwlOperation operation) throws IOException { return false; }
+ public void setInputPath(JobContext jobContext, String location) throws IOException {}
+ public void setOriginalSchema(JobContext jobContext, OwlSchema schema) throws IOException {}
+ public void setOutputSchema(JobContext jobContext, OwlSchema schema) throws IOException {}
+ public void setPartitionValues(JobContext jobContext, OwlPartitionValues partitionValues) throws IOException {}
+}
+
+/**
+ * Test OwlInputFormat.isFeatureSupported():
+ *
+ * Create several mocked partitions with storage drivers supporting
+ * or not supporting given feature, assemble partitions into a mock
+ * OwlJobInfo and call OwlInputFormat.isFeatureSupported()
+ */
+
+public class TestIsFeatureSupported extends OwlTestCase {
+
+ /**
+ * Iterate over all OwlOperations (currently only one)
+ * and test isFeatureSupported() consolidation logic on each
+ */
+ @Test
+ public static void testIsFeatureSupported() {
+ for (OwlInputFormat.OwlOperation feature: OwlInputFormat.OwlOperation.values()) {
+ testFeature(feature);
+ }
+ }
+
+ /**
+ * Test OwlInputFormat.isFeatureSupported() for a given feature by constructing
+ * the OwlJobInfo with the following permutations of partitions supporting (true)
+ * and not supporting (false) the feature:
+ *
+ * 1. true
+ * 2. false
+ * 3. true, true, true
+ * 4. true, false, true
+ * 5. false, false, false
+ *
+ * @param feature OwlOperation to be tested
+ */
+ private static void testFeature(OwlInputFormat.OwlOperation feature) {
+ boolean[] list1 = {true};
+ testCombination(feature, list1, true);
+ boolean[] list2 = {false};
+ testCombination(feature, list2, false);
+ boolean[] list3 = {true, true, true};
+ testCombination(feature, list3, true);
+ boolean[] list4 = {true, false, true};
+ testCombination(feature, list4, false);
+ boolean[] list5 = {false, false, false};
+ testCombination(feature, list5, false);
+ }
+
+ /**
+ * Test a given combination of support features.
+ * @see #testFeature
+ *
+ * Use OwlInputYesSD and OwlInputNoSD classes and construct OwlTableInputInfo
+ * object with a list of partitions with specified feature support.
+ * Then - check if OwlInputFormat.isFeatureSupported() returns correct result
+ *
+ * @param feature OwlOperation to be tested
+ * @param list combination to be tested
+ * @param result expected result
+ */
+ private static void testCombination(OwlInputFormat.OwlOperation feature, boolean[] list, boolean result) {
+ /*
+ * Mock a list of OwlPartitionInfo given a specified list of partitions
+ */
+ List<OwlPartitionInfo> opiList = new ArrayList<OwlPartitionInfo>();
+ for (boolean aList : list) {
+ /*
+ * Mock OwlPartitionInfo -> OwlLoaderInfo -> driverClass name
+ */
+ OwlLoaderInfo oli = mock(OwlLoaderInfo.class);
+ String driverClass = (aList ? OwlInputYesSD.class : OwlInputNoSD.class).getName();
+ when(oli.getInputDriverClass()).thenReturn(driverClass);
+ OwlPartitionInfo opi = mock(OwlPartitionInfo.class);
+ when(opi.getLoaderInfo()).thenReturn(oli);
+ opiList.add(opi);
+ }
+
+ /*
+ * Mock OwlTableInputInfo -> OwlJobInfo -> partition list
+ */
+ OwlJobInfo jobInfo = mock(OwlJobInfo.class);
+ when(jobInfo.getPartitions()).thenReturn(opiList);
+ OwlTableInputInfo inputInfo = mock(OwlTableInputInfo.class);
+ when(inputInfo.getJobInfo()).thenReturn(jobInfo);
+
+ /*
+ * Now check if OwlInputFormat.isFeatureSupported() matches provided outcome
+ */
+ String callName = "OwlInputFormat.isFeatureSupported(partitions:";
+ try {
+ System.out.println(callName + Arrays.toString(list) + ", " + feature.toString() + ") == " + result);
+ assertEquals(callName + Arrays.toString(list) + ", " + feature.toString() + ") failed:",
+ OwlInputFormat.isFeatureSupported(inputInfo, feature), result);
+ } catch (IOException e) {
+ fail(callName + Arrays.toString(list) + ", " + feature.toString() + ") throws "+e.getMessage());
+ }
+ }
+
+}
+
Added: hadoop/pig/trunk/contrib/owl/java/test/org/apache/hadoop/owl/mapreduce/TestKeyPushDown.java
URL: http://svn.apache.org/viewvc/hadoop/pig/trunk/contrib/owl/java/test/org/apache/hadoop/owl/mapreduce/TestKeyPushDown.java?rev=934649&view=auto
==============================================================================
--- hadoop/pig/trunk/contrib/owl/java/test/org/apache/hadoop/owl/mapreduce/TestKeyPushDown.java (added)
+++ hadoop/pig/trunk/contrib/owl/java/test/org/apache/hadoop/owl/mapreduce/TestKeyPushDown.java Thu Apr 15 23:56:44 2010
@@ -0,0 +1,149 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.owl.mapreduce;
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Map;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.mapreduce.Job;
+
+import org.apache.hadoop.owl.OwlTestCase;
+import org.apache.hadoop.owl.common.ErrorType;
+import org.apache.hadoop.owl.common.OwlException;
+import org.apache.hadoop.owl.driver.DriverVerificationUtil;
+import org.apache.hadoop.owl.driver.OwlDriver;
+import org.apache.hadoop.owl.driver.OwlTableBuilder;
+import org.apache.hadoop.owl.protocol.ColumnType;
+import org.apache.hadoop.owl.protocol.OwlColumnSchema;
+import org.apache.hadoop.owl.protocol.OwlDatabase;
+import org.apache.hadoop.owl.protocol.OwlKeyValue;
+import org.apache.hadoop.owl.protocol.OwlLoaderInfo;
+import org.apache.hadoop.owl.protocol.OwlSchema;
+import org.apache.hadoop.owl.protocol.OwlTable;
+import org.apache.hadoop.owl.protocol.OwlTableName;
+import org.apache.hadoop.owl.protocol.OwlKey.DataType;
+
+import org.junit.Test;
+
+/**
+ * Testing BasicTableOutputFormat and TableInputFormat using Local FS
+ */
+public class TestKeyPushDown extends OwlTestCase {
+
+ private static final String OWL_KEY_JOB_INFO= "mapreduce.lib.owl.job.info";
+ private static String dbname = "testkeypushdowndb";
+ private static String tableName = "testkeypushdowntab";
+ private static String partitionLocation = "test/keypushdown/partition";
+ private static OwlTableName name = new OwlTableName(dbname, tableName);
+ private static OwlDriver driver;
+ static List<OwlKeyValue> keyValues;
+ static OwlKeyValue k1, k2;
+
+ private static boolean initialized = false;
+ private static int numTestsRemaining = 1;
+
+ @SuppressWarnings("boxing")
+ static void publish(int v1, String v2, String location) throws OwlException {
+ k1.setIntValue(v1);
+ k2.setStringValue(v2);
+
+ OwlSchema schema1 = new OwlSchema();
+ schema1.addColumnSchema(new OwlColumnSchema("c1", ColumnType.INT));
+ schema1.addColumnSchema(new OwlColumnSchema("c2", ColumnType.STRING));
+ driver.publish(name, keyValues, null, location, schema1, new OwlLoaderInfo("org.apache.hadoop.zebra.owl.ZebraInputDriver"));
+ }
+
+ public static void initialize() throws IOException {
+ if (!initialized){
+ driver = new OwlDriver(getUri());
+
+ DriverVerificationUtil.dropOwlTableIfExists(driver, name);
+ DriverVerificationUtil.dropOwlDatabaseIfExists(driver, dbname);
+
+ driver.createOwlDatabase(new OwlDatabase(dbname, "hdfs://localhost:9000/data/testdatabase"));
+
+ OwlSchema schema = new OwlSchema();
+ schema.addColumnSchema(new OwlColumnSchema("c1", ColumnType.INT));
+
+ OwlTable table = new OwlTableBuilder().
+ setName(name).
+ addPartition("c1", DataType.INT).
+ addPartition("c2", DataType.STRING).
+ setSchema(schema).
+ build();
+ driver.createOwlTable(table);
+ keyValues = new ArrayList<OwlKeyValue>();
+ k1 = new OwlKeyValue("c1", 2008);
+ k2 = new OwlKeyValue("c2", "us");
+
+ keyValues.add(k1);
+ keyValues.add(k2);
+ publish(2008, "us", partitionLocation + "1");
+ publish(2008, "uk", partitionLocation + "2");
+
+ initialized = true;
+ }
+ }
+
+ //simple counter for local mode testing
+ static OwlJobInfo getJobInfo(String filter) throws Exception {
+ Configuration conf = new Configuration();
+ Job job = new Job(conf, "keypushdown test");
+ job.setInputFormatClass(OwlInputFormat.class); // input settings
+
+ OwlTableName tabName = new OwlTableName(dbname, tableName); //Specify server url, db name, table name and partition predicate through OwlTableInput object
+ OwlTableInputInfo inputInfo = new OwlTableInputInfo(getUri(), tabName, filter);
+ OwlInputFormat.setInput(job, inputInfo);
+ String jobString = job.getConfiguration().get(OWL_KEY_JOB_INFO);
+ if( jobString == null ) {
+ throw new OwlException(ErrorType.ERROR_INPUT_UNINITIALIZED);
+ }
+ return (OwlJobInfo) SerializeUtil.deserialize(jobString);
+ }
+
+
+ @Test
+ public static void testKeyPushed() throws Exception {
+ initialize();
+ OwlJobInfo jobInfo = getJobInfo("c1 = 2008 and c2 = \"us\"");
+ assertEquals(1, jobInfo.getPartitions().size());
+
+ OwlPartitionInfo partitionInfo = jobInfo.getPartitions().get(0);
+ Map<String, OwlKeyValue> map = partitionInfo.getPartitionValues().getPartitionMap();
+ assertEquals(2, map.size());
+
+ OwlKeyValue k1 = map.get("c1");
+ assertEquals((Integer) 2008, k1.getIntValue());
+
+ OwlKeyValue k2 = map.get("c2");
+ assertEquals("us", k2.getStringValue());
+ cleanup();
+ }
+
+ public static void cleanup() throws IOException {
+ numTestsRemaining--;
+ if (numTestsRemaining == 0){
+ driver.dropOwlTable(new OwlTable(name));
+ driver.dropOwlDatabase(new OwlDatabase(dbname, null));
+ }
+ }
+}
+
Added: hadoop/pig/trunk/contrib/owl/java/test/org/apache/hadoop/owl/mapreduce/TestNonPartitioned.java
URL: http://svn.apache.org/viewvc/hadoop/pig/trunk/contrib/owl/java/test/org/apache/hadoop/owl/mapreduce/TestNonPartitioned.java?rev=934649&view=auto
==============================================================================
--- hadoop/pig/trunk/contrib/owl/java/test/org/apache/hadoop/owl/mapreduce/TestNonPartitioned.java (added)
+++ hadoop/pig/trunk/contrib/owl/java/test/org/apache/hadoop/owl/mapreduce/TestNonPartitioned.java Thu Apr 15 23:56:44 2010
@@ -0,0 +1,190 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.owl.mapreduce;
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.LocalFileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.io.BytesWritable;
+import org.apache.hadoop.mapreduce.Job;
+import org.apache.hadoop.mapreduce.Mapper;
+import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;
+
+import org.apache.hadoop.owl.OwlTestCase;
+import org.apache.hadoop.owl.common.OwlException;
+import org.apache.hadoop.owl.driver.OwlDriver;
+import org.apache.hadoop.owl.driver.OwlTableBuilder;
+import org.apache.hadoop.owl.protocol.ColumnType;
+import org.apache.hadoop.owl.protocol.OwlColumnSchema;
+import org.apache.hadoop.owl.protocol.OwlDatabase;
+import org.apache.hadoop.owl.protocol.OwlKeyValue;
+import org.apache.hadoop.owl.protocol.OwlLoaderInfo;
+import org.apache.hadoop.owl.protocol.OwlSchema;
+import org.apache.hadoop.owl.protocol.OwlTable;
+import org.apache.hadoop.owl.protocol.OwlTableName;
+import org.apache.hadoop.owl.protocol.OwlKey.DataType;
+import org.apache.hadoop.owl.testdriver.CompareOwlObjects;
+import org.apache.hadoop.owl.testdriver.SampleDriverStub;
+import org.apache.hadoop.owl.testdriver.StorageDriverStub;
+
+import org.apache.pig.data.Tuple;
+
+import org.junit.Test;
+
+/**
+ * Test case for non partitioned owltable access through OwlInputFormat
+ */
+public class TestNonPartitioned extends OwlTestCase {
+
+ private String dbname = "testnonpartOIFdb";
+ private String tableName = "testnonpartOIFTable";
+ private String dataLocation = "test/TestNonPartitioned";
+
+
+ private OwlSchema schema;
+
+ private OwlTableName name = new OwlTableName(dbname, tableName);
+ private OwlDriver driver;
+
+ private StorageDriverStub stub;
+
+ public TestNonPartitioned() {
+ //Only for junit to use
+ }
+
+ public TestNonPartitioned(StorageDriverStub stub) {
+ this.stub = stub;
+ }
+
+ @SuppressWarnings("boxing")
+ void publish(String location, OwlSchema schema) throws OwlException {
+ driver.publish(name, null, null, location, schema, new OwlLoaderInfo(stub.getDriverClass().getName()));
+ }
+
+ static volatile int rowCount = 0;
+ static Object lockObject = new Object();
+
+ public static class MapClass extends
+ Mapper<BytesWritable, Tuple, BytesWritable, Tuple> {
+
+ @Override
+ public void map(BytesWritable key, Tuple value, Context context
+ ) throws IOException, InterruptedException {
+ {
+ //System.out.println("In map function -> key " + key.toString() + " value " + value.toString());
+ context.write(key, value);
+ synchronized (lockObject) {
+ rowCount++;
+ }
+ }
+ }
+ }
+
+ void runMRTest(int expectedRowCount) throws Exception {
+
+ Configuration conf = new Configuration();
+ Job job = new Job(conf, "non partitioned test");
+ job.setJarByClass(TestNonPartitioned.class);
+ job.setMapperClass(TestNonPartitioned.MapClass.class);
+
+ // input settings
+ job.setInputFormatClass(OwlInputFormat.class);
+ job.setOutputKeyClass(BytesWritable.class);
+ job.setOutputValueClass(stub.getOutputTupleClass());
+
+ //Specify server url, db name, table name and partition predicate through OwlTableInput object
+ OwlTableName tabName = new OwlTableName(dbname, tableName);
+ OwlTableInputInfo inputInfo = new OwlTableInputInfo(getUri(), tabName, null);
+ OwlInputFormat.setInput(job, inputInfo);
+
+
+ FileSystem fs = new LocalFileSystem();
+
+ stub.dropData(new Path(fs.getWorkingDirectory(), dataLocation + "/outputfile"));
+ FileOutputFormat.setOutputPath(job, new Path(dataLocation + "/outputfile"));
+
+ rowCount = 0;
+ job.waitForCompletion(true);
+
+ System.out.println("Filter " + null + ", Map job got " + rowCount + " rows");
+ assertEquals(expectedRowCount, rowCount);
+ }
+
+ private void cleanup(boolean ignoreException) throws Exception {
+ try {
+ driver.dropOwlTable(new OwlTable(name));
+ } catch(Exception e){ if( ! ignoreException ) throw e; }
+ try {
+ driver.dropOwlDatabase(new OwlDatabase(dbname, null));
+ } catch(Exception e){ if( ! ignoreException ) throw e; }
+ try {
+ FileSystem fs = new LocalFileSystem();
+ stub.dropData(new Path(fs.getWorkingDirectory(), dataLocation));
+ }catch(Exception e){ if( ! ignoreException ) throw e; }
+ }
+
+ public void initialize() throws Exception {
+ FileSystem fs = new LocalFileSystem();
+
+ driver = new OwlDriver(getUri());
+ cleanup(true);
+
+ driver.createOwlDatabase(new OwlDatabase(dbname, "hdfs://localhost:9000/data/testNonPartitioned"));
+
+ schema = new OwlSchema();
+ schema.addColumnSchema(new OwlColumnSchema("k1", ColumnType.INT));
+ schema.addColumnSchema(new OwlColumnSchema("k2", ColumnType.STRING));
+
+ OwlTable table = new OwlTableBuilder().
+ setName(name).
+ setSchema(schema).
+ build();
+
+ driver.createOwlTable(table);
+
+
+ stub.createRandomData(new Path(fs.getWorkingDirectory(), dataLocation), dataLocation, 1, 11, schema);
+
+ publish(dataLocation, schema);
+ }
+
+ public void nonPartitionedTest() throws Exception {
+ runMRTest(11);
+ }
+
+ public static void runTest(StorageDriverStub stub) throws Exception {
+ TestNonPartitioned test = new TestNonPartitioned(stub);
+
+ test.initialize();
+ test.nonPartitionedTest();
+ test.cleanup(false);
+ }
+
+ @Test
+ public static void testSampleStorageDriverNonPartitioned() throws Exception {
+ runTest(new SampleDriverStub());
+ }
+}
+
+
+
Added: hadoop/pig/trunk/contrib/owl/java/test/org/apache/hadoop/owl/mapreduce/TestOwlSplit.java
URL: http://svn.apache.org/viewvc/hadoop/pig/trunk/contrib/owl/java/test/org/apache/hadoop/owl/mapreduce/TestOwlSplit.java?rev=934649&view=auto
==============================================================================
--- hadoop/pig/trunk/contrib/owl/java/test/org/apache/hadoop/owl/mapreduce/TestOwlSplit.java (added)
+++ hadoop/pig/trunk/contrib/owl/java/test/org/apache/hadoop/owl/mapreduce/TestOwlSplit.java Thu Apr 15 23:56:44 2010
@@ -0,0 +1,115 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.owl.mapreduce;
+
+import java.io.ByteArrayInputStream;
+import java.io.ByteArrayOutputStream;
+import java.io.DataInput;
+import java.io.DataInputStream;
+import java.io.DataOutput;
+import java.io.DataOutputStream;
+import java.io.IOException;
+import org.apache.hadoop.io.Writable;
+import org.apache.hadoop.mapreduce.InputSplit;
+import org.apache.hadoop.owl.OwlTestCase;
+import org.apache.hadoop.owl.common.OwlException;
+import org.apache.hadoop.owl.common.OwlUtil;
+import org.apache.hadoop.owl.protocol.ColumnType;
+import org.apache.hadoop.owl.protocol.OwlColumnSchema;
+import org.apache.hadoop.owl.protocol.OwlLoaderInfo;
+import org.apache.hadoop.owl.protocol.OwlSchema;
+import org.junit.Test;
+
+public class TestOwlSplit extends OwlTestCase{
+ // this private TestSplit class extends InputSplit
+ private static class TestSplit extends InputSplit implements Writable{
+
+ public TestSplit(){}
+
+ @Override
+ public long getLength() throws IOException, InterruptedException {
+ return 0;
+ }
+
+ @Override
+ public String[] getLocations() throws IOException, InterruptedException {
+ return null;
+ }
+
+ public void readFields(DataInput arg0) throws IOException {
+ }
+
+ public void write(DataOutput arg0) throws IOException {
+ }
+ }
+
+ private static OwlSplit owlSplit;
+
+ public TestOwlSplit() {
+ }
+
+
+ @Test
+ public static void testSerialization() throws OwlException {
+
+ OwlSchema partitionSchema = new OwlSchema();
+ partitionSchema.addColumnSchema(new OwlColumnSchema("c1", ColumnType.STRING));
+
+ OwlSchema tableSchema = new OwlSchema();
+ tableSchema.addColumnSchema(new OwlColumnSchema("d1", ColumnType.STRING));
+
+ OwlLoaderInfo loaderInfo = new OwlLoaderInfo("zebraInputSplit");
+ String location = "hdfs://tmp";
+
+ OwlPartitionInfo partitionInfo = new OwlPartitionInfo(partitionSchema, loaderInfo, location);
+
+ TestSplit testSplit = new TestSplit();
+ InputSplit baseSplit = testSplit;
+
+ owlSplit = new OwlSplit(partitionInfo, baseSplit, tableSchema);
+
+ ByteArrayOutputStream bos = new ByteArrayOutputStream();
+ DataOutput output = new DataOutputStream(bos);
+
+ try {
+ // we need to first create an DataOutputStream for OwlSplit to write into
+ owlSplit.write(output);
+ // remember to close the DataOutputStream to make sure the data has been written
+ bos.close();
+
+ ByteArrayInputStream bis = new ByteArrayInputStream(bos.toByteArray());
+ DataInputStream dataInputStream = new DataInputStream(bis);
+
+ OwlSplit newOwlSplit = new OwlSplit();
+ newOwlSplit.readFields(dataInputStream);
+
+ // compare newOwlSplit and owlSplit
+ OwlPartitionInfo newPartitionInfo = newOwlSplit.getPartitionInfo();
+ // compare location string
+ assertEquals(newPartitionInfo.getLocation(), owlSplit.getPartitionInfo().getLocation());
+ // compare schemas
+ assertEquals(newPartitionInfo.getPartitionSchema(), (owlSplit.getPartitionInfo().getPartitionSchema()));
+ assertEquals(newOwlSplit.getTableSchema(), owlSplit.getTableSchema());
+
+ } catch (IOException e) {
+ e.printStackTrace();
+ }
+
+ }
+
+}
Added: hadoop/pig/trunk/contrib/owl/java/test/org/apache/hadoop/owl/mapreduce/TestPartitionKey.java
URL: http://svn.apache.org/viewvc/hadoop/pig/trunk/contrib/owl/java/test/org/apache/hadoop/owl/mapreduce/TestPartitionKey.java?rev=934649&view=auto
==============================================================================
--- hadoop/pig/trunk/contrib/owl/java/test/org/apache/hadoop/owl/mapreduce/TestPartitionKey.java (added)
+++ hadoop/pig/trunk/contrib/owl/java/test/org/apache/hadoop/owl/mapreduce/TestPartitionKey.java Thu Apr 15 23:56:44 2010
@@ -0,0 +1,251 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.owl.mapreduce;
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.LocalFileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.io.BytesWritable;
+import org.apache.hadoop.mapreduce.Job;
+import org.apache.hadoop.mapreduce.Mapper;
+import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;
+
+import org.apache.hadoop.owl.OwlTestCase;
+import org.apache.hadoop.owl.common.OwlException;
+import org.apache.hadoop.owl.driver.OwlDriver;
+import org.apache.hadoop.owl.driver.OwlTableBuilder;
+import org.apache.hadoop.owl.protocol.ColumnType;
+import org.apache.hadoop.owl.protocol.OwlColumnSchema;
+import org.apache.hadoop.owl.protocol.OwlDatabase;
+import org.apache.hadoop.owl.protocol.OwlKeyValue;
+import org.apache.hadoop.owl.protocol.OwlLoaderInfo;
+import org.apache.hadoop.owl.protocol.OwlSchema;
+import org.apache.hadoop.owl.protocol.OwlTable;
+import org.apache.hadoop.owl.protocol.OwlTableName;
+import org.apache.hadoop.owl.protocol.OwlKey.DataType;
+import org.apache.hadoop.owl.testdriver.StorageDriverStub;
+import org.apache.hadoop.owl.testdriver.SampleDriverStub;
+
+import org.apache.pig.data.DefaultTuple;
+import org.apache.pig.data.Tuple;
+
+import org.junit.Assert;
+import org.junit.Test;
+
+/**
+ * Test case which tests whether schema evolution works and partition key values are present in output.
+ * Creates two partitions, one with 2 columns and another with 3 columns, setOutputSchema is set to
+ * fetch two data columns and the two partition columns.
+ */
+public class TestPartitionKey extends OwlTestCase {
+
+ private String dbname = "testPVdb";
+ private String tableName = "testPVtable";
+ private String dataLocation = "test/TestPartitionValue";
+ private String partitionLocation1 = dataLocation + "/partition1";
+ private String partitionLocation2 = dataLocation + "/partition2";
+
+ private OwlTableName name = new OwlTableName(dbname, tableName);
+ private OwlDriver driver;
+
+ private List<OwlKeyValue> keyValues;
+ private OwlKeyValue k1, k2;
+ private StorageDriverStub stub;
+
+ public TestPartitionKey() {
+ //Only for junit to use
+ }
+
+ public TestPartitionKey(StorageDriverStub stub) {
+ this.stub = stub;
+ }
+
+ @SuppressWarnings("boxing")
+ void publish(int v1, String v2, String location, OwlSchema schema) throws OwlException {
+ k1.setIntValue(v1);
+ k2.setStringValue(v2);
+
+ driver.publish(name, keyValues, null, location, schema, new OwlLoaderInfo(stub.getDriverClass().getName()));
+ }
+
+ volatile int mapCounter = 0;
+ Object lockObject = new Object();
+
+ public class MapClass extends
+ Mapper<BytesWritable, Tuple, BytesWritable, Tuple> {
+
+ @Override
+ public void map(BytesWritable key, Tuple value, Context context
+ ) throws IOException, InterruptedException {
+ {
+ synchronized(lockObject) {
+ mapCounter++;
+ }
+
+ System.out.println("In map2 function -> key " + new String(key.getBytes()) + " value " + value.toString());
+ context.write(key, value);
+
+ Assert.assertEquals("tuple size", 4, value.size());
+ Object c2 = value.get(0);
+ Object p2 = value.get(1);
+ Object c3 = value.get(2);
+ Object p1 = value.get(3);
+
+ Assert.assertNotNull("c2 value", c2);
+ Assert.assertNotNull("p1 value", p1);
+ Assert.assertNotNull("p2 value", p2);
+
+ String keyString = new String(key.getBytes());
+ if( keyString.startsWith(partitionLocation1) ) {
+ //If first partition, c3 should be null and p2 should be "uk"
+ Assert.assertNull("c3 value", c3);
+ Assert.assertEquals("p1 value", 2008, p1);
+ Assert.assertEquals("p2 value", "us", p2);
+ } else if( keyString.startsWith(partitionLocation2) ) {
+ //If second partition, c3 should be non null and p2 should be "uk"
+ Assert.assertNotNull("c3 value", c3);
+ Assert.assertEquals("p1 value", 2008, p1);
+ Assert.assertEquals("p2 value", "uk", p2);
+ } else {
+ Assert.fail("Unknown partition key found" + keyString);
+ }
+
+ //No errors in mapper, reset counter
+ synchronized(lockObject) {
+ mapCounter--;
+ }
+ }
+ }
+ }
+
+ void runMRTest(String filter) throws Exception {
+
+ Configuration conf = new Configuration();
+ Job job = new Job(conf, "partition value test");
+ job.setJarByClass(TestPartitionKey.class);
+ job.setMapperClass(TestPartitionKey.MapClass.class);
+
+ // input settings
+ job.setInputFormatClass(OwlInputFormat.class);
+
+ //Specify server url, db name, table name and partition predicate through OwlTableInput object
+ OwlTableName tabName = new OwlTableName(dbname, tableName);
+ OwlTableInputInfo inputInfo = new OwlTableInputInfo(getUri(), tabName, filter);
+ OwlInputFormat.setInput(job, inputInfo);
+
+ OwlSchema schema = new OwlSchema();
+ schema.addColumnSchema(new OwlColumnSchema("c2", ColumnType.STRING));
+ schema.addColumnSchema(new OwlColumnSchema("p2", ColumnType.STRING));
+ schema.addColumnSchema(new OwlColumnSchema("c3", ColumnType.INT));
+ schema.addColumnSchema(new OwlColumnSchema("p1", ColumnType.STRING));
+
+ //Set output schema having partition keys also
+ OwlInputFormat.setOutputSchema(job, schema);
+ FileSystem fs = new LocalFileSystem();
+
+ stub.dropData(new Path(fs.getWorkingDirectory(), dataLocation + "/outputfile"));
+ FileOutputFormat.setOutputPath(job, new Path(dataLocation + "/outputfile"));
+ job.setOutputKeyClass(BytesWritable.class);
+ job.setOutputValueClass(DefaultTuple.class);
+
+ job.waitForCompletion(true);
+
+ assertEquals("Mapper completion status", 0, mapCounter);
+ }
+
+ private void cleanup(boolean ignoreException) throws Exception {
+ try {
+ driver.dropOwlTable(new OwlTable(name));
+ } catch(Exception e){ if( ! ignoreException ) throw e; }
+ try {
+ driver.dropOwlDatabase(new OwlDatabase(dbname, null));
+ } catch(Exception e){ if( ! ignoreException ) throw e; }
+ try {
+ FileSystem fs = new LocalFileSystem();
+ stub.dropData(new Path(fs.getWorkingDirectory(), dataLocation));
+ }catch(Exception e){ if( ! ignoreException ) throw e; }
+ }
+
+ public void initialize() throws Exception {
+ FileSystem fs = new LocalFileSystem();
+
+ driver = new OwlDriver(getUri());
+ cleanup(true);
+
+ driver.createOwlDatabase(new OwlDatabase(dbname, "hdfs://localhost:9000/data/testPartitionValues"));
+
+ OwlSchema schema = new OwlSchema();
+ schema.addColumnSchema(new OwlColumnSchema("c1", ColumnType.INT));
+
+ OwlTable table = new OwlTableBuilder().
+ setName(name).
+ addPartition("p1", DataType.INT).
+ addPartition("p2", DataType.STRING).
+ setSchema(schema).
+ build();
+
+ driver.createOwlTable(table);
+
+ keyValues = new ArrayList<OwlKeyValue>();
+ k1 = new OwlKeyValue("p1", 2008);
+ k2 = new OwlKeyValue("p2", "us");
+ keyValues.add(k1);
+ keyValues.add(k2);
+
+ OwlSchema schema1 = new OwlSchema();
+ schema1.addColumnSchema(new OwlColumnSchema("c1", ColumnType.INT));
+ schema1.addColumnSchema(new OwlColumnSchema("c2", ColumnType.STRING));
+
+ //Seconds partitions schema has extra column
+ OwlSchema schema2 = new OwlSchema();
+ schema2.addColumnSchema(new OwlColumnSchema("c1", ColumnType.INT));
+ schema2.addColumnSchema(new OwlColumnSchema("c2", ColumnType.STRING));
+ schema2.addColumnSchema(new OwlColumnSchema("c3", ColumnType.INT));
+
+ stub.createRandomData(new Path(fs.getWorkingDirectory(), partitionLocation1), partitionLocation1, 1, 11, schema1);
+ publish(2008, "us", partitionLocation1, schema1);
+
+ stub.createRandomData(new Path(fs.getWorkingDirectory(), partitionLocation2), partitionLocation1, 1, 12, schema2);
+ publish(2008, "uk", partitionLocation2, schema2);
+ }
+
+ public void partitionValueMRTest() throws Exception {
+ runMRTest("p1 = 2008 and p2 = \"us\"");
+ runMRTest("p1 = 2008 and p2 = \"uk\"");
+ runMRTest("p1 = 2008");
+ }
+
+ public static void runTest(StorageDriverStub stub) throws Exception {
+ TestPartitionKey test = new TestPartitionKey(stub);
+
+ test.initialize();
+ test.partitionValueMRTest();
+ test.cleanup(false);
+ }
+
+ @Test
+ public static void testSampleStorageDriverPK() throws Exception {
+ runTest(new SampleDriverStub());
+ }
+}
+