You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@gora.apache.org by en...@apache.org on 2010/10/11 17:40:29 UTC
svn commit: r1021390 [6/9] - in /incubator/gora/trunk: ./ bin/ conf/ docs/
gora-cassandra/ gora-cassandra/ivy/ gora-cassandra/src/examples/java/
gora-cassandra/src/main/java/org/apache/
gora-cassandra/src/main/java/org/apache/gora/ gora-cassandra/src/m...
Added: incubator/gora/trunk/gora-core/src/test/java/org/apache/gora/avro/store/TestAvroStore.java
URL: http://svn.apache.org/viewvc/incubator/gora/trunk/gora-core/src/test/java/org/apache/gora/avro/store/TestAvroStore.java?rev=1021390&view=auto
==============================================================================
--- incubator/gora/trunk/gora-core/src/test/java/org/apache/gora/avro/store/TestAvroStore.java (added)
+++ incubator/gora/trunk/gora-core/src/test/java/org/apache/gora/avro/store/TestAvroStore.java Mon Oct 11 15:40:22 2010
@@ -0,0 +1,153 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.gora.avro.store;
+
+import static org.apache.gora.examples.WebPageDataCreator.URLS;
+import static org.apache.gora.examples.WebPageDataCreator.URL_INDEXES;
+import static org.apache.gora.examples.WebPageDataCreator.createWebPageData;
+
+import java.io.IOException;
+
+import junit.framework.Assert;
+
+import org.apache.gora.avro.store.AvroStore;
+import org.apache.gora.avro.store.AvroStore.CodecType;
+import org.apache.gora.examples.generated.Employee;
+import org.apache.gora.examples.generated.WebPage;
+import org.apache.gora.query.Query;
+import org.apache.gora.query.Result;
+import org.apache.gora.store.DataStore;
+import org.apache.gora.store.DataStoreFactory;
+import org.apache.gora.store.DataStoreTestUtil;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.Path;
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Test;
+
+/**
+ * Test case for {@link AvroStore}.
+ */
+public class TestAvroStore {
+
+ public static final String EMPLOYEE_OUTPUT =
+ System.getProperty("test.build.data") + "/testavrostore/employee.data";
+ public static final String WEBPAGE_OUTPUT =
+ System.getProperty("test.build.data") + "/testavrostore/webpage.data";
+
+ protected AvroStore<String,Employee> employeeStore;
+ protected AvroStore<String,WebPage> webPageStore;
+ protected Configuration conf = new Configuration();
+
+ @Before
+ public void setUp() throws Exception {
+ employeeStore = createEmployeeDataStore();
+ employeeStore.initialize(String.class, Employee.class, DataStoreFactory.properties);
+ employeeStore.setOutputPath(EMPLOYEE_OUTPUT);
+ employeeStore.setInputPath(EMPLOYEE_OUTPUT);
+
+ webPageStore = new AvroStore<String, WebPage>();
+ webPageStore.initialize(String.class, WebPage.class, DataStoreFactory.properties);
+ webPageStore.setOutputPath(WEBPAGE_OUTPUT);
+ webPageStore.setInputPath(WEBPAGE_OUTPUT);
+ }
+
+ @SuppressWarnings("unchecked")
+ protected AvroStore<String, Employee> createEmployeeDataStore() {
+ return (AvroStore<String, Employee>) DataStoreFactory.getDataStore(
+ AvroStore.class, String.class, Employee.class);
+ }
+
+ protected AvroStore<String, WebPage> createWebPageDataStore() {
+ return new AvroStore<String, WebPage>();
+ }
+
+ @After
+ public void tearDown() throws Exception {
+ deletePath(employeeStore.getOutputPath());
+ deletePath(webPageStore.getOutputPath());
+
+ employeeStore.close();
+ webPageStore.close();
+ }
+
+ private void deletePath(String output) throws IOException {
+ if(output != null) {
+ Path path = new Path(output);
+ path.getFileSystem(conf).delete(path, true);
+ }
+ }
+
+ @Test
+ public void testNewInstance() throws IOException {
+ DataStoreTestUtil.testNewPersistent(employeeStore);
+ }
+
+ @Test
+ public void testCreateSchema() throws IOException {
+ DataStoreTestUtil.testCreateEmployeeSchema(employeeStore);
+ }
+
+ @Test
+ public void testAutoCreateSchema() throws IOException {
+ DataStoreTestUtil.testAutoCreateSchema(employeeStore);
+ }
+
+ @Test
+ public void testPut() throws IOException {
+ DataStoreTestUtil.testPutEmployee(employeeStore);
+ }
+
+ @Test
+ public void testQuery() throws IOException {
+ createWebPageData(webPageStore);
+ webPageStore.close();
+
+ webPageStore.setInputPath(webPageStore.getOutputPath());
+ testQueryWebPages(webPageStore);
+ }
+
+ @Test
+ public void testQueryBinaryEncoder() throws IOException {
+ webPageStore.setCodecType(CodecType.BINARY);
+ webPageStore.setInputPath(webPageStore.getOutputPath());
+
+ createWebPageData(webPageStore);
+ webPageStore.close();
+ testQueryWebPages(webPageStore);
+ }
+
+ //AvroStore should be closed so that Hadoop file is completely flushed,
+ //so below test is copied and modified to close the store after pushing data
+ public static void testQueryWebPages(DataStore<String, WebPage> store)
+ throws IOException {
+
+ Query<String, WebPage> query = store.newQuery();
+ Result<String, WebPage> result = query.execute();
+
+ int i=0;
+ while(result.next()) {
+ WebPage page = result.get();
+ DataStoreTestUtil.assertWebPage(page, URL_INDEXES.get(page.getUrl().toString()));
+ i++;
+ }
+ Assert.assertEquals(i, URLS.length);
+ }
+
+}
Added: incubator/gora/trunk/gora-core/src/test/java/org/apache/gora/avro/store/TestDataFileAvroStore.java
URL: http://svn.apache.org/viewvc/incubator/gora/trunk/gora-core/src/test/java/org/apache/gora/avro/store/TestDataFileAvroStore.java?rev=1021390&view=auto
==============================================================================
--- incubator/gora/trunk/gora-core/src/test/java/org/apache/gora/avro/store/TestDataFileAvroStore.java (added)
+++ incubator/gora/trunk/gora-core/src/test/java/org/apache/gora/avro/store/TestDataFileAvroStore.java Mon Oct 11 15:40:22 2010
@@ -0,0 +1,43 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.gora.avro.store;
+
+import org.apache.gora.avro.store.AvroStore;
+import org.apache.gora.avro.store.DataFileAvroStore;
+import org.apache.gora.examples.generated.Employee;
+import org.apache.gora.examples.generated.WebPage;
+
+/**
+ * Test case for {@link DataFileAvroStore}.
+ */
+public class TestDataFileAvroStore extends TestAvroStore {
+
+ @Override
+ protected AvroStore<String, Employee> createEmployeeDataStore() {
+ return new DataFileAvroStore<String, Employee>();
+ }
+
+ @Override
+ protected AvroStore<String, WebPage> createWebPageDataStore() {
+ return new DataFileAvroStore<String, WebPage>();
+ }
+
+ //import all tests from super class
+
+}
Added: incubator/gora/trunk/gora-core/src/test/java/org/apache/gora/mapreduce/DataStoreMapReduceTestBase.java
URL: http://svn.apache.org/viewvc/incubator/gora/trunk/gora-core/src/test/java/org/apache/gora/mapreduce/DataStoreMapReduceTestBase.java?rev=1021390&view=auto
==============================================================================
--- incubator/gora/trunk/gora-core/src/test/java/org/apache/gora/mapreduce/DataStoreMapReduceTestBase.java (added)
+++ incubator/gora/trunk/gora-core/src/test/java/org/apache/gora/mapreduce/DataStoreMapReduceTestBase.java Mon Oct 11 15:40:22 2010
@@ -0,0 +1,77 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.gora.mapreduce;
+
+import java.io.IOException;
+
+import org.apache.gora.examples.generated.WebPage;
+import org.apache.gora.store.DataStore;
+import org.apache.hadoop.mapred.HadoopTestCase;
+import org.apache.hadoop.mapred.JobConf;
+import org.junit.Before;
+import org.junit.Test;
+
+
+/**
+ * Base class for Mapreduce based tests. This is just a convenience
+ * class, which actually only uses {@link MapReduceTestUtils} methods to
+ * run the tests.
+ */
+@SuppressWarnings("deprecation")
+public abstract class DataStoreMapReduceTestBase extends HadoopTestCase {
+
+ private DataStore<String, WebPage> webPageStore;
+ private JobConf job;
+
+ public DataStoreMapReduceTestBase(int mrMode, int fsMode, int taskTrackers,
+ int dataNodes) throws IOException {
+ super(mrMode, fsMode, taskTrackers, dataNodes);
+ }
+
+ public DataStoreMapReduceTestBase() throws IOException {
+ this(HadoopTestCase.CLUSTER_MR, HadoopTestCase.DFS_FS, 2, 2);
+ }
+
+ @Override
+ @Before
+ public void setUp() throws Exception {
+ super.setUp();
+ webPageStore = createWebPageDataStore();
+ job = createJobConf();
+ }
+
+ @Override
+ public void tearDown() throws Exception {
+ super.tearDown();
+ webPageStore.close();
+ }
+
+ protected abstract DataStore<String, WebPage> createWebPageDataStore()
+ throws IOException;
+
+ @Test
+ public void testCountQuery() throws Exception {
+ MapReduceTestUtils.testCountQuery(webPageStore, job);
+ }
+
+ @Test
+ public void testWordCount() throws Exception {
+ MapReduceTestUtils.testCountQuery(webPageStore, job);
+ }
+}
Added: incubator/gora/trunk/gora-core/src/test/java/org/apache/gora/mapreduce/MapReduceTestUtils.java
URL: http://svn.apache.org/viewvc/incubator/gora/trunk/gora-core/src/test/java/org/apache/gora/mapreduce/MapReduceTestUtils.java?rev=1021390&view=auto
==============================================================================
--- incubator/gora/trunk/gora-core/src/test/java/org/apache/gora/mapreduce/MapReduceTestUtils.java (added)
+++ incubator/gora/trunk/gora-core/src/test/java/org/apache/gora/mapreduce/MapReduceTestUtils.java Mon Oct 11 15:40:22 2010
@@ -0,0 +1,103 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.gora.mapreduce;
+
+import java.io.IOException;
+import java.util.HashMap;
+import java.util.Map;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.gora.examples.WebPageDataCreator;
+import org.apache.gora.examples.generated.TokenDatum;
+import org.apache.gora.examples.generated.WebPage;
+import org.apache.gora.examples.mapreduce.QueryCounter;
+import org.apache.gora.examples.mapreduce.WordCount;
+import org.apache.gora.query.Query;
+import org.apache.gora.store.DataStore;
+import org.apache.hadoop.conf.Configuration;
+import org.junit.Assert;
+
+public class MapReduceTestUtils {
+
+ private static final Log log = LogFactory.getLog(MapReduceTestUtils.class);
+
+ /** Tests by running the {@link QueryCounter} mapreduce job */
+ public static void testCountQuery(DataStore<String, WebPage> dataStore
+ , Configuration conf)
+ throws Exception {
+
+ dataStore.setConf(conf);
+
+ //create input
+ WebPageDataCreator.createWebPageData(dataStore);
+
+
+ QueryCounter<String,WebPage> counter = new QueryCounter<String,WebPage>(conf);
+ Query<String,WebPage> query = dataStore.newQuery();
+ query.setFields(WebPage._ALL_FIELDS);
+
+ dataStore.close();
+
+
+ //run the job
+ log.info("running count query job");
+ long result = counter.countQuery(dataStore, query);
+ log.info("finished count query job");
+
+ //assert results
+ Assert.assertEquals(WebPageDataCreator.URLS.length, result);
+
+ }
+
+ public static void testWordCount(Configuration conf,
+ DataStore<String,WebPage> inStore, DataStore<String,
+ TokenDatum> outStore) throws Exception {
+ inStore.setConf(conf);
+ outStore.setConf(conf);
+
+ //create input
+ WebPageDataCreator.createWebPageData(inStore);
+
+ //run the job
+ WordCount wordCount = new WordCount(conf);
+ wordCount.wordCount(inStore, outStore);
+
+ //assert results
+ HashMap<String, Integer> actualCounts = new HashMap<String, Integer>();
+ for(String content : WebPageDataCreator.CONTENTS) {
+ for(String token:content.split(" ")) {
+ Integer count = actualCounts.get(token);
+ if(count == null)
+ count = 0;
+ actualCounts.put(token, ++count);
+ }
+ }
+ for(Map.Entry<String, Integer> entry:actualCounts.entrySet()) {
+ assertTokenCount(outStore, entry.getKey(), entry.getValue());
+ }
+ }
+
+ private static void assertTokenCount(DataStore<String, TokenDatum> outStore,
+ String token, int count) throws IOException {
+ TokenDatum datum = outStore.get(token, null);
+ Assert.assertNotNull("token:" + token + " cannot be found in datastore", datum);
+ Assert.assertEquals("count for token:" + token + " is wrong", count, datum.getCount());
+ }
+}
Added: incubator/gora/trunk/gora-core/src/test/java/org/apache/gora/mapreduce/TestGoraInputFormat.java
URL: http://svn.apache.org/viewvc/incubator/gora/trunk/gora-core/src/test/java/org/apache/gora/mapreduce/TestGoraInputFormat.java?rev=1021390&view=auto
==============================================================================
--- incubator/gora/trunk/gora-core/src/test/java/org/apache/gora/mapreduce/TestGoraInputFormat.java (added)
+++ incubator/gora/trunk/gora-core/src/test/java/org/apache/gora/mapreduce/TestGoraInputFormat.java Mon Oct 11 15:40:22 2010
@@ -0,0 +1,70 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.gora.mapreduce;
+
+import java.io.IOException;
+import java.util.Arrays;
+import java.util.List;
+
+import junit.framework.Assert;
+
+import org.apache.gora.examples.generated.Employee;
+import org.apache.gora.mapreduce.GoraInputFormat;
+import org.apache.gora.mapreduce.GoraInputSplit;
+import org.apache.gora.mock.persistency.MockPersistent;
+import org.apache.gora.mock.query.MockQuery;
+import org.apache.gora.mock.store.MockDataStore;
+import org.apache.gora.query.PartitionQuery;
+import org.apache.hadoop.mapreduce.InputSplit;
+import org.apache.hadoop.mapreduce.Job;
+import org.junit.Test;
+
+public class TestGoraInputFormat {
+
+ public List<InputSplit> getInputSplits()
+ throws IOException, InterruptedException {
+
+ Job job = new Job();
+ MockDataStore store = MockDataStore.get();
+
+ MockQuery query = store.newQuery();
+ query.setFields(Employee._ALL_FIELDS);
+ GoraInputFormat.setInput(job, query, false);
+
+ GoraInputFormat<String, MockPersistent> inputFormat
+ = new GoraInputFormat<String, MockPersistent>();
+
+ inputFormat.setConf(job.getConfiguration());
+
+ return inputFormat.getSplits(job);
+ }
+
+ @Test
+ @SuppressWarnings("rawtypes")
+ public void testGetSplits() throws IOException, InterruptedException {
+ List<InputSplit> splits = getInputSplits();
+
+ Assert.assertTrue(splits.size() > 0);
+
+ InputSplit split = splits.get(0);
+ PartitionQuery query = ((GoraInputSplit)split).getQuery();
+ Assert.assertTrue(Arrays.equals(Employee._ALL_FIELDS, query.getFields()));
+ }
+
+}
\ No newline at end of file
Added: incubator/gora/trunk/gora-core/src/test/java/org/apache/gora/mapreduce/TestGoraInputSplit.java
URL: http://svn.apache.org/viewvc/incubator/gora/trunk/gora-core/src/test/java/org/apache/gora/mapreduce/TestGoraInputSplit.java?rev=1021390&view=auto
==============================================================================
--- incubator/gora/trunk/gora-core/src/test/java/org/apache/gora/mapreduce/TestGoraInputSplit.java (added)
+++ incubator/gora/trunk/gora-core/src/test/java/org/apache/gora/mapreduce/TestGoraInputSplit.java Mon Oct 11 15:40:22 2010
@@ -0,0 +1,77 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.gora.mapreduce;
+
+import java.io.IOException;
+import java.util.List;
+
+import junit.framework.Assert;
+
+import org.apache.gora.mapreduce.GoraInputSplit;
+import org.apache.gora.mock.persistency.MockPersistent;
+import org.apache.gora.mock.query.MockQuery;
+import org.apache.gora.mock.store.MockDataStore;
+import org.apache.gora.query.PartitionQuery;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.io.TestWritable;
+import org.junit.Test;
+
+/**
+ * Test case for {@link GoraInputSplit}.
+ */
+public class TestGoraInputSplit {
+
+ private Configuration conf = new Configuration();
+
+ private List<PartitionQuery<String, MockPersistent>>
+ getPartitions() throws IOException {
+ MockDataStore store = MockDataStore.get();
+ MockQuery query = store.newQuery();
+
+ List<PartitionQuery<String, MockPersistent>> partitions =
+ store.getPartitions(query);
+ return partitions;
+ }
+
+ @Test
+ public void testGetLocations() throws IOException {
+ List<PartitionQuery<String, MockPersistent>> partitions =
+ getPartitions();
+
+ int i=0;;
+ for(PartitionQuery<String, MockPersistent> partition : partitions) {
+ GoraInputSplit split = new GoraInputSplit(conf, partition);
+ Assert.assertEquals(split.getLocations().length, 1);
+ Assert.assertEquals(split.getLocations()[0], MockDataStore.LOCATIONS[i++]);
+ }
+ }
+
+ @Test
+ public void testReadWrite() throws Exception {
+
+ List<PartitionQuery<String, MockPersistent>> partitions =
+ getPartitions();
+
+ for(PartitionQuery<String, MockPersistent> partition : partitions) {
+ GoraInputSplit split = new GoraInputSplit(conf, partition);
+ TestWritable.testWritable(split);
+ }
+ }
+
+}
Added: incubator/gora/trunk/gora-core/src/test/java/org/apache/gora/mapreduce/TestPersistentSerialization.java
URL: http://svn.apache.org/viewvc/incubator/gora/trunk/gora-core/src/test/java/org/apache/gora/mapreduce/TestPersistentSerialization.java?rev=1021390&view=auto
==============================================================================
--- incubator/gora/trunk/gora-core/src/test/java/org/apache/gora/mapreduce/TestPersistentSerialization.java (added)
+++ incubator/gora/trunk/gora-core/src/test/java/org/apache/gora/mapreduce/TestPersistentSerialization.java Mon Oct 11 15:40:22 2010
@@ -0,0 +1,107 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.gora.mapreduce;
+
+import junit.framework.Assert;
+
+import org.apache.avro.util.Utf8;
+import org.apache.gora.examples.WebPageDataCreator;
+import org.apache.gora.examples.generated.Employee;
+import org.apache.gora.examples.generated.WebPage;
+import org.apache.gora.mapreduce.PersistentDeserializer;
+import org.apache.gora.mapreduce.PersistentSerialization;
+import org.apache.gora.mapreduce.PersistentSerializer;
+import org.apache.gora.memory.store.MemStore;
+import org.apache.gora.query.Result;
+import org.apache.gora.store.DataStoreFactory;
+import org.apache.gora.store.DataStoreTestUtil;
+import org.apache.gora.util.TestIOUtils;
+import org.junit.Test;
+
+/** Test class for {@link PersistentSerialization}, {@link PersistentSerializer}
+ * and {@link PersistentDeserializer}
+ */
+public class TestPersistentSerialization {
+
+ @SuppressWarnings("unchecked")
+ @Test
+ public void testSerdeEmployee() throws Exception {
+
+ MemStore<String, Employee> store = DataStoreFactory.getDataStore(
+ MemStore.class, String.class, Employee.class);
+
+ Employee employee = DataStoreTestUtil.createEmployee(store);
+
+ TestIOUtils.testSerializeDeserialize(employee);
+ }
+
+ @Test
+ public void testSerdeEmployeeOneField() throws Exception {
+ Employee employee = new Employee();
+ employee.setSsn(new Utf8("11111"));
+
+ TestIOUtils.testSerializeDeserialize(employee);
+ }
+
+ @Test
+ public void testSerdeEmployeeTwoFields() throws Exception {
+ Employee employee = new Employee();
+ employee.setSsn(new Utf8("11111"));
+ employee.setSalary(100);
+
+ TestIOUtils.testSerializeDeserialize(employee);
+ }
+
+ @SuppressWarnings("unchecked")
+ @Test
+ public void testSerdeWebPage() throws Exception {
+
+ MemStore<String, WebPage> store = DataStoreFactory.getDataStore(
+ MemStore.class, String.class, WebPage.class);
+ WebPageDataCreator.createWebPageData(store);
+
+ Result<String, WebPage> result = store.newQuery().execute();
+
+ int i=0;
+ while(result.next()) {
+ WebPage page = result.get();
+ TestIOUtils.testSerializeDeserialize(page);
+ i++;
+ }
+ Assert.assertEquals(WebPageDataCreator.URLS.length, i);
+ }
+
+ @Test
+ public void testSerdeMultipleWebPages() throws Exception {
+ WebPage page1 = new WebPage();
+ WebPage page2 = new WebPage();
+ WebPage page3 = new WebPage();
+
+ page1.setUrl(new Utf8("foo"));
+ page2.setUrl(new Utf8("baz"));
+ page3.setUrl(new Utf8("bar"));
+
+ page1.addToParsedContent(new Utf8("coo"));
+
+ page2.putToOutlinks(new Utf8("a"), new Utf8("b"));
+
+ TestIOUtils.testSerializeDeserialize(page1, page2, page3);
+ }
+
+}
Added: incubator/gora/trunk/gora-core/src/test/java/org/apache/gora/memory/store/TestMemStore.java
URL: http://svn.apache.org/viewvc/incubator/gora/trunk/gora-core/src/test/java/org/apache/gora/memory/store/TestMemStore.java?rev=1021390&view=auto
==============================================================================
--- incubator/gora/trunk/gora-core/src/test/java/org/apache/gora/memory/store/TestMemStore.java (added)
+++ incubator/gora/trunk/gora-core/src/test/java/org/apache/gora/memory/store/TestMemStore.java Mon Oct 11 15:40:22 2010
@@ -0,0 +1,44 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.gora.memory.store;
+
+import org.apache.gora.examples.generated.Employee;
+import org.apache.gora.examples.generated.WebPage;
+import org.apache.gora.memory.store.MemStore;
+import org.apache.gora.store.DataStore;
+import org.apache.gora.store.DataStoreFactory;
+import org.apache.gora.store.DataStoreTestBase;
+
+/**
+ * Test case for {@link MemStore}.
+ */
+public class TestMemStore extends DataStoreTestBase {
+
+ @SuppressWarnings("unchecked")
+ @Override
+ protected DataStore<String, Employee> createEmployeeDataStore() {
+ return DataStoreFactory.getDataStore(MemStore.class, String.class, Employee.class);
+ }
+
+ @SuppressWarnings("unchecked")
+ @Override
+ protected DataStore<String, WebPage> createWebPageDataStore() {
+ return DataStoreFactory.getDataStore(MemStore.class, String.class, WebPage.class);
+ }
+}
Added: incubator/gora/trunk/gora-core/src/test/java/org/apache/gora/mock/persistency/MockPersistent.java
URL: http://svn.apache.org/viewvc/incubator/gora/trunk/gora-core/src/test/java/org/apache/gora/mock/persistency/MockPersistent.java?rev=1021390&view=auto
==============================================================================
--- incubator/gora/trunk/gora-core/src/test/java/org/apache/gora/mock/persistency/MockPersistent.java (added)
+++ incubator/gora/trunk/gora-core/src/test/java/org/apache/gora/mock/persistency/MockPersistent.java Mon Oct 11 15:40:22 2010
@@ -0,0 +1,100 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.gora.mock.persistency;
+
+import org.apache.avro.Schema;
+import org.apache.gora.persistency.Persistent;
+import org.apache.gora.persistency.StateManager;
+import org.apache.gora.persistency.impl.PersistentBase;
+
+public class MockPersistent extends PersistentBase {
+
+ public static final String FOO = "foo";
+ public static final String BAZ = "baz";
+
+ public static final String[] _ALL_FIELDS = {FOO, BAZ};
+
+ private int foo;
+ private int baz;
+
+ public MockPersistent() {
+ }
+
+ public MockPersistent(StateManager stateManager) {
+ super(stateManager);
+ }
+
+ @Override
+ public Object get(int field) {
+ switch(field) {
+ case 0: return foo;
+ case 1: return baz;
+ }
+ return null;
+ }
+
+ @Override
+ public void put(int field, Object value) {
+ switch(field) {
+ case 0: foo = (Integer)value;
+ case 1: baz = (Integer)value;
+ }
+ }
+
+ @Override
+ public Schema getSchema() {
+ return Schema.parse("{\"type\":\"record\",\"name\":\"MockPersistent\",\"namespace\":\"org.apache.gora.mock.persistency\",\"fields\":[{\"name\":\"foo\",\"type\":\"int\"},{\"name\":\"baz\",\"type\":\"int\"}]}");
+ }
+
+ public void setFoo(int foo) {
+ this.foo = foo;
+ }
+
+ public void setBaz(int baz) {
+ this.baz = baz;
+ }
+
+ public int getFoo() {
+ return foo;
+ }
+
+ public int getBaz() {
+ return baz;
+ }
+
+ @Override
+ public String getField(int index) {
+ return null;
+ }
+
+ @Override
+ public int getFieldIndex(String field) {
+ return 0;
+ }
+
+ @Override
+ public String[] getFields() {
+ return null;
+ }
+
+ @Override
+ public Persistent newInstance(StateManager stateManager) {
+ return new MockPersistent(stateManager);
+ }
+}
Added: incubator/gora/trunk/gora-core/src/test/java/org/apache/gora/mock/query/MockQuery.java
URL: http://svn.apache.org/viewvc/incubator/gora/trunk/gora-core/src/test/java/org/apache/gora/mock/query/MockQuery.java?rev=1021390&view=auto
==============================================================================
--- incubator/gora/trunk/gora-core/src/test/java/org/apache/gora/mock/query/MockQuery.java (added)
+++ incubator/gora/trunk/gora-core/src/test/java/org/apache/gora/mock/query/MockQuery.java Mon Oct 11 15:40:22 2010
@@ -0,0 +1,35 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.gora.mock.query;
+
+import org.apache.gora.mock.persistency.MockPersistent;
+import org.apache.gora.query.impl.QueryBase;
+import org.apache.gora.store.DataStore;
+
+public class MockQuery extends QueryBase<String, MockPersistent> {
+
+ public MockQuery() {
+ super(null);
+ }
+
+ public MockQuery(DataStore<String, MockPersistent> dataStore) {
+ super(dataStore);
+ }
+
+}
Added: incubator/gora/trunk/gora-core/src/test/java/org/apache/gora/mock/store/MockDataStore.java
URL: http://svn.apache.org/viewvc/incubator/gora/trunk/gora-core/src/test/java/org/apache/gora/mock/store/MockDataStore.java?rev=1021390&view=auto
==============================================================================
--- incubator/gora/trunk/gora-core/src/test/java/org/apache/gora/mock/store/MockDataStore.java (added)
+++ incubator/gora/trunk/gora-core/src/test/java/org/apache/gora/mock/store/MockDataStore.java Mon Oct 11 15:40:22 2010
@@ -0,0 +1,139 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.gora.mock.store;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
+
+import org.apache.gora.mock.persistency.MockPersistent;
+import org.apache.gora.mock.query.MockQuery;
+import org.apache.gora.query.PartitionQuery;
+import org.apache.gora.query.Query;
+import org.apache.gora.query.Result;
+import org.apache.gora.query.impl.PartitionQueryImpl;
+import org.apache.gora.store.DataStoreFactory;
+import org.apache.gora.store.impl.DataStoreBase;
+
+public class MockDataStore extends DataStoreBase<String, MockPersistent> {
+
+ public static final int NUM_PARTITIONS = 5;
+ public static final String[] LOCATIONS = {"foo1", "foo2", "foo3", "foo4", "foo1"};
+
+ public static MockDataStore get() {
+ MockDataStore dataStore = DataStoreFactory.getDataStore(MockDataStore.class
+ , String.class, MockPersistent.class);
+ return dataStore;
+ }
+
+ public MockDataStore() { }
+
+ @Override
+ public String getSchemaName() {
+ return null;
+ }
+
+ @Override
+ public void close() throws IOException {
+ }
+
+ @Override
+ public void createSchema() throws IOException {
+ }
+
+ @Override
+ public void deleteSchema() throws IOException {
+ }
+
+ @Override
+ public void truncateSchema() throws IOException {
+ }
+
+ @Override
+ public boolean schemaExists() throws IOException {
+ return true;
+ }
+
+ @Override
+ public boolean delete(String key) throws IOException {
+ return false;
+ }
+
+ @Override
+ public long deleteByQuery(Query<String, MockPersistent> query)
+ throws IOException {
+ return 0;
+ }
+
+ @Override
+ public Result<String, MockPersistent> execute(
+ Query<String, MockPersistent> query) throws IOException {
+ return null;
+ }
+
+ @Override
+ public void flush() throws IOException {
+ }
+
+ @Override
+ public MockPersistent get(String key, String[] fields) throws IOException {
+ return null;
+ }
+
+ @Override
+ public Class<String> getKeyClass() {
+ return String.class;
+ }
+
+ @Override
+ public List<PartitionQuery<String, MockPersistent>> getPartitions(
+ Query<String, MockPersistent> query) throws IOException {
+
+ ArrayList<PartitionQuery<String, MockPersistent>> list =
+ new ArrayList<PartitionQuery<String,MockPersistent>>();
+
+ for(int i=0; i<NUM_PARTITIONS; i++) {
+ list.add(new PartitionQueryImpl<String, MockPersistent>(query, LOCATIONS[i]));
+ }
+
+ return list;
+ }
+
+ @Override
+ public Class<MockPersistent> getPersistentClass() {
+ return MockPersistent.class;
+ }
+
+ @Override
+ public MockQuery newQuery() {
+ return new MockQuery(this);
+ }
+
+ @Override
+ public void put(String key, MockPersistent obj) throws IOException {
+ }
+
+ @Override
+ public void setKeyClass(Class<String> keyClass) {
+ }
+
+ @Override
+ public void setPersistentClass(Class<MockPersistent> persistentClass) {
+ }
+}
Added: incubator/gora/trunk/gora-core/src/test/java/org/apache/gora/persistency/impl/TestPersistentBase.java
URL: http://svn.apache.org/viewvc/incubator/gora/trunk/gora-core/src/test/java/org/apache/gora/persistency/impl/TestPersistentBase.java?rev=1021390&view=auto
==============================================================================
--- incubator/gora/trunk/gora-core/src/test/java/org/apache/gora/persistency/impl/TestPersistentBase.java (added)
+++ incubator/gora/trunk/gora-core/src/test/java/org/apache/gora/persistency/impl/TestPersistentBase.java Mon Oct 11 15:40:22 2010
@@ -0,0 +1,120 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.gora.persistency.impl;
+
+import java.io.IOException;
+import java.nio.ByteBuffer;
+
+import org.apache.avro.util.Utf8;
+import org.apache.gora.examples.generated.Employee;
+import org.apache.gora.examples.generated.WebPage;
+import org.apache.gora.memory.store.MemStore;
+import org.apache.gora.store.DataStoreFactory;
+import org.apache.gora.store.DataStoreTestUtil;
+import org.junit.Assert;
+import org.junit.Test;
+
+/**
+ * Testcase for PersistentBase class
+ */
+public class TestPersistentBase {
+
+ @Test
+ public void testGetFields() {
+ WebPage page = new WebPage();
+ String[] fields = page.getFields();
+ Assert.assertArrayEquals(WebPage._ALL_FIELDS, fields);
+ }
+
+ @Test
+ public void testGetField() {
+ WebPage page = new WebPage();
+ for(int i=0; i<WebPage._ALL_FIELDS.length; i++) {
+ String field = page.getField(i);
+ Assert.assertEquals(WebPage._ALL_FIELDS[i], field);
+ }
+ }
+
+ @Test
+ public void testGetFieldIndex() {
+ WebPage page = new WebPage();
+ for(int i=0; i<WebPage._ALL_FIELDS.length; i++) {
+ int index = page.getFieldIndex(WebPage._ALL_FIELDS[i]);
+ Assert.assertEquals(i, index);
+ }
+ }
+
+ @Test
+ public void testFieldsWithTwoClasses() {
+ WebPage page = new WebPage();
+ for(int i=0; i<WebPage._ALL_FIELDS.length; i++) {
+ int index = page.getFieldIndex(WebPage._ALL_FIELDS[i]);
+ Assert.assertEquals(i, index);
+ }
+ Employee employee = new Employee();
+ for(int i=0; i<Employee._ALL_FIELDS.length; i++) {
+ int index = employee.getFieldIndex(Employee._ALL_FIELDS[i]);
+ Assert.assertEquals(i, index);
+ }
+ }
+
+ @Test
+ public void testClear() {
+
+ //test clear all fields
+ WebPage page = new WebPage();
+ page.setUrl(new Utf8("http://foo.com"));
+ page.addToParsedContent(new Utf8("foo"));
+ page.putToOutlinks(new Utf8("foo"), new Utf8("bar"));
+ page.setContent(ByteBuffer.wrap("foo baz bar".getBytes()));
+
+ page.clear();
+
+ Assert.assertNull(page.getUrl());
+ Assert.assertEquals(0, page.getParsedContent().size());
+ Assert.assertEquals(0, page.getOutlinks().size());
+ Assert.assertNull(page.getContent());
+
+ //set fields again
+ page.setUrl(new Utf8("http://bar.com"));
+ page.addToParsedContent(new Utf8("bar"));
+ page.putToOutlinks(new Utf8("bar"), new Utf8("baz"));
+ page.setContent(ByteBuffer.wrap("foo baz bar barbaz".getBytes()));
+
+ //test clear new object
+ page = new WebPage();
+ page.clear();
+
+ //test primitive fields
+ Employee employee = new Employee();
+ employee.clear();
+ }
+
+ @Test
+ public void testClone() throws IOException {
+ //more tests for clone are in TestPersistentDatumReader
+ @SuppressWarnings("unchecked")
+ MemStore<String, Employee> store = DataStoreFactory.getDataStore(
+ MemStore.class, String.class, Employee.class);
+
+ Employee employee = DataStoreTestUtil.createEmployee(store);
+
+ Assert.assertEquals(employee, employee.clone());
+ }
+}
Added: incubator/gora/trunk/gora-core/src/test/java/org/apache/gora/persistency/impl/TestStateManagerImpl.java
URL: http://svn.apache.org/viewvc/incubator/gora/trunk/gora-core/src/test/java/org/apache/gora/persistency/impl/TestStateManagerImpl.java?rev=1021390&view=auto
==============================================================================
--- incubator/gora/trunk/gora-core/src/test/java/org/apache/gora/persistency/impl/TestStateManagerImpl.java (added)
+++ incubator/gora/trunk/gora-core/src/test/java/org/apache/gora/persistency/impl/TestStateManagerImpl.java Mon Oct 11 15:40:22 2010
@@ -0,0 +1,115 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.gora.persistency.impl;
+
+import java.io.IOException;
+
+import junit.framework.Assert;
+
+import org.apache.avro.util.Utf8;
+import org.apache.gora.examples.generated.Employee;
+import org.apache.gora.mock.persistency.MockPersistent;
+import org.apache.gora.persistency.impl.StateManagerImpl;
+import org.junit.Before;
+import org.junit.Test;
+
+/**
+ * Test case for {@link StateManagerImpl}
+ */
+public class TestStateManagerImpl {
+
+ private StateManagerImpl stateManager;
+ private MockPersistent persistent;
+
+ @Before
+ public void setUp() {
+ this.stateManager = new StateManagerImpl();
+ this.persistent = new MockPersistent(stateManager);
+ }
+
+ @Test
+ public void testDirty() {
+ Assert.assertFalse(stateManager.isDirty(persistent));
+ stateManager.setDirty(persistent);
+ Assert.assertTrue(stateManager.isDirty(persistent));
+ }
+
+ @Test
+ public void testDirty2() {
+ Assert.assertFalse(stateManager.isDirty(persistent, 0));
+ Assert.assertFalse(stateManager.isDirty(persistent, 1));
+ stateManager.setDirty(persistent, 0);
+ Assert.assertTrue(stateManager.isDirty(persistent, 0));
+ Assert.assertFalse(stateManager.isDirty(persistent, 1));
+ }
+
+ @Test
+ public void testClearDirty() {
+ Assert.assertFalse(stateManager.isDirty(persistent));
+ stateManager.setDirty(persistent, 0);
+ stateManager.clearDirty(persistent);
+ Assert.assertFalse(this.stateManager.isDirty(persistent));
+ }
+
+ @Test
+ public void testReadable() throws IOException {
+ Assert.assertFalse(stateManager.isReadable(persistent, 0));
+ Assert.assertFalse(stateManager.isReadable(persistent, 1));
+ stateManager.setReadable(persistent, 0);
+ Assert.assertTrue(stateManager.isReadable(persistent, 0));
+ Assert.assertFalse(stateManager.isReadable(persistent, 1));
+ }
+
+ @Test
+ public void testReadable2() {
+ stateManager = new StateManagerImpl();
+ Employee employee = new Employee(stateManager);
+ Assert.assertFalse(stateManager.isReadable(employee, 0));
+ Assert.assertFalse(stateManager.isReadable(employee, 1));
+ employee.setName(new Utf8("foo"));
+ Assert.assertTrue(stateManager.isReadable(employee, 0));
+ Assert.assertFalse(stateManager.isReadable(employee, 1));
+ }
+
+ @Test
+ public void testClearReadable() {
+ stateManager.setReadable(persistent, 0);
+ stateManager.clearReadable(persistent);
+ Assert.assertFalse(stateManager.isReadable(persistent, 0));
+ }
+
+ @Test
+ public void testIsNew() {
+ //newly created objects should be new
+ Assert.assertTrue(persistent.isNew());
+ }
+
+ @Test
+ public void testNew() {
+ stateManager.setNew(persistent);
+ Assert.assertTrue(persistent.isNew());
+ }
+
+ @Test
+ public void testClearNew() {
+ stateManager.clearNew(persistent);
+ Assert.assertFalse(persistent.isNew());
+ }
+
+}
Added: incubator/gora/trunk/gora-core/src/test/java/org/apache/gora/query/impl/TestPartitionQueryImpl.java
URL: http://svn.apache.org/viewvc/incubator/gora/trunk/gora-core/src/test/java/org/apache/gora/query/impl/TestPartitionQueryImpl.java?rev=1021390&view=auto
==============================================================================
--- incubator/gora/trunk/gora-core/src/test/java/org/apache/gora/query/impl/TestPartitionQueryImpl.java (added)
+++ incubator/gora/trunk/gora-core/src/test/java/org/apache/gora/query/impl/TestPartitionQueryImpl.java Mon Oct 11 15:40:22 2010
@@ -0,0 +1,48 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.gora.query.impl;
+
+import org.apache.gora.mock.persistency.MockPersistent;
+import org.apache.gora.mock.query.MockQuery;
+import org.apache.gora.mock.store.MockDataStore;
+import org.apache.gora.query.impl.PartitionQueryImpl;
+import org.apache.hadoop.io.TestWritable;
+import org.junit.Test;
+
+/**
+ * Test case for {@link PartitionQueryImpl}
+ */
+public class TestPartitionQueryImpl {
+
+ private MockDataStore dataStore = MockDataStore.get();
+
+ @Test
+ public void testReadWrite() throws Exception {
+
+ MockQuery baseQuery = dataStore.newQuery();
+ baseQuery.setStartKey("start");
+ baseQuery.setLimit(42);
+
+ PartitionQueryImpl<String, MockPersistent>
+ query = new PartitionQueryImpl<String, MockPersistent>(baseQuery);
+
+ TestWritable.testWritable(query);
+ }
+
+}
Added: incubator/gora/trunk/gora-core/src/test/java/org/apache/gora/query/impl/TestQueryBase.java
URL: http://svn.apache.org/viewvc/incubator/gora/trunk/gora-core/src/test/java/org/apache/gora/query/impl/TestQueryBase.java?rev=1021390&view=auto
==============================================================================
--- incubator/gora/trunk/gora-core/src/test/java/org/apache/gora/query/impl/TestQueryBase.java (added)
+++ incubator/gora/trunk/gora-core/src/test/java/org/apache/gora/query/impl/TestQueryBase.java Mon Oct 11 15:40:22 2010
@@ -0,0 +1,64 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+
+package org.apache.gora.query.impl;
+
+import junit.framework.Assert;
+
+import org.apache.gora.mock.query.MockQuery;
+import org.apache.gora.mock.store.MockDataStore;
+import org.apache.gora.query.impl.QueryBase;
+import org.apache.gora.util.TestIOUtils;
+import org.junit.Before;
+import org.junit.Test;
+
+/**
+ * Test case for {@link QueryBase}.
+ */
+public class TestQueryBase {
+
+ private MockDataStore dataStore = MockDataStore.get();
+ private MockQuery query;
+
+ private static final String[] FIELDS = {"foo", "baz", "bar"};
+ private static final String START_KEY = "1_start";
+ private static final String END_KEY = "2_end";
+
+ @Before
+ public void setUp() {
+ query = dataStore.newQuery(); //MockQuery extends QueryBase
+ }
+
+ @Test
+ public void testReadWrite() throws Exception {
+ query.setFields(FIELDS);
+ query.setKeyRange(START_KEY, END_KEY);
+ TestIOUtils.testSerializeDeserialize(query);
+
+ Assert.assertNotNull(query.getDataStore());
+ }
+
+ @Test
+ public void testReadWrite2() throws Exception {
+ query.setLimit(1000);
+ query.setTimeRange(0, System.currentTimeMillis());
+ TestIOUtils.testSerializeDeserialize(query);
+ }
+
+}
Added: incubator/gora/trunk/gora-core/src/test/java/org/apache/gora/store/DataStoreTestBase.java
URL: http://svn.apache.org/viewvc/incubator/gora/trunk/gora-core/src/test/java/org/apache/gora/store/DataStoreTestBase.java?rev=1021390&view=auto
==============================================================================
--- incubator/gora/trunk/gora-core/src/test/java/org/apache/gora/store/DataStoreTestBase.java (added)
+++ incubator/gora/trunk/gora-core/src/test/java/org/apache/gora/store/DataStoreTestBase.java Mon Oct 11 15:40:22 2010
@@ -0,0 +1,368 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.gora.store;
+
+import java.io.IOException;
+import java.nio.ByteBuffer;
+
+import junit.framework.Assert;
+
+import org.apache.avro.util.Utf8;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.gora.GoraTestDriver;
+import org.apache.gora.examples.generated.Employee;
+import org.apache.gora.examples.generated.Metadata;
+import org.apache.gora.examples.generated.WebPage;
+import org.apache.gora.store.DataStore;
+import org.junit.After;
+import org.junit.AfterClass;
+import org.junit.Before;
+import org.junit.BeforeClass;
+import org.junit.Test;
+
+/**
+ * A base class for {@link DataStore} tests. This is just a convenience
+ * class, which actually only uses {@link DataStoreTestUtil} methods to
+ * run the tests. Not all test cases can extend this class (like TestHBaseStore),
+ * so all test logic shuold reside in DataStoreTestUtil class.
+ */
+public abstract class DataStoreTestBase {
+
+ public static final Log log = LogFactory.getLog(DataStoreTestBase.class);
+
+ protected static GoraTestDriver testDriver;
+
+ protected DataStore<String,Employee> employeeStore;
+ protected DataStore<String,WebPage> webPageStore;
+
+ @Deprecated
+ protected abstract DataStore<String,Employee> createEmployeeDataStore() throws IOException ;
+
+ @Deprecated
+ protected abstract DataStore<String,WebPage> createWebPageDataStore() throws IOException;
+
+ /** junit annoyingly forces BeforeClass to be static, so this method
+ * should be called from a static block
+ */
+ protected static void setTestDriver(GoraTestDriver driver) {
+ testDriver = driver;
+ }
+
+ private static boolean setUpClassCalled = false;
+
+ @BeforeClass
+ public static void setUpClass() throws Exception {
+ if(testDriver != null && !setUpClassCalled) {
+ log.info("setting up class");
+ testDriver.setUpClass();
+ setUpClassCalled = true;
+ }
+ }
+
+ @AfterClass
+ public static void tearDownClass() throws Exception {
+ if(testDriver != null) {
+ log.info("tearing down class");
+ testDriver.tearDownClass();
+ }
+ }
+
+ @Before
+ public void setUp() throws Exception {
+ //There is an issue in JUnit 4 tests in Eclipse where TestSqlStore static
+ //methods are not called BEFORE setUpClass. I think this is a bug in
+ //JUnitRunner in Eclipse. Below is a workaround for that problem.
+ if(!setUpClassCalled) setUpClass();
+
+ log.info("setting up test");
+ if(testDriver != null) {
+ employeeStore = testDriver.createDataStore(String.class, Employee.class);
+ webPageStore = testDriver.createDataStore(String.class, WebPage.class);
+ testDriver.setUp();
+ } else {
+ employeeStore = createEmployeeDataStore();
+ webPageStore = createWebPageDataStore();
+
+ employeeStore.truncateSchema();
+ webPageStore.truncateSchema();
+ }
+ }
+
+ @After
+ public void tearDown() throws Exception {
+ log.info("tearing down test");
+ if(testDriver != null) {
+ testDriver.tearDown();
+ }
+ //employeeStore.close();
+ //webPageStore.close();
+ }
+
+ @Test
+ public void testNewInstance() throws IOException {
+ log.info("test method: testNewInstance");
+ DataStoreTestUtil.testNewPersistent(employeeStore);
+ }
+
+ @Test
+ public void testCreateSchema() throws Exception {
+ log.info("test method: testCreateSchema");
+ DataStoreTestUtil.testCreateEmployeeSchema(employeeStore);
+ assertSchemaExists("Employee");
+ }
+
+ // Override this to assert that schema is created correctly
+ public void assertSchemaExists(String schemaName) throws Exception {
+ }
+
+ @Test
+ public void testAutoCreateSchema() throws Exception {
+ log.info("test method: testAutoCreateSchema");
+ DataStoreTestUtil.testAutoCreateSchema(employeeStore);
+ assertAutoCreateSchema();
+ }
+
+ public void assertAutoCreateSchema() throws Exception {
+ assertSchemaExists("Employee");
+ }
+
+ @Test
+ public void testTruncateSchema() throws Exception {
+ log.info("test method: testTruncateSchema");
+ DataStoreTestUtil.testTruncateSchema(webPageStore);
+ assertSchemaExists("WebPage");
+ }
+
+ @Test
+ public void testDeleteSchema() throws IOException {
+ log.info("test method: testDeleteSchema");
+ DataStoreTestUtil.testDeleteSchema(webPageStore);
+ }
+
+ @Test
+ public void testSchemaExists() throws Exception {
+ log.info("test method: testSchemaExists");
+ DataStoreTestUtil.testSchemaExists(webPageStore);
+ assertSchemaExists("WebPage");
+ }
+
+ @Test
+ public void testPut() throws IOException {
+ log.info("test method: testPut");
+ Employee employee = DataStoreTestUtil.testPutEmployee(employeeStore);
+ assertPut(employee);
+ }
+
+ public void assertPut(Employee employee) throws IOException {
+ }
+
+ @Test
+ public void testPutNested() throws IOException {
+ log.info("test method: testPutNested");
+
+ String revUrl = "foo.com:http/";
+ String url = "http://foo.com/";
+
+ webPageStore.createSchema();
+ WebPage page = webPageStore.newPersistent();
+ Metadata metadata = new Metadata();
+ metadata.setVersion(1);
+ metadata.putToData(new Utf8("foo"), new Utf8("baz"));
+
+ page.setMetadata(metadata);
+ page.setUrl(new Utf8(url));
+
+ webPageStore.put(revUrl, page);
+ webPageStore.flush();
+
+ page = webPageStore.get(revUrl);
+ metadata = page.getMetadata();
+ Assert.assertNotNull(metadata);
+ Assert.assertEquals(1, metadata.getVersion());
+ Assert.assertEquals(new Utf8("baz"), metadata.getData().get(new Utf8("foo")));
+ }
+
+ @Test
+ public void testPutArray() throws IOException {
+ log.info("test method: testPutArray");
+ webPageStore.createSchema();
+ WebPage page = webPageStore.newPersistent();
+
+ String[] tokens = {"example", "content", "in", "example.com"};
+
+ for(String token: tokens) {
+ page.addToParsedContent(new Utf8(token));
+ }
+
+ webPageStore.put("com.example/http", page);
+ webPageStore.close();
+
+ assertPutArray();
+ }
+
+ public void assertPutArray() throws IOException {
+ }
+
+ @Test
+ public void testPutBytes() throws IOException {
+ log.info("test method: testPutBytes");
+ webPageStore.createSchema();
+ WebPage page = webPageStore.newPersistent();
+ page.setUrl(new Utf8("http://example.com"));
+ byte[] contentBytes = "example content in example.com".getBytes();
+ ByteBuffer buff = ByteBuffer.wrap(contentBytes);
+ page.setContent(buff);
+
+ webPageStore.put("com.example/http", page);
+ webPageStore.close();
+
+ assertPutBytes(contentBytes);
+ }
+
+ public void assertPutBytes(byte[] contentBytes) throws IOException {
+ }
+
+ @Test
+ public void testPutMap() throws IOException {
+ log.info("test method: testPutMap");
+ webPageStore.createSchema();
+
+ WebPage page = webPageStore.newPersistent();
+
+ page.setUrl(new Utf8("http://example.com"));
+ page.putToOutlinks(new Utf8("http://example2.com"), new Utf8("anchor2"));
+ page.putToOutlinks(new Utf8("http://example3.com"), new Utf8("anchor3"));
+ page.putToOutlinks(new Utf8("http://example3.com"), new Utf8("anchor4"));
+ webPageStore.put("com.example/http", page);
+ webPageStore.close();
+
+ assertPutMap();
+ }
+
+ public void assertPutMap() throws IOException {
+ }
+
+ @Test
+ public void testUpdate() throws IOException {
+ log.info("test method: testUpdate");
+ DataStoreTestUtil.testUpdateEmployee(employeeStore);
+ DataStoreTestUtil.testUpdateWebPage(webPageStore);
+ }
+
+ public void testEmptyUpdate() throws IOException {
+ DataStoreTestUtil.testEmptyUpdateEmployee(employeeStore);
+ }
+
+ @Test
+ public void testGet() throws IOException {
+ log.info("test method: testGet");
+ DataStoreTestUtil.testGetEmployee(employeeStore);
+ }
+
+ @Test
+ public void testGetWithFields() throws IOException {
+ log.info("test method: testGetWithFields");
+ DataStoreTestUtil.testGetEmployeeWithFields(employeeStore);
+ }
+
+ @Test
+ public void testGetWebPage() throws IOException {
+ log.info("test method: testGetWebPage");
+ DataStoreTestUtil.testGetWebPage(webPageStore);
+ }
+
+ @Test
+ public void testGetWebPageDefaultFields() throws IOException {
+ log.info("test method: testGetWebPageDefaultFields");
+ DataStoreTestUtil.testGetWebPageDefaultFields(webPageStore);
+ }
+
+ @Test
+ public void testGetNonExisting() throws Exception {
+ log.info("test method: testGetNonExisting");
+ DataStoreTestUtil.testGetEmployeeNonExisting(employeeStore);
+ }
+
+ @Test
+ public void testQuery() throws IOException {
+ log.info("test method: testQuery");
+ DataStoreTestUtil.testQueryWebPages(webPageStore);
+ }
+
+ @Test
+ public void testQueryStartKey() throws IOException {
+ log.info("test method: testQueryStartKey");
+ DataStoreTestUtil.testQueryWebPageStartKey(webPageStore);
+ }
+
+ @Test
+ public void testQueryEndKey() throws IOException {
+ log.info("test method: testQueryEndKey");
+ DataStoreTestUtil.testQueryWebPageEndKey(webPageStore);
+ }
+
+ @Test
+ public void testQueryKeyRange() throws IOException {
+ log.info("test method: testQueryKetRange");
+ DataStoreTestUtil.testQueryWebPageKeyRange(webPageStore);
+ }
+
+ @Test
+ public void testQueryWebPageSingleKey() throws IOException {
+ log.info("test method: testQueryWebPageSingleKey");
+ DataStoreTestUtil.testQueryWebPageSingleKey(webPageStore);
+ }
+
+ @Test
+ public void testQueryWebPageSingleKeyDefaultFields() throws IOException {
+ log.info("test method: testQuerySingleKeyDefaultFields");
+ DataStoreTestUtil.testQueryWebPageSingleKeyDefaultFields(webPageStore);
+ }
+
+ @Test
+ public void testQueryWebPageQueryEmptyResults() throws IOException {
+ log.info("test method: testQueryEmptyResults");
+ DataStoreTestUtil.testQueryWebPageEmptyResults(webPageStore);
+ }
+
+ @Test
+ public void testDelete() throws IOException {
+ log.info("test method: testDelete");
+ DataStoreTestUtil.testDelete(webPageStore);
+ }
+
+ @Test
+ public void testDeleteByQuery() throws IOException {
+ log.info("test method: testDeleteByQuery");
+ DataStoreTestUtil.testDeleteByQuery(webPageStore);
+ }
+
+ @Test
+ public void testDeleteByQueryFields() throws IOException {
+ log.info("test method: testQueryByQueryFields");
+ DataStoreTestUtil.testDeleteByQueryFields(webPageStore);
+ }
+
+ @Test
+ public void testGetPartitions() throws IOException {
+ log.info("test method: testGetPartitions");
+ DataStoreTestUtil.testGetPartitions(webPageStore);
+ }
+}
Added: incubator/gora/trunk/gora-core/src/test/java/org/apache/gora/store/DataStoreTestUtil.java
URL: http://svn.apache.org/viewvc/incubator/gora/trunk/gora-core/src/test/java/org/apache/gora/store/DataStoreTestUtil.java?rev=1021390&view=auto
==============================================================================
--- incubator/gora/trunk/gora-core/src/test/java/org/apache/gora/store/DataStoreTestUtil.java (added)
+++ incubator/gora/trunk/gora-core/src/test/java/org/apache/gora/store/DataStoreTestUtil.java Mon Oct 11 15:40:22 2010
@@ -0,0 +1,704 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.gora.store;
+
+import static org.apache.gora.examples.WebPageDataCreator.ANCHORS;
+import static org.apache.gora.examples.WebPageDataCreator.CONTENTS;
+import static org.apache.gora.examples.WebPageDataCreator.LINKS;
+import static org.apache.gora.examples.WebPageDataCreator.SORTED_URLS;
+import static org.apache.gora.examples.WebPageDataCreator.URLS;
+import static org.apache.gora.examples.WebPageDataCreator.URL_INDEXES;
+import static org.apache.gora.examples.WebPageDataCreator.createWebPageData;
+
+import java.io.IOException;
+import java.nio.ByteBuffer;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+
+import junit.framework.Assert;
+
+import org.apache.avro.generic.GenericArray;
+import org.apache.avro.util.Utf8;
+import org.apache.gora.examples.WebPageDataCreator;
+import org.apache.gora.examples.generated.Employee;
+import org.apache.gora.examples.generated.WebPage;
+import org.apache.gora.persistency.Persistent;
+import org.apache.gora.query.PartitionQuery;
+import org.apache.gora.query.Query;
+import org.apache.gora.query.Result;
+import org.apache.gora.store.DataStore;
+import org.apache.gora.util.ByteUtils;
+import org.apache.gora.util.StringUtils;
+
+/**
+ * Test utilities for DataStores
+ */
+public class DataStoreTestUtil {
+
+ public static final long YEAR_IN_MS = 365L * 24L * 60L * 60L * 1000L;
+ private static final int NUM_KEYS = 4;
+
+ public static <K, T extends Persistent> void testNewPersistent(
+ DataStore<K,T> dataStore) throws IOException {
+
+ T obj1 = dataStore.newPersistent();
+ T obj2 = dataStore.newPersistent();
+
+ Assert.assertEquals(dataStore.getPersistentClass(),
+ obj1.getClass());
+ Assert.assertNotNull(obj1);
+ Assert.assertNotNull(obj2);
+ Assert.assertFalse( obj1 == obj2 );
+ }
+
+ public static <K> Employee createEmployee(
+ DataStore<K, Employee> dataStore) throws IOException {
+
+ Employee employee = dataStore.newPersistent();
+ employee.setName(new Utf8("Random Joe"));
+ employee.setDateOfBirth( System.currentTimeMillis() - 20L * YEAR_IN_MS );
+ employee.setSalary(100000);
+ employee.setSsn(new Utf8("101010101010"));
+ return employee;
+ }
+
+ public static void testAutoCreateSchema(DataStore<String,Employee> dataStore)
+ throws IOException {
+ //should not throw exception
+ dataStore.put("foo", createEmployee(dataStore));
+ }
+
+ public static void testCreateEmployeeSchema(DataStore<String, Employee> dataStore)
+ throws IOException {
+ dataStore.createSchema();
+
+ //should not throw exception
+ dataStore.createSchema();
+ }
+
+ public static void testTruncateSchema(DataStore<String, WebPage> dataStore)
+ throws IOException {
+ dataStore.createSchema();
+ WebPageDataCreator.createWebPageData(dataStore);
+ dataStore.truncateSchema();
+
+ assertEmptyResults(dataStore.newQuery());
+ }
+
+ public static void testDeleteSchema(DataStore<String, WebPage> dataStore)
+ throws IOException {
+ dataStore.createSchema();
+ WebPageDataCreator.createWebPageData(dataStore);
+ dataStore.deleteSchema();
+ dataStore.createSchema();
+
+ assertEmptyResults(dataStore.newQuery());
+ }
+
+ public static<K, T extends Persistent> void testSchemaExists(
+ DataStore<K, T> dataStore) throws IOException {
+ dataStore.createSchema();
+
+ Assert.assertTrue(dataStore.schemaExists());
+
+ dataStore.deleteSchema();
+ Assert.assertFalse(dataStore.schemaExists());
+ }
+
+ public static void testGetEmployee(DataStore<String, Employee> dataStore)
+ throws IOException {
+ dataStore.createSchema();
+ Employee employee = DataStoreTestUtil.createEmployee(dataStore);
+ String ssn = employee.getSsn().toString();
+ dataStore.put(ssn, employee);
+ dataStore.flush();
+
+ Employee after = dataStore.get(ssn, Employee._ALL_FIELDS);
+
+ Assert.assertEquals(employee, after);
+ }
+
+ public static void testGetEmployeeNonExisting(DataStore<String, Employee> dataStore)
+ throws IOException {
+ Employee employee = dataStore.get("_NON_EXISTING_SSN_FOR_EMPLOYEE_");
+ Assert.assertNull(employee);
+ }
+
+ public static void testGetEmployeeWithFields(DataStore<String, Employee> dataStore)
+ throws IOException {
+ Employee employee = DataStoreTestUtil.createEmployee(dataStore);
+ String ssn = employee.getSsn().toString();
+ dataStore.put(ssn, employee);
+ dataStore.flush();
+
+ String[] fields = employee.getFields();
+ for(Set<String> subset : StringUtils.powerset(fields)) {
+ if(subset.isEmpty())
+ continue;
+ Employee after = dataStore.get(ssn, subset.toArray(new String[subset.size()]));
+ Employee expected = new Employee();
+ for(String field:subset) {
+ int index = expected.getFieldIndex(field);
+ expected.put(index, employee.get(index));
+ }
+
+ Assert.assertEquals(expected, after);
+ }
+ }
+
+ public static Employee testPutEmployee(DataStore<String, Employee> dataStore)
+ throws IOException {
+ dataStore.createSchema();
+ Employee employee = DataStoreTestUtil.createEmployee(dataStore);
+ return employee;
+ }
+
+ public static void testEmptyUpdateEmployee(DataStore<String, Employee> dataStore)
+ throws IOException {
+ dataStore.createSchema();
+ long ssn = 1234567890L;
+ String ssnStr = Long.toString(ssn);
+ long now = System.currentTimeMillis();
+
+ Employee employee = dataStore.newPersistent();
+ employee.setName(new Utf8("John Doe"));
+ employee.setDateOfBirth(now - 20L * YEAR_IN_MS);
+ employee.setSalary(100000);
+ employee.setSsn(new Utf8(ssnStr));
+ dataStore.put(employee.getSsn().toString(), employee);
+
+ dataStore.flush();
+
+ employee = dataStore.get(ssnStr);
+ dataStore.put(ssnStr, employee);
+
+ dataStore.flush();
+
+ employee = dataStore.newPersistent();
+ dataStore.put(Long.toString(ssn + 1), employee);
+
+ dataStore.flush();
+
+ employee = dataStore.get(Long.toString(ssn + 1));
+ Assert.assertNull(employee);
+ }
+
+ public static void testUpdateEmployee(DataStore<String, Employee> dataStore)
+ throws IOException {
+ dataStore.createSchema();
+ long ssn = 1234567890L;
+ long now = System.currentTimeMillis();
+
+ for (int i = 0; i < 5; i++) {
+ Employee employee = dataStore.newPersistent();
+ employee.setName(new Utf8("John Doe " + i));
+ employee.setDateOfBirth(now - 20L * YEAR_IN_MS);
+ employee.setSalary(100000);
+ employee.setSsn(new Utf8(Long.toString(ssn + i)));
+ dataStore.put(employee.getSsn().toString(), employee);
+ }
+
+ dataStore.flush();
+
+ for (int i = 0; i < 1; i++) {
+ Employee employee = dataStore.newPersistent();
+ employee.setName(new Utf8("John Doe " + (i + 5)));
+ employee.setDateOfBirth(now - 18L * YEAR_IN_MS);
+ employee.setSalary(120000);
+ employee.setSsn(new Utf8(Long.toString(ssn + i)));
+ dataStore.put(employee.getSsn().toString(), employee);
+ }
+
+ dataStore.flush();
+
+ for (int i = 0; i < 1; i++) {
+ String key = Long.toString(ssn + i);
+ Employee employee = dataStore.get(key);
+ Assert.assertEquals(now - 18L * YEAR_IN_MS, employee.getDateOfBirth());
+ Assert.assertEquals("John Doe " + (i + 5), employee.getName().toString());
+ Assert.assertEquals(120000, employee.getSalary());
+ }
+ }
+
+ public static void testUpdateWebPage(DataStore<String, WebPage> dataStore)
+ throws IOException {
+ dataStore.createSchema();
+
+ String[] urls = {"http://a.com/a", "http://b.com/b", "http://c.com/c",
+ "http://d.com/d", "http://e.com/e", "http://f.com/f", "http://g.com/g"};
+ String content = "content";
+ String parsedContent = "parsedContent";
+ String anchor = "anchor";
+
+ int parsedContentCount = 0;
+
+
+ for (int i = 0; i < urls.length; i++) {
+ WebPage webPage = dataStore.newPersistent();
+ webPage.setUrl(new Utf8(urls[i]));
+ for (parsedContentCount = 0; parsedContentCount < 5; parsedContentCount++) {
+ webPage.addToParsedContent(new Utf8(parsedContent + i + "," + parsedContentCount));
+ }
+ for (int j = 0; j < urls.length; j += 2) {
+ webPage.putToOutlinks(new Utf8(anchor + j), new Utf8(urls[j]));
+ }
+ dataStore.put(webPage.getUrl().toString(), webPage);
+ }
+
+ dataStore.flush();
+
+ for (int i = 0; i < urls.length; i++) {
+ WebPage webPage = dataStore.get(urls[i]);
+ webPage.setContent(ByteBuffer.wrap(ByteUtils.toBytes(content + i)));
+ for (parsedContentCount = 5; parsedContentCount < 10; parsedContentCount++) {
+ webPage.addToParsedContent(new Utf8(parsedContent + i + "," + parsedContentCount));
+ }
+ webPage.getOutlinks().clear();
+ for (int j = 1; j < urls.length; j += 2) {
+ webPage.putToOutlinks(new Utf8(anchor + j), new Utf8(urls[j]));
+ }
+ dataStore.put(webPage.getUrl().toString(), webPage);
+ }
+
+ dataStore.flush();
+
+ for (int i = 0; i < urls.length; i++) {
+ WebPage webPage = dataStore.get(urls[i]);
+ Assert.assertEquals(content + i, ByteUtils.toString(webPage.getContent().array()));
+ Assert.assertEquals(10, webPage.getParsedContent().size());
+ int j = 0;
+ for (Utf8 pc : webPage.getParsedContent()) {
+ Assert.assertEquals(parsedContent + i + "," + j, pc.toString());
+ j++;
+ }
+ int count = 0;
+ for (j = 1; j < urls.length; j += 2) {
+ Utf8 link = webPage.getOutlinks().get(new Utf8(anchor + j));
+ Assert.assertNotNull(link);
+ Assert.assertEquals(urls[j], link.toString());
+ count++;
+ }
+ Assert.assertEquals(count, webPage.getOutlinks().size());
+ }
+
+ for (int i = 0; i < urls.length; i++) {
+ WebPage webPage = dataStore.get(urls[i]);
+ for (int j = 0; j < urls.length; j += 2) {
+ webPage.putToOutlinks(new Utf8(anchor + j), new Utf8(urls[j]));
+ }
+ dataStore.put(webPage.getUrl().toString(), webPage);
+ }
+
+ dataStore.flush();
+
+ for (int i = 0; i < urls.length; i++) {
+ WebPage webPage = dataStore.get(urls[i]);
+ int count = 0;
+ for (int j = 0; j < urls.length; j++) {
+ Utf8 link = webPage.getOutlinks().get(new Utf8(anchor + j));
+ Assert.assertNotNull(link);
+ Assert.assertEquals(urls[j], link.toString());
+ count++;
+ }
+ }
+ }
+
+ public static void assertWebPage(WebPage page, int i) {
+ Assert.assertNotNull(page);
+
+ Assert.assertEquals(URLS[i], page.getUrl().toString());
+ Assert.assertTrue("content error:" + new String(page.getContent().array()) +
+ " actual=" + CONTENTS[i] + " i=" + i
+ , Arrays.equals(page.getContent().array()
+ , CONTENTS[i].getBytes()));
+
+ GenericArray<Utf8> parsedContent = page.getParsedContent();
+ Assert.assertNotNull(parsedContent);
+ Assert.assertTrue(parsedContent.size() > 0);
+
+ int j=0;
+ String[] tokens = CONTENTS[i].split(" ");
+ for(Utf8 token : parsedContent) {
+ Assert.assertEquals(tokens[j++], token.toString());
+ }
+
+ if(LINKS[i].length > 0) {
+ Assert.assertNotNull(page.getOutlinks());
+ Assert.assertTrue(page.getOutlinks().size() > 0);
+ for(j=0; j<LINKS[i].length; j++) {
+ Assert.assertEquals(ANCHORS[i][j],
+ page.getFromOutlinks(new Utf8(URLS[LINKS[i][j]])).toString());
+ }
+ } else {
+ Assert.assertTrue(page.getOutlinks() == null || page.getOutlinks().isEmpty());
+ }
+ }
+
+ private static void testGetWebPage(DataStore<String, WebPage> store, String[] fields)
+ throws IOException {
+ createWebPageData(store);
+
+ for(int i=0; i<URLS.length; i++) {
+ WebPage page = store.get(URLS[i], fields);
+ assertWebPage(page, i);
+ }
+ }
+
+ public static void testGetWebPage(DataStore<String, WebPage> store) throws IOException {
+ testGetWebPage(store, WebPage._ALL_FIELDS);
+ }
+
+ public static void testGetWebPageDefaultFields(DataStore<String, WebPage> store)
+ throws IOException {
+ testGetWebPage(store, null);
+ }
+
+ private static void testQueryWebPageSingleKey(DataStore<String, WebPage> store
+ , String[] fields) throws IOException {
+
+ createWebPageData(store);
+
+ for(int i=0; i<URLS.length; i++) {
+ Query<String, WebPage> query = store.newQuery();
+ query.setFields(fields);
+ query.setKey(URLS[i]);
+ Result<String, WebPage> result = query.execute();
+ Assert.assertTrue(result.next());
+ WebPage page = result.get();
+ assertWebPage(page, i);
+ Assert.assertFalse(result.next());
+ }
+ }
+
+ public static void testQueryWebPageSingleKey(DataStore<String, WebPage> store)
+ throws IOException {
+ testQueryWebPageSingleKey(store, WebPage._ALL_FIELDS);
+ }
+
+ public static void testQueryWebPageSingleKeyDefaultFields(
+ DataStore<String, WebPage> store) throws IOException {
+ testQueryWebPageSingleKey(store, null);
+ }
+
+ public static void testQueryWebPageKeyRange(DataStore<String, WebPage> store,
+ boolean setStartKeys, boolean setEndKeys)
+ throws IOException {
+ createWebPageData(store);
+
+ //create sorted set of urls
+ List<String> sortedUrls = new ArrayList<String>();
+ for(String url: URLS) {
+ sortedUrls.add(url);
+ }
+ Collections.sort(sortedUrls);
+
+ //try all ranges
+ for(int i=0; i<sortedUrls.size(); i++) {
+ for(int j=i; j<sortedUrls.size(); j++) {
+ Query<String, WebPage> query = store.newQuery();
+ if(setStartKeys)
+ query.setStartKey(sortedUrls.get(i));
+ if(setEndKeys)
+ query.setEndKey(sortedUrls.get(j));
+ Result<String, WebPage> result = query.execute();
+
+ int r=0;
+ while(result.next()) {
+ WebPage page = result.get();
+ assertWebPage(page, URL_INDEXES.get(page.getUrl().toString()));
+ r++;
+ }
+
+ int expectedLength = (setEndKeys ? j+1: sortedUrls.size()) -
+ (setStartKeys ? i: 0);
+ Assert.assertEquals(expectedLength, r);
+ if(!setEndKeys)
+ break;
+ }
+ if(!setStartKeys)
+ break;
+ }
+ }
+
+ public static void testQueryWebPages(DataStore<String, WebPage> store)
+ throws IOException {
+ testQueryWebPageKeyRange(store, false, false);
+ }
+
+ public static void testQueryWebPageStartKey(DataStore<String, WebPage> store)
+ throws IOException {
+ testQueryWebPageKeyRange(store, true, false);
+ }
+
+ public static void testQueryWebPageEndKey(DataStore<String, WebPage> store)
+ throws IOException {
+ testQueryWebPageKeyRange(store, false, true);
+ }
+
+ public static void testQueryWebPageKeyRange(DataStore<String, WebPage> store)
+ throws IOException {
+ testQueryWebPageKeyRange(store, true, true);
+ }
+
+ public static void testQueryWebPageEmptyResults(DataStore<String, WebPage> store)
+ throws IOException {
+ createWebPageData(store);
+
+ //query empty results
+ Query<String, WebPage> query = store.newQuery();
+ query.setStartKey("aa");
+ query.setEndKey("ab");
+ assertEmptyResults(query);
+
+ //query empty results for one key
+ query = store.newQuery();
+ query.setKey("aa");
+ assertEmptyResults(query);
+ }
+
+ public static<K,T extends Persistent> void assertEmptyResults(Query<K, T> query)
+ throws IOException {
+ assertNumResults(query, 0);
+ }
+
+ public static<K,T extends Persistent> void assertNumResults(Query<K, T>query
+ , long numResults) throws IOException {
+ Result<K, T> result = query.execute();
+ int actualNumResults = 0;
+ while(result.next()) {
+ actualNumResults++;
+ }
+ result.close();
+ Assert.assertEquals(numResults, actualNumResults);
+ }
+
+ public static void testGetPartitions(DataStore<String, WebPage> store)
+ throws IOException {
+ createWebPageData(store);
+ testGetPartitions(store, store.newQuery());
+ }
+
+ public static void testGetPartitions(DataStore<String, WebPage> store
+ , Query<String, WebPage> query) throws IOException {
+ List<PartitionQuery<String, WebPage>> partitions = store.getPartitions(query);
+
+ Assert.assertNotNull(partitions);
+ Assert.assertTrue(partitions.size() > 0);
+
+ for(PartitionQuery<String, WebPage> partition:partitions) {
+ Assert.assertNotNull(partition);
+ }
+
+ assertPartitions(store, query, partitions);
+ }
+
+ public static void assertPartitions(DataStore<String, WebPage> store,
+ Query<String, WebPage> query, List<PartitionQuery<String,WebPage>> partitions)
+ throws IOException {
+
+ int count = 0, partitionsCount = 0;
+ Map<String, Integer> results = new HashMap<String, Integer>();
+ Map<String, Integer> partitionResults = new HashMap<String, Integer>();
+
+ //execute query and count results
+ Result<String, WebPage> result = store.execute(query);
+ Assert.assertNotNull(result);
+
+ while(result.next()) {
+ Assert.assertNotNull(result.getKey());
+ Assert.assertNotNull(result.get());
+ results.put(result.getKey(), result.get().hashCode()); //keys are not reused, so this is safe
+ count++;
+ }
+ result.close();
+
+ Assert.assertTrue(count > 0); //assert that results is not empty
+ Assert.assertEquals(count, results.size()); //assert that keys are unique
+
+ for(PartitionQuery<String, WebPage> partition:partitions) {
+ Assert.assertNotNull(partition);
+
+ result = store.execute(partition);
+ Assert.assertNotNull(result);
+
+ while(result.next()) {
+ Assert.assertNotNull(result.getKey());
+ Assert.assertNotNull(result.get());
+ partitionResults.put(result.getKey(), result.get().hashCode());
+ partitionsCount++;
+ }
+ result.close();
+
+ Assert.assertEquals(partitionsCount, partitionResults.size()); //assert that keys are unique
+ }
+
+ Assert.assertTrue(partitionsCount > 0);
+ Assert.assertEquals(count, partitionsCount);
+
+ for(Map.Entry<String, Integer> r : results.entrySet()) {
+ Integer p = partitionResults.get(r.getKey());
+ Assert.assertNotNull(p);
+ Assert.assertEquals(r.getValue(), p);
+ }
+ }
+
+ public static void testDelete(DataStore<String, WebPage> store) throws IOException {
+ WebPageDataCreator.createWebPageData(store);
+ //delete one by one
+
+ int deletedSoFar = 0;
+ for(String url : URLS) {
+ Assert.assertTrue(store.delete(url));
+ store.flush();
+
+ //assert that it is actually deleted
+ Assert.assertNull(store.get(url));
+
+ //assert that other records are not deleted
+ assertNumResults(store.newQuery(), URLS.length - ++deletedSoFar);
+ }
+ }
+
+ public static void testDeleteByQuery(DataStore<String, WebPage> store)
+ throws IOException {
+
+ Query<String, WebPage> query;
+
+ //test 1 - delete all
+ WebPageDataCreator.createWebPageData(store);
+
+ query = store.newQuery();
+
+ assertNumResults(store.newQuery(), URLS.length);
+ store.deleteByQuery(query);
+ assertEmptyResults(store.newQuery());
+
+
+ //test 2 - delete all
+ WebPageDataCreator.createWebPageData(store);
+
+ query = store.newQuery();
+ query.setFields(WebPage._ALL_FIELDS);
+
+ assertNumResults(store.newQuery(), URLS.length);
+ store.deleteByQuery(query);
+ assertEmptyResults(store.newQuery());
+
+
+ //test 3 - delete all
+ WebPageDataCreator.createWebPageData(store);
+
+ query = store.newQuery();
+ query.setKeyRange("a", "z"); //all start with "http://"
+
+ assertNumResults(store.newQuery(), URLS.length);
+ store.deleteByQuery(query);
+ assertEmptyResults(store.newQuery());
+
+
+ //test 4 - delete some
+ WebPageDataCreator.createWebPageData(store);
+ query = store.newQuery();
+ query.setEndKey(SORTED_URLS[NUM_KEYS]);
+
+ assertNumResults(store.newQuery(), URLS.length);
+ store.deleteByQuery(query);
+ assertNumResults(store.newQuery(), URLS.length - (NUM_KEYS+1));
+
+ store.truncateSchema();
+
+ }
+
+ public static void testDeleteByQueryFields(DataStore<String, WebPage> store)
+ throws IOException {
+
+ Query<String, WebPage> query;
+
+ //test 5 - delete all with some fields
+ WebPageDataCreator.createWebPageData(store);
+
+ query = store.newQuery();
+ query.setFields(WebPage.Field.OUTLINKS.getName()
+ , WebPage.Field.PARSED_CONTENT.getName(), WebPage.Field.CONTENT.getName());
+
+ assertNumResults(store.newQuery(), URLS.length);
+ store.deleteByQuery(query);
+ store.deleteByQuery(query);
+ store.deleteByQuery(query);//don't you love that HBase sometimes does not delete arbitrarily
+ assertNumResults(store.newQuery(), URLS.length);
+
+ //assert that data is deleted
+ for (int i = 0; i < SORTED_URLS.length; i++) {
+ WebPage page = store.get(SORTED_URLS[i]);
+ Assert.assertNotNull(page);
+
+ Assert.assertNotNull(page.getUrl());
+ Assert.assertEquals(page.getUrl().toString(), SORTED_URLS[i]);
+ Assert.assertEquals(0, page.getOutlinks().size());
+ Assert.assertEquals(0, page.getParsedContent().size());
+ if(page.getContent() != null) {
+ System.out.println("url:" + page.getUrl().toString());
+ System.out.println( "limit:" + page.getContent().limit());
+ } else {
+ Assert.assertNull(page.getContent());
+ }
+ }
+
+ //test 6 - delete some with some fields
+ WebPageDataCreator.createWebPageData(store);
+
+ query = store.newQuery();
+ query.setFields(WebPage.Field.URL.getName());
+ String startKey = SORTED_URLS[NUM_KEYS];
+ String endKey = SORTED_URLS[SORTED_URLS.length - NUM_KEYS];
+ query.setStartKey(startKey);
+ query.setEndKey(endKey);
+
+ assertNumResults(store.newQuery(), URLS.length);
+ store.deleteByQuery(query);
+ store.deleteByQuery(query);
+ store.deleteByQuery(query);//don't you love that HBase sometimes does not delete arbitrarily
+
+ assertNumResults(store.newQuery(), URLS.length);
+
+ //assert that data is deleted
+ for (int i = 0; i < URLS.length; i++) {
+ WebPage page = store.get(URLS[i]);
+ Assert.assertNotNull(page);
+ if( URLS[i].compareTo(startKey) < 0 || URLS[i].compareTo(endKey) >= 0) {
+ //not deleted
+ assertWebPage(page, i);
+ } else {
+ //deleted
+ Assert.assertNull(page.getUrl());
+ Assert.assertNotNull(page.getOutlinks());
+ Assert.assertNotNull(page.getParsedContent());
+ Assert.assertNotNull(page.getContent());
+ Assert.assertTrue(page.getOutlinks().size() > 0);
+ Assert.assertTrue(page.getParsedContent().size() > 0);
+ }
+ }
+
+ }
+}