You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@gora.apache.org by le...@apache.org on 2014/06/04 18:36:24 UTC
[07/50] [abbrv] GORA-321. Merge GORA_94 into Gora trunk
http://git-wip-us.apache.org/repos/asf/gora/blob/136fc595/gora-core/src/main/java/org/apache/gora/util/IOUtils.java
----------------------------------------------------------------------
diff --git a/gora-core/src/main/java/org/apache/gora/util/IOUtils.java b/gora-core/src/main/java/org/apache/gora/util/IOUtils.java
index 094262e..a8fa313 100644
--- a/gora-core/src/main/java/org/apache/gora/util/IOUtils.java
+++ b/gora-core/src/main/java/org/apache/gora/util/IOUtils.java
@@ -38,11 +38,12 @@ import org.apache.avro.io.BinaryEncoder;
import org.apache.avro.io.Decoder;
import org.apache.avro.io.DecoderFactory;
import org.apache.avro.io.Encoder;
-import org.apache.avro.ipc.ByteBufferInputStream;
-import org.apache.avro.ipc.ByteBufferOutputStream;
-import org.apache.gora.avro.PersistentDatumReader;
-import org.apache.gora.avro.PersistentDatumWriter;
-import org.apache.gora.persistency.impl.PersistentBase;
+import org.apache.avro.io.EncoderFactory;
+import org.apache.avro.specific.SpecificDatumReader;
+import org.apache.avro.specific.SpecificDatumWriter;
+import org.apache.avro.specific.SpecificRecord;
+import org.apache.avro.util.ByteBufferInputStream;
+import org.apache.avro.util.ByteBufferOutputStream;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.io.DataInputBuffer;
import org.apache.hadoop.io.DataOutputBuffer;
@@ -152,25 +153,47 @@ public class IOUtils {
/**
* Serializes the field object using the datumWriter.
*/
- public static<T extends PersistentBase> void serialize(OutputStream os,
- PersistentDatumWriter<T> datumWriter, Schema schema, Object object)
+ public static<T extends SpecificRecord> void serialize(OutputStream os,
+ SpecificDatumWriter<T> datumWriter, Schema schema, T object)
throws IOException {
- BinaryEncoder encoder = new BinaryEncoder(os);
- datumWriter.write(schema, object, encoder);
+ BinaryEncoder encoder = EncoderFactory.get().binaryEncoder(os, null);
+ datumWriter.write(object, encoder);
encoder.flush();
}
/**
* Serializes the field object using the datumWriter.
*/
- public static<T extends PersistentBase> byte[] serialize(PersistentDatumWriter<T> datumWriter
- , Schema schema, Object object) throws IOException {
+ public static<T> void serialize(OutputStream os,
+ SpecificDatumWriter<T> datumWriter, Schema schema, T object)
+ throws IOException {
+
+ BinaryEncoder encoder = EncoderFactory.get().binaryEncoder(os, null);
+ datumWriter.write(object, encoder);
+ encoder.flush();
+ }
+
+ /**
+ * Serializes the field object using the datumWriter.
+ */
+ public static<T extends SpecificRecord> byte[] serialize(SpecificDatumWriter<T> datumWriter
+ , Schema schema, T object) throws IOException {
ByteArrayOutputStream os = new ByteArrayOutputStream();
serialize(os, datumWriter, schema, object);
return os.toByteArray();
}
+ /**
+ * Serializes the field object using the datumWriter.
+ */
+ public static<T> byte[] serialize(SpecificDatumWriter<T> datumWriter
+ , Schema schema, T object) throws IOException {
+ ByteArrayOutputStream os = new ByteArrayOutputStream();
+ serialize(os, datumWriter, schema, object);
+ return os.toByteArray();
+ }
+
/** Deserializes the object in the given datainput using
* available Hadoop serializations.
* @throws IOException
@@ -239,36 +262,33 @@ public class IOUtils {
/**
* Deserializes the field object using the datumReader.
*/
- @SuppressWarnings("unchecked")
- public static<K, T extends PersistentBase> K deserialize(InputStream is,
- PersistentDatumReader<T> datumReader, Schema schema, K object)
+ public static<K, T extends SpecificRecord> T deserialize(InputStream is,
+ SpecificDatumReader<T> datumReader, Schema schema, T object)
throws IOException {
- decoder = DecoderFactory.defaultFactory().createBinaryDecoder(is, decoder);
- return (K)datumReader.read(object, schema, decoder);
+ decoder = DecoderFactory.get().binaryDecoder(is, decoder);
+ return (T)datumReader.read(object, decoder);
}
/**
* Deserializes the field object using the datumReader.
*/
- @SuppressWarnings("unchecked")
- public static<K, T extends PersistentBase> K deserialize(byte[] bytes,
- PersistentDatumReader<T> datumReader, Schema schema, K object)
+ public static<K, T extends SpecificRecord> T deserialize(byte[] bytes,
+ SpecificDatumReader<T> datumReader, Schema schema, T object)
throws IOException {
- decoder = DecoderFactory.defaultFactory().createBinaryDecoder(bytes, decoder);
- return (K)datumReader.read(object, schema, decoder);
+ decoder = DecoderFactory.get().binaryDecoder(bytes, decoder);
+ return (T)datumReader.read(object, decoder);
}
-
/**
- * Serializes the field object using the datumWriter.
+ * Deserializes the field object using the datumReader.
*/
- public static<T extends PersistentBase> byte[] deserialize(PersistentDatumWriter<T> datumWriter
- , Schema schema, Object object) throws IOException {
- ByteArrayOutputStream os = new ByteArrayOutputStream();
- serialize(os, datumWriter, schema, object);
- return os.toByteArray();
+ public static<K, T> T deserialize(byte[] bytes,
+ SpecificDatumReader<T> datumReader, Schema schema, T object)
+ throws IOException {
+ decoder = DecoderFactory.get().binaryDecoder(bytes, decoder);
+ return (T)datumReader.read(object, decoder);
}
-
+
/**
* Writes a byte[] to the output, representing whether each given field is null
* or not. A Vint and ceil( fields.length / 8 ) bytes are written to the output.
http://git-wip-us.apache.org/repos/asf/gora/blob/136fc595/gora-core/src/main/java/org/apache/gora/util/ReflectionUtils.java
----------------------------------------------------------------------
diff --git a/gora-core/src/main/java/org/apache/gora/util/ReflectionUtils.java b/gora-core/src/main/java/org/apache/gora/util/ReflectionUtils.java
index fd8c498..00f94e4 100644
--- a/gora-core/src/main/java/org/apache/gora/util/ReflectionUtils.java
+++ b/gora-core/src/main/java/org/apache/gora/util/ReflectionUtils.java
@@ -20,6 +20,10 @@ package org.apache.gora.util;
import java.lang.reflect.Constructor;
import java.lang.reflect.InvocationTargetException;
+import java.lang.reflect.Method;
+
+import org.apache.avro.specific.SpecificRecordBuilderBase;
+import org.apache.gora.persistency.Persistent;
/**
* Utility methods related to reflection
@@ -100,4 +104,10 @@ public class ReflectionUtils {
return clazz.getField(fieldName).get(null);
}
+
+ public static <T extends Persistent> SpecificRecordBuilderBase<T> classBuilder(Class<T> clazz) throws SecurityException
+ , NoSuchMethodException, IllegalArgumentException, IllegalAccessException, InvocationTargetException {
+ return (SpecificRecordBuilderBase<T>) clazz.getMethod("newBuilder").invoke(null);
+ }
+
}
http://git-wip-us.apache.org/repos/asf/gora/blob/136fc595/gora-core/src/test/conf/hadoop-metrics2.properties
----------------------------------------------------------------------
diff --git a/gora-core/src/test/conf/hadoop-metrics2.properties b/gora-core/src/test/conf/hadoop-metrics2.properties
new file mode 100644
index 0000000..e5a9785
--- /dev/null
+++ b/gora-core/src/test/conf/hadoop-metrics2.properties
@@ -0,0 +1,8 @@
+# Configuration of the "dfs" context for null
+dfs.class=org.apache.hadoop.metrics.spi.NullContext
+
+# Configuration of the "mapred" context for null
+mapred.class=org.apache.hadoop.metrics.spi.NullContext
+
+# Configuration of the "jvm" context for null
+jvm.class=org.apache.hadoop.metrics.spi.NullContext
http://git-wip-us.apache.org/repos/asf/gora/blob/136fc595/gora-core/src/test/java/org/apache/gora/GoraTestDriver.java
----------------------------------------------------------------------
diff --git a/gora-core/src/test/java/org/apache/gora/GoraTestDriver.java b/gora-core/src/test/java/org/apache/gora/GoraTestDriver.java
index e5170c2..817d57b 100644
--- a/gora-core/src/test/java/org/apache/gora/GoraTestDriver.java
+++ b/gora-core/src/test/java/org/apache/gora/GoraTestDriver.java
@@ -18,7 +18,6 @@
package org.apache.gora;
-import java.io.IOException;
import java.util.HashSet;
import java.util.Properties;
@@ -40,15 +39,15 @@ public class GoraTestDriver {
protected static final Logger log = LoggerFactory.getLogger(GoraTestDriver.class);
- protected Class<? extends DataStore> dataStoreClass;
+ protected Class<? extends DataStore<?, ?>> dataStoreClass;
protected Configuration conf = new Configuration();
@SuppressWarnings("rawtypes")
protected HashSet<DataStore> dataStores;
- @SuppressWarnings("rawtypes")
+ @SuppressWarnings({ "rawtypes", "unchecked" })
protected GoraTestDriver(Class<? extends DataStore> dataStoreClass) {
- this.dataStoreClass = dataStoreClass;
+ this.dataStoreClass = (Class<? extends DataStore<?, ?>>) dataStoreClass;
this.dataStores = new HashSet<DataStore>();
}
@@ -71,7 +70,7 @@ public class GoraTestDriver {
*/
public void setUp() throws Exception {
log.info("setting up test");
- for(DataStore store : dataStores) {
+ for(DataStore<?, ?> store : dataStores) {
store.truncateSchema();
}
}
@@ -112,4 +111,8 @@ public class GoraTestDriver {
public Class<?> getDataStoreClass() {
return dataStoreClass;
}
+
+ public Configuration getConfiguration() {
+ return this.conf;
+ }
}
http://git-wip-us.apache.org/repos/asf/gora/blob/136fc595/gora-core/src/test/java/org/apache/gora/avro/TestPersistentDatumReader.java
----------------------------------------------------------------------
diff --git a/gora-core/src/test/java/org/apache/gora/avro/TestPersistentDatumReader.java b/gora-core/src/test/java/org/apache/gora/avro/TestPersistentDatumReader.java
index a81fd3b..e69de29 100644
--- a/gora-core/src/test/java/org/apache/gora/avro/TestPersistentDatumReader.java
+++ b/gora-core/src/test/java/org/apache/gora/avro/TestPersistentDatumReader.java
@@ -1,104 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.gora.avro;
-
-import java.io.IOException;
-
-import org.apache.avro.util.Utf8;
-import org.apache.gora.examples.WebPageDataCreator;
-import org.apache.gora.examples.generated.Employee;
-import org.apache.gora.examples.generated.WebPage;
-import org.apache.gora.memory.store.MemStore;
-import org.apache.gora.persistency.impl.PersistentBase;
-import org.apache.gora.query.Query;
-import org.apache.gora.query.Result;
-import org.apache.gora.store.DataStore;
-import org.apache.gora.store.DataStoreFactory;
-import org.apache.gora.store.DataStoreTestUtil;
-import org.apache.hadoop.conf.Configuration;
-import org.junit.Test;
-import static org.junit.Assert.assertNotNull;
-import static org.junit.Assert.assertEquals;
-
-/**
- * Test case for {@link PersistentDatumReader}.
- */
-public class TestPersistentDatumReader {
-
- private PersistentDatumReader<WebPage> webPageDatumReader
- = new PersistentDatumReader<WebPage>();
- private Configuration conf = new Configuration();
-
- private void testClone(PersistentBase persistent) throws IOException {
- PersistentBase cloned = ((PersistentBase)webPageDatumReader.clone(persistent, persistent.getSchema()));
- assertClone(persistent, cloned);
- }
-
- private void assertClone(PersistentBase persistent, PersistentBase cloned) {
- assertNotNull("cloned object is null", cloned);
- assertEquals("cloned object is not equal to original object", persistent, cloned);
- }
-
- @Test
- public void testCloneEmployee() throws Exception {
- @SuppressWarnings("unchecked")
- MemStore<String, Employee> store = DataStoreFactory.getDataStore(
- MemStore.class, String.class, Employee.class, conf);
-
- Employee employee = DataStoreTestUtil.createEmployee(store);
-
- testClone(employee);
- }
-
- @Test
- public void testCloneEmployeeOneField() throws Exception {
- Employee employee = new Employee();
- employee.setSsn(new Utf8("11111"));
-
- testClone(employee);
- }
-
- @Test
- public void testCloneEmployeeTwoFields() throws Exception {
- Employee employee = new Employee();
- employee.setSsn(new Utf8("11111"));
- employee.setSalary(100);
-
- testClone(employee);
- }
-
- @Test
- public void testCloneWebPage() throws Exception {
- @SuppressWarnings("unchecked")
- DataStore<String, WebPage> store = DataStoreFactory.createDataStore(
- MemStore.class, String.class, WebPage.class, conf);
- WebPageDataCreator.createWebPageData(store);
-
- Query<String, WebPage> query = store.newQuery();
- Result<String, WebPage> result = query.execute();
-
- int tested = 0;
- while(result.next()) {
- WebPage page = result.get();
- testClone(page);
- tested++;
- }
- assertEquals(WebPageDataCreator.URLS.length, tested);
- }
-}
http://git-wip-us.apache.org/repos/asf/gora/blob/136fc595/gora-core/src/test/java/org/apache/gora/examples/TestWebPageDataCreator.java
----------------------------------------------------------------------
diff --git a/gora-core/src/test/java/org/apache/gora/examples/TestWebPageDataCreator.java b/gora-core/src/test/java/org/apache/gora/examples/TestWebPageDataCreator.java
new file mode 100644
index 0000000..fde39ab
--- /dev/null
+++ b/gora-core/src/test/java/org/apache/gora/examples/TestWebPageDataCreator.java
@@ -0,0 +1,19 @@
+package org.apache.gora.examples;
+
+import java.io.IOException;
+
+import org.apache.gora.examples.generated.WebPage;
+import org.apache.gora.memory.store.MemStore;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.junit.runners.JUnit4;
+
+@RunWith(JUnit4.class)
+public class TestWebPageDataCreator {
+
+ @Test public void testCreatesData() throws IOException{
+ MemStore<String, WebPage> dataStore = new MemStore<String, WebPage>();
+ WebPageDataCreator.createWebPageData(dataStore);
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/gora/blob/136fc595/gora-core/src/test/java/org/apache/gora/filter/TestMapFieldValueFilter.java
----------------------------------------------------------------------
diff --git a/gora-core/src/test/java/org/apache/gora/filter/TestMapFieldValueFilter.java b/gora-core/src/test/java/org/apache/gora/filter/TestMapFieldValueFilter.java
index 553c788..b53207a 100644
--- a/gora-core/src/test/java/org/apache/gora/filter/TestMapFieldValueFilter.java
+++ b/gora-core/src/test/java/org/apache/gora/filter/TestMapFieldValueFilter.java
@@ -32,6 +32,7 @@ public class TestMapFieldValueFilter {
@Test
public void testSerialization() throws IOException {
MapFieldValueFilter<String, WebPage> filter = new MapFieldValueFilter<String, WebPage>();
+ //set filter field name as metadata
filter.setFieldName(WebPage.Field.METADATA.toString());
filter.setMapKey(new Utf8("fetchTime"));
filter.setFilterOp(FilterOp.EQUALS);
@@ -50,16 +51,17 @@ public class TestMapFieldValueFilter {
@Test
public void testFilterBasics() {
MapFieldValueFilter<String, WebPage> filter = new MapFieldValueFilter<String, WebPage>();
+ //set filter field name as outlinks
filter.setFieldName(WebPage.Field.OUTLINKS.toString());
filter.setMapKey(new Utf8("example"));
filter.setFilterOp(FilterOp.EQUALS);
filter.setFilterIfMissing(true);
filter.getOperands().add(new Utf8("http://example.org"));
- WebPage page = new WebPage();
- page.putToOutlinks(new Utf8("example"), new Utf8("http://example.org"));
+ WebPage page = WebPage.newBuilder().build();
+ page.getOutlinks().put(new Utf8("example"), new Utf8("http://example.org"));
assertFalse(filter.filter("irrelevant", page));
- page.putToOutlinks(new Utf8("example"), new Utf8("http://example2.com"));
+ page.getOutlinks().put(new Utf8("example"), new Utf8("http://example2.com"));
assertTrue(filter.filter("irrelevant", page));
page = new WebPage();
assertTrue(filter.filter("irrelevant", page));
@@ -71,17 +73,18 @@ public class TestMapFieldValueFilter {
@Test
public void testFilterEntryInMap() {
MapFieldValueFilter<String, WebPage> filter = new MapFieldValueFilter<String, WebPage>();
+ //set filter field name as outlinks
filter.setFieldName(WebPage.Field.OUTLINKS.toString());
filter.setMapKey(new Utf8("foobar.whatever"));
filter.setFilterOp(FilterOp.EQUALS);
filter.setFilterIfMissing(true);
filter.getOperands().add(new Utf8("Click here for foobar!"));
- WebPage page = new WebPage();
+ WebPage page = WebPage.newBuilder().build();
assertTrue(filter.filter("irrelevant", page));
- page.putToOutlinks(new Utf8("foobar.whatever"), new Utf8("Mismatch!"));
+ page.getOutlinks().put(new Utf8("foobar.whatever"), new Utf8("Mismatch!"));
assertTrue(filter.filter("irrelevant", page));
- page.putToOutlinks(new Utf8("foobar.whatever"), new Utf8("Click here for foobar!"));
+ page.getOutlinks().put(new Utf8("foobar.whatever"), new Utf8("Click here for foobar!"));
assertFalse(filter.filter("irrelevant", page));
}
http://git-wip-us.apache.org/repos/asf/gora/blob/136fc595/gora-core/src/test/java/org/apache/gora/filter/TestSingleFieldValueFilter.java
----------------------------------------------------------------------
diff --git a/gora-core/src/test/java/org/apache/gora/filter/TestSingleFieldValueFilter.java b/gora-core/src/test/java/org/apache/gora/filter/TestSingleFieldValueFilter.java
index a03ba93..6390b0d 100644
--- a/gora-core/src/test/java/org/apache/gora/filter/TestSingleFieldValueFilter.java
+++ b/gora-core/src/test/java/org/apache/gora/filter/TestSingleFieldValueFilter.java
@@ -33,6 +33,7 @@ public class TestSingleFieldValueFilter {
@Test
public void testSerialization() throws IOException {
SingleFieldValueFilter<String, WebPage> filter = new SingleFieldValueFilter<String, WebPage>();
+ //set filter field to url
filter.setFieldName(WebPage.Field.URL.toString());
filter.setFilterOp(FilterOp.EQUALS);
filter.setFilterIfMissing(true);
@@ -49,12 +50,13 @@ public class TestSingleFieldValueFilter {
@Test
public void testFilterBasics() {
SingleFieldValueFilter<String, WebPage> filter = new SingleFieldValueFilter<String, WebPage>();
+ //set filter field to url
filter.setFieldName(WebPage.Field.URL.toString());
filter.setFilterOp(FilterOp.EQUALS);
filter.setFilterIfMissing(true);
filter.getOperands().add(new Utf8("example.org"));
- WebPage page = new WebPage();
+ WebPage page = WebPage.newBuilder().build();
page.setUrl(new Utf8("example.org"));
assertFalse(filter.filter("irrelevant", page));
page.setUrl(new Utf8("mismatch.whatever"));
@@ -70,12 +72,13 @@ public class TestSingleFieldValueFilter {
@Test
public void testFilterInequals() {
SingleFieldValueFilter<String, WebPage> filter = new SingleFieldValueFilter<String, WebPage>();
+ //set filter field to url
filter.setFieldName(WebPage.Field.URL.toString());
filter.setFilterOp(FilterOp.NOT_EQUALS);
filter.setFilterIfMissing(true);
filter.getOperands().add(new Utf8("example.org"));
- WebPage page = new WebPage();
+ WebPage page = WebPage.newBuilder().build();
page.setUrl(new Utf8("example.org"));
assertTrue(filter.filter("irrelevant", page));
http://git-wip-us.apache.org/repos/asf/gora/blob/136fc595/gora-core/src/test/java/org/apache/gora/mapreduce/DataStoreMapReduceTestBase.java
----------------------------------------------------------------------
diff --git a/gora-core/src/test/java/org/apache/gora/mapreduce/DataStoreMapReduceTestBase.java b/gora-core/src/test/java/org/apache/gora/mapreduce/DataStoreMapReduceTestBase.java
index 28acdcd..c3778af 100644
--- a/gora-core/src/test/java/org/apache/gora/mapreduce/DataStoreMapReduceTestBase.java
+++ b/gora-core/src/test/java/org/apache/gora/mapreduce/DataStoreMapReduceTestBase.java
@@ -37,7 +37,6 @@ import org.slf4j.LoggerFactory;
* class, which actually only uses {@link MapReduceTestUtils} methods to
* run the tests.
*/
-@SuppressWarnings("deprecation")
public abstract class DataStoreMapReduceTestBase extends HadoopTestCase {
public static final Logger LOG = LoggerFactory.getLogger(DataStoreMapReduceTestBase.class);
@@ -71,8 +70,8 @@ public abstract class DataStoreMapReduceTestBase extends HadoopTestCase {
@Override
public void tearDown() throws Exception {
LOG.info("Tearing down Hadoop Test Case...");
- super.tearDown();
webPageStore.close();
+ super.tearDown();
}
protected abstract DataStore<String, WebPage> createWebPageDataStore()
http://git-wip-us.apache.org/repos/asf/gora/blob/136fc595/gora-core/src/test/java/org/apache/gora/mapreduce/MapReduceTestUtils.java
----------------------------------------------------------------------
diff --git a/gora-core/src/test/java/org/apache/gora/mapreduce/MapReduceTestUtils.java b/gora-core/src/test/java/org/apache/gora/mapreduce/MapReduceTestUtils.java
index f837403..12ccea4 100644
--- a/gora-core/src/test/java/org/apache/gora/mapreduce/MapReduceTestUtils.java
+++ b/gora-core/src/test/java/org/apache/gora/mapreduce/MapReduceTestUtils.java
@@ -19,10 +19,10 @@
package org.apache.gora.mapreduce;
import java.util.HashMap;
+import java.util.List;
import java.util.Map;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
+import org.apache.avro.Schema.Field;
import org.apache.gora.examples.WebPageDataCreator;
import org.apache.gora.examples.generated.TokenDatum;
import org.apache.gora.examples.generated.WebPage;
@@ -32,14 +32,20 @@ import org.apache.gora.query.Query;
import org.apache.gora.store.DataStore;
import org.apache.gora.store.impl.DataStoreBase;
import org.apache.hadoop.conf.Configuration;
+
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertNotNull;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
public class MapReduceTestUtils {
private static final Logger log = LoggerFactory.getLogger(MapReduceTestUtils.class);
- /** Tests by running the {@link QueryCounter} mapreduce job */
+ /**
+ * Tests by running the {@link org.apache.gora.examples.mapreduce.QueryCounter}
+ * mapreduce job
+ */
public static void testCountQuery(DataStore<String, WebPage> dataStore, Configuration conf)
throws Exception {
@@ -50,7 +56,12 @@ public class MapReduceTestUtils {
QueryCounter<String,WebPage> counter = new QueryCounter<String,WebPage>(conf);
Query<String,WebPage> query = dataStore.newQuery();
- query.setFields(WebPage._ALL_FIELDS);
+ List<Field> fields = WebPage.SCHEMA$.getFields();
+ String[] fieldNames = new String[fields.size() - 1];
+ for(int i = 0; i< fieldNames.length; i++){
+ fieldNames[i] = fields.get(i+1).name();
+ }
+ query.setFields(fieldNames);
dataStore.close();
@@ -65,7 +76,7 @@ public class MapReduceTestUtils {
public static void testWordCount(Configuration conf, DataStore<String,WebPage> inStore, DataStore<String,
TokenDatum> outStore) throws Exception {
- //Datastore now has to be a Hadoop based datastore
+ //Datastore now has to be a Hadoop based datastore
((DataStoreBase<String,WebPage>)inStore).setConf(conf);
((DataStoreBase<String,TokenDatum>)outStore).setConf(conf);
@@ -97,6 +108,6 @@ public class MapReduceTestUtils {
String token, int count) throws Exception {
TokenDatum datum = outStore.get(token, null);
assertNotNull("token:" + token + " cannot be found in datastore", datum);
- assertEquals("count for token:" + token + " is wrong", count, datum.getCount());
+ assertEquals("count for token:" + token + " is wrong", count, datum.getCount().intValue());
}
}
http://git-wip-us.apache.org/repos/asf/gora/blob/136fc595/gora-core/src/test/java/org/apache/gora/mapreduce/TestGoraInputFormat.java
----------------------------------------------------------------------
diff --git a/gora-core/src/test/java/org/apache/gora/mapreduce/TestGoraInputFormat.java b/gora-core/src/test/java/org/apache/gora/mapreduce/TestGoraInputFormat.java
index eba0400..e9a7b22 100644
--- a/gora-core/src/test/java/org/apache/gora/mapreduce/TestGoraInputFormat.java
+++ b/gora-core/src/test/java/org/apache/gora/mapreduce/TestGoraInputFormat.java
@@ -22,6 +22,7 @@ import java.io.IOException;
import java.util.Arrays;
import java.util.List;
+import org.apache.avro.Schema.Field;
import org.apache.gora.examples.generated.Employee;
import org.apache.gora.mock.persistency.MockPersistent;
import org.apache.gora.mock.query.MockQuery;
@@ -41,7 +42,8 @@ public class TestGoraInputFormat {
MockDataStore store = MockDataStore.get();
MockQuery query = store.newQuery();
- query.setFields(Employee._ALL_FIELDS);
+
+ query.setFields(getEmployeeFieldNames());
GoraInputFormat.setInput(job, query, false);
GoraInputFormat<String, MockPersistent> inputFormat
@@ -52,6 +54,15 @@ public class TestGoraInputFormat {
return inputFormat.getSplits(job);
}
+ /**
+ * First, asserts that the attempt to obtain splits results in
+ * greater than 0 splits which can be used for computation.
+ * We then check that the partition query (obtained by using the
+ * splits) has the same fields as we would expect by directly
+ * accessing the fields of an Employee object.
+ * @throws IOException
+ * @throws InterruptedException
+ */
@Test
@SuppressWarnings("rawtypes")
public void testGetSplits() throws IOException, InterruptedException {
@@ -61,7 +72,16 @@ public class TestGoraInputFormat {
InputSplit split = splits.get(0);
PartitionQuery query = ((GoraInputSplit)split).getQuery();
- assertTrue(Arrays.equals(Employee._ALL_FIELDS, query.getFields()));
+ assertTrue(Arrays.equals(getEmployeeFieldNames(), query.getFields()));
+ }
+
+ private static String[] getEmployeeFieldNames(){
+ List<Field> fields = Employee.SCHEMA$.getFields();
+ String[] fieldNames = new String[fields.size()];
+ for(int i = 0; i< fieldNames.length; i++){
+ fieldNames[i] = fields.get(i).name();
+ }
+ return fieldNames;
}
}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/gora/blob/136fc595/gora-core/src/test/java/org/apache/gora/mapreduce/TestPersistentSerialization.java
----------------------------------------------------------------------
diff --git a/gora-core/src/test/java/org/apache/gora/mapreduce/TestPersistentSerialization.java b/gora-core/src/test/java/org/apache/gora/mapreduce/TestPersistentSerialization.java
index 47841a1..7048c0c 100644
--- a/gora-core/src/test/java/org/apache/gora/mapreduce/TestPersistentSerialization.java
+++ b/gora-core/src/test/java/org/apache/gora/mapreduce/TestPersistentSerialization.java
@@ -18,6 +18,9 @@
package org.apache.gora.mapreduce;
+import java.util.ArrayList;
+import java.util.HashMap;
+
import org.apache.avro.util.Utf8;
import org.apache.gora.examples.WebPageDataCreator;
import org.apache.gora.examples.generated.Employee;
@@ -31,11 +34,19 @@ import org.apache.hadoop.conf.Configuration;
import org.junit.Test;
import static org.junit.Assert.assertEquals;
-/** Test class for {@link PersistentSerialization}, {@link PersistentSerializer}
- * and {@link PersistentDeserializer}
+/**
+ * Test class for {@link PersistentSerialization}, {@link PersistentSerializer}
+ * and {@link PersistentDeserializer}
*/
public class TestPersistentSerialization {
+ /**
+ * Creates an Employee object in-memory setting several fields to dirty.
+ * Asserts that it can be serialized and
+ * deserialzed without loosing data. We do this by asserting
+ * what we get 'before' and 'after' (de)serialization processes.
+ * @throws Exception
+ */
@SuppressWarnings("unchecked")
@Test
public void testSerdeEmployee() throws Exception {
@@ -48,23 +59,45 @@ public class TestPersistentSerialization {
TestIOUtils.testSerializeDeserialize(employee);
}
+ /**
+ * Creates an Employee object but only sets one field as dirty.
+ * We then do (de)serialization and check 'before' and 'after'
+ * states.
+ * @throws Exception
+ */
@Test
public void testSerdeEmployeeOneField() throws Exception {
- Employee employee = new Employee();
+ Employee employee = Employee.newBuilder().build();
employee.setSsn(new Utf8("11111"));
TestIOUtils.testSerializeDeserialize(employee);
}
+ /**
+ * Creates an Employee object setting only two fields as dirty.
+ * We then do (de)serialization and check 'before' and 'after'
+ * states.
+ * @throws Exception
+ */
@Test
public void testSerdeEmployeeTwoFields() throws Exception {
- Employee employee = new Employee();
+ Employee employee = Employee.newBuilder().build();
employee.setSsn(new Utf8("11111"));
employee.setSalary(100);
TestIOUtils.testSerializeDeserialize(employee);
}
+ /**
+ * Creates an WebPage object in-memory setting several fields to dirty.
+ * Run a query over the persistent data.
+ * Asserts that the results can be serialized and
+ * deserialzed without loosing data. We do this by asserting
+ * what we get 'before' and 'after' (de)serialization processes.
+ * Also simple assertion for equal number of URL's in WebPage
+ * and results.
+ * @throws Exception
+ */
@SuppressWarnings("unchecked")
@Test
public void testSerdeWebPage() throws Exception {
@@ -75,8 +108,8 @@ public class TestPersistentSerialization {
Result<String, WebPage> result = store.newQuery().execute();
- int i=0;
- while(result.next()) {
+ int i = 0;
+ while (result.next()) {
WebPage page = result.get();
TestIOUtils.testSerializeDeserialize(page);
i++;
@@ -84,19 +117,26 @@ public class TestPersistentSerialization {
assertEquals(WebPageDataCreator.URLS.length, i);
}
+ /**
+ * Creates multiple WebPage objects setting several fields to dirty.
+ * Asserts that the data can be serialized and
+ * deserialzed without loosing data. We do this by asserting
+ * what we get 'before' and 'after' (de)serialization processes.
+ * @throws Exception
+ */
@Test
public void testSerdeMultipleWebPages() throws Exception {
- WebPage page1 = new WebPage();
- WebPage page2 = new WebPage();
- WebPage page3 = new WebPage();
+ WebPage page1 = WebPage.newBuilder().build();
+ WebPage page2 = WebPage.newBuilder().build();
+ WebPage page3 = WebPage.newBuilder().build();
page1.setUrl(new Utf8("foo"));
page2.setUrl(new Utf8("baz"));
page3.setUrl(new Utf8("bar"));
-
- page1.addToParsedContent(new Utf8("coo"));
-
- page2.putToOutlinks(new Utf8("a"), new Utf8("b"));
+ page1.setParsedContent(new ArrayList<CharSequence>());
+ page1.getParsedContent().add(new Utf8("coo"));
+ page2.setOutlinks(new HashMap<CharSequence, CharSequence>());
+ page2.getOutlinks().put(new Utf8("a"), new Utf8("b"));
TestIOUtils.testSerializeDeserialize(page1, page2, page3);
}
http://git-wip-us.apache.org/repos/asf/gora/blob/136fc595/gora-core/src/test/java/org/apache/gora/mock/persistency/MockPersistent.java
----------------------------------------------------------------------
diff --git a/gora-core/src/test/java/org/apache/gora/mock/persistency/MockPersistent.java b/gora-core/src/test/java/org/apache/gora/mock/persistency/MockPersistent.java
index 1cf0bfb..b33629b 100644
--- a/gora-core/src/test/java/org/apache/gora/mock/persistency/MockPersistent.java
+++ b/gora-core/src/test/java/org/apache/gora/mock/persistency/MockPersistent.java
@@ -20,7 +20,7 @@ package org.apache.gora.mock.persistency;
import org.apache.avro.Schema;
import org.apache.gora.persistency.Persistent;
-import org.apache.gora.persistency.StateManager;
+import org.apache.gora.persistency.Tombstone;
import org.apache.gora.persistency.impl.PersistentBase;
public class MockPersistent extends PersistentBase {
@@ -36,9 +36,6 @@ public class MockPersistent extends PersistentBase {
public MockPersistent() {
}
- public MockPersistent(StateManager stateManager) {
- super(stateManager);
- }
@Override
public Object get(int field) {
@@ -79,22 +76,13 @@ public class MockPersistent extends PersistentBase {
}
@Override
- public String getField(int index) {
- return null;
- }
-
- @Override
- public int getFieldIndex(String field) {
- return 0;
+ public Tombstone getTombstone() {
+ return new Tombstone(){};
}
@Override
- public String[] getFields() {
- return null;
+ public Persistent newInstance() {
+ return new MockPersistent();
}
- @Override
- public Persistent newInstance(StateManager stateManager) {
- return new MockPersistent(stateManager);
- }
}
http://git-wip-us.apache.org/repos/asf/gora/blob/136fc595/gora-core/src/test/java/org/apache/gora/persistency/TestListGenericArray.java
----------------------------------------------------------------------
diff --git a/gora-core/src/test/java/org/apache/gora/persistency/TestListGenericArray.java b/gora-core/src/test/java/org/apache/gora/persistency/TestListGenericArray.java
index a049a51..e69de29 100644
--- a/gora-core/src/test/java/org/apache/gora/persistency/TestListGenericArray.java
+++ b/gora-core/src/test/java/org/apache/gora/persistency/TestListGenericArray.java
@@ -1,57 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.gora.persistency;
-
-import org.apache.avro.Schema;
-import org.apache.avro.util.Utf8;
-import static org.junit.Assert.assertFalse;
-import org.junit.Test;
-
-/**
- * Testcase for ListGenericArray class
- */
-public class TestListGenericArray {
-
- @Test
- public void testHashCode() {
- ListGenericArray array = new ListGenericArray(Schema.create(Schema.Type.STRING));
- boolean stackOverflowError = false;
- array.add(new Utf8("array test"));
- try {
- int hashCode = array.hashCode();
- }
- catch (StackOverflowError e) {
- stackOverflowError = true;
- }
- assertFalse(stackOverflowError);
- }
-
- @Test
- public void testCompareTo() {
- ListGenericArray array = new ListGenericArray(Schema.create(Schema.Type.STRING));
- boolean stackOverflowError = false;
- array.add(new Utf8("array comparison test"));
- try {
- int compareTo = array.compareTo(array);
- } catch (StackOverflowError e) {
- stackOverflowError = true;
- }
- assertFalse(stackOverflowError);
- }
-}
http://git-wip-us.apache.org/repos/asf/gora/blob/136fc595/gora-core/src/test/java/org/apache/gora/persistency/impl/TestPersistentBase.java
----------------------------------------------------------------------
diff --git a/gora-core/src/test/java/org/apache/gora/persistency/impl/TestPersistentBase.java b/gora-core/src/test/java/org/apache/gora/persistency/impl/TestPersistentBase.java
index 806743a..ab730be 100644
--- a/gora-core/src/test/java/org/apache/gora/persistency/impl/TestPersistentBase.java
+++ b/gora-core/src/test/java/org/apache/gora/persistency/impl/TestPersistentBase.java
@@ -20,7 +20,10 @@ package org.apache.gora.persistency.impl;
import java.io.IOException;
import java.nio.ByteBuffer;
-
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import org.apache.avro.Schema.Field;
import org.apache.avro.util.Utf8;
import org.apache.gora.examples.generated.Employee;
import org.apache.gora.examples.generated.WebPage;
@@ -28,10 +31,8 @@ import org.apache.gora.memory.store.MemStore;
import org.apache.gora.store.DataStoreFactory;
import org.apache.gora.store.DataStoreTestUtil;
import org.apache.hadoop.conf.Configuration;
-import static org.junit.Assert.assertArrayEquals;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertNull;
-
import org.junit.Test;
/**
@@ -39,53 +40,83 @@ import org.junit.Test;
*/
public class TestPersistentBase {
+ /**
+ * Assert that the list of fields from the WebPage Schema
+ * are as we expect. This is done by creating and accessing
+ * a WebPage object, then comparing the results against
+ * static fields of the WebPage.SCHEMA$.
+ */
@Test
public void testGetFields() {
- WebPage page = new WebPage();
- String[] fields = page.getFields();
- assertArrayEquals(WebPage._ALL_FIELDS, fields);
+ WebPage page = WebPage.newBuilder().build();
+ List<Field> fields = page.getSchema().getFields();
+ assertEquals(WebPage.SCHEMA$.getFields(), fields);
}
+ /**
+ * Assert that individual field values are as we would
+ * expect from directly accessing WebPage.SCHEMA$ values.
+ */
@Test
public void testGetField() {
- WebPage page = new WebPage();
- for(int i=0; i<WebPage._ALL_FIELDS.length; i++) {
- String field = page.getField(i);
- assertEquals(WebPage._ALL_FIELDS[i], field);
+ WebPage page = WebPage.newBuilder().build();
+ for(int i=0; i<WebPage.SCHEMA$.getFields().toArray().length; i++) {
+ Field field = page.getSchema().getFields().get(i);
+ assertEquals(WebPage.SCHEMA$.getFields().get(i), field);
}
}
+ /**
+ * Assert that field positions as found within the SCHEMA array
+ * are as we would expect by accessing them directly.
+ */
@Test
public void testGetFieldIndex() {
- WebPage page = new WebPage();
- for(int i=0; i<WebPage._ALL_FIELDS.length; i++) {
- int index = page.getFieldIndex(WebPage._ALL_FIELDS[i]);
+ WebPage page = WebPage.newBuilder().build();
+ for(int i=0; i<WebPage.SCHEMA$.getFields().toArray().length; i++) {
+ int index = page.getSchema().getFields().get(i).pos();
assertEquals(i, index);
}
}
+ /**
+ * Assert that field positions as found within the SCHEMA array
+ * are as we would expect by accessing them directly.
+ * This tests for both WebPage and Employee data beans.
+ */
@Test
public void testFieldsWithTwoClasses() {
- WebPage page = new WebPage();
- for(int i=0; i<WebPage._ALL_FIELDS.length; i++) {
- int index = page.getFieldIndex(WebPage._ALL_FIELDS[i]);
+ WebPage page = WebPage.newBuilder().build();
+ for(int i=0; i<WebPage.SCHEMA$.getFields().toArray().length; i++) {
+ int index = page.getSchema().getFields().get(i).pos();
assertEquals(i, index);
}
- Employee employee = new Employee();
- for(int i=0; i<Employee._ALL_FIELDS.length; i++) {
- int index = employee.getFieldIndex(Employee._ALL_FIELDS[i]);
+ Employee employee = Employee.newBuilder().build();
+ for(int i=0; i<Employee.SCHEMA$.getFields().toArray().length; i++) {
+ int index = employee.getSchema().getFields().get(i).pos();
assertEquals(i, index);
}
}
+ /**
+ * First we create a new WebPage object, to which we add some
+ * field values. This makes the fields dirty as we have not
+ * flushed them to the datastore. We then clear the dirty
+ * fields and assert that the values DO NOT exist for the
+ * field we previously made dirty.
+ * We then set new values for fields, consequently making them
+ * dirty, before testing the clearing of an entirely new object
+ * has all fields as null as they should be clean.
+ */
@Test
public void testClear() {
//test clear all fields
- WebPage page = new WebPage();
+ WebPage page = WebPage.newBuilder().build();
+
page.setUrl(new Utf8("http://foo.com"));
- page.addToParsedContent(new Utf8("foo"));
- page.putToOutlinks(new Utf8("foo"), new Utf8("bar"));
+ page.getParsedContent().add(new Utf8("foo"));
+ page.getOutlinks().put(new Utf8("foo"), new Utf8("bar"));
page.setContent(ByteBuffer.wrap("foo baz bar".getBytes()));
page.clear();
@@ -97,19 +128,21 @@ public class TestPersistentBase {
//set fields again
page.setUrl(new Utf8("http://bar.com"));
- page.addToParsedContent(new Utf8("bar"));
- page.putToOutlinks(new Utf8("bar"), new Utf8("baz"));
+ page.getParsedContent().add(new Utf8("bar"));
+ page.getOutlinks().put(new Utf8("bar"), new Utf8("baz"));
page.setContent(ByteBuffer.wrap("foo baz bar barbaz".getBytes()));
//test clear new object
- page = new WebPage();
+ page = WebPage.newBuilder().build();
page.clear();
-
- //test primitive fields
- Employee employee = new Employee();
- employee.clear();
}
+ /**
+ * Tests and asserts that an in-memory representation of the
+ * Employee object is Equal to a clone of the same object.
+ * @throws IOException
+ * @throws Exception
+ */
@Test
public void testClone() throws IOException, Exception {
//more tests for clone are in TestPersistentDatumReader
@@ -119,6 +152,6 @@ public class TestPersistentBase {
Employee employee = DataStoreTestUtil.createEmployee(store);
- assertEquals(employee, employee.clone());
+ assertEquals(employee, Employee.newBuilder(employee).build());
}
}
http://git-wip-us.apache.org/repos/asf/gora/blob/136fc595/gora-core/src/test/java/org/apache/gora/persistency/impl/TestStateManagerImpl.java
----------------------------------------------------------------------
diff --git a/gora-core/src/test/java/org/apache/gora/persistency/impl/TestStateManagerImpl.java b/gora-core/src/test/java/org/apache/gora/persistency/impl/TestStateManagerImpl.java
index e95673a..e69de29 100644
--- a/gora-core/src/test/java/org/apache/gora/persistency/impl/TestStateManagerImpl.java
+++ b/gora-core/src/test/java/org/apache/gora/persistency/impl/TestStateManagerImpl.java
@@ -1,115 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.gora.persistency.impl;
-
-import java.io.IOException;
-
-import org.apache.avro.util.Utf8;
-import org.apache.gora.examples.generated.Employee;
-import org.apache.gora.mock.persistency.MockPersistent;
-import org.apache.gora.persistency.impl.StateManagerImpl;
-import org.junit.Before;
-import org.junit.Test;
-import static junit.framework.Assert.assertFalse;
-import static junit.framework.Assert.assertTrue;
-
-/**
- * Test case for {@link StateManagerImpl}
- */
-public class TestStateManagerImpl {
-
- private StateManagerImpl stateManager;
- private MockPersistent persistent;
-
- @Before
- public void setUp() {
- this.stateManager = new StateManagerImpl();
- this.persistent = new MockPersistent(stateManager);
- }
-
- @Test
- public void testDirty() {
- assertFalse(stateManager.isDirty(persistent));
- stateManager.setDirty(persistent);
- assertTrue(stateManager.isDirty(persistent));
- }
-
- @Test
- public void testDirty2() {
- assertFalse(stateManager.isDirty(persistent, 0));
- assertFalse(stateManager.isDirty(persistent, 1));
- stateManager.setDirty(persistent, 0);
- assertTrue(stateManager.isDirty(persistent, 0));
- assertFalse(stateManager.isDirty(persistent, 1));
- }
-
- @Test
- public void testClearDirty() {
- assertFalse(stateManager.isDirty(persistent));
- stateManager.setDirty(persistent, 0);
- stateManager.clearDirty(persistent);
- assertFalse(this.stateManager.isDirty(persistent));
- }
-
- @Test
- public void testReadable() throws IOException {
- assertFalse(stateManager.isReadable(persistent, 0));
- assertFalse(stateManager.isReadable(persistent, 1));
- stateManager.setReadable(persistent, 0);
- assertTrue(stateManager.isReadable(persistent, 0));
- assertFalse(stateManager.isReadable(persistent, 1));
- }
-
- @Test
- public void testReadable2() {
- stateManager = new StateManagerImpl();
- Employee employee = new Employee(stateManager);
- assertFalse(stateManager.isReadable(employee, 0));
- assertFalse(stateManager.isReadable(employee, 1));
- employee.setName(new Utf8("foo"));
- assertTrue(stateManager.isReadable(employee, 0));
- assertFalse(stateManager.isReadable(employee, 1));
- }
-
- @Test
- public void testClearReadable() {
- stateManager.setReadable(persistent, 0);
- stateManager.clearReadable(persistent);
- assertFalse(stateManager.isReadable(persistent, 0));
- }
-
- @Test
- public void testIsNew() {
- //newly created objects should be new
- assertTrue(persistent.isNew());
- }
-
- @Test
- public void testNew() {
- stateManager.setNew(persistent);
- assertTrue(persistent.isNew());
- }
-
- @Test
- public void testClearNew() {
- stateManager.clearNew(persistent);
- assertFalse(persistent.isNew());
- }
-
-}
http://git-wip-us.apache.org/repos/asf/gora/blob/136fc595/gora-core/src/test/java/org/apache/gora/store/DataStoreTestBase.java
----------------------------------------------------------------------
diff --git a/gora-core/src/test/java/org/apache/gora/store/DataStoreTestBase.java b/gora-core/src/test/java/org/apache/gora/store/DataStoreTestBase.java
index 95cd596..e890aa3 100644
--- a/gora-core/src/test/java/org/apache/gora/store/DataStoreTestBase.java
+++ b/gora-core/src/test/java/org/apache/gora/store/DataStoreTestBase.java
@@ -19,22 +19,16 @@
package org.apache.gora.store;
import java.io.IOException;
-import java.nio.ByteBuffer;
-
-import org.apache.avro.util.Utf8;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.apache.gora.GoraTestDriver;
import org.apache.gora.examples.generated.Employee;
-import org.apache.gora.examples.generated.Metadata;
import org.apache.gora.examples.generated.WebPage;
import org.junit.After;
import org.junit.AfterClass;
import org.junit.Before;
import org.junit.BeforeClass;
import org.junit.Test;
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertNotNull;
/**
* A base class for {@link DataStore} tests. This is just a convenience
@@ -216,7 +210,11 @@ public abstract class DataStoreTestBase {
public void testUpdate() throws IOException, Exception {
log.info("test method: testUpdate");
DataStoreTestUtil.testUpdateEmployee(employeeStore);
- DataStoreTestUtil.testUpdateWebPage(webPageStore);
+ DataStoreTestUtil.testUpdateWebPagePutToArray(webPageStore);
+ DataStoreTestUtil.testUpdateWebPagePutToNotNullableMap(webPageStore);
+ DataStoreTestUtil.testUpdateWebPagePutToNullableMap(webPageStore);
+ DataStoreTestUtil.testUpdateWebPageRemoveMapEntry(webPageStore);
+ DataStoreTestUtil.testUpdateWebPageRemoveField(webPageStore);
}
public void testEmptyUpdate() throws IOException, Exception {
@@ -243,7 +241,7 @@ public abstract class DataStoreTestBase {
@Test
/**
- * Tests put and get a record with a double nested recursive record
+ * Tests put and get a record with a double nested recursive record
* Employee with a boss (nested).
* @throws IOException
* @throws Exception
@@ -255,8 +253,9 @@ public abstract class DataStoreTestBase {
@Test
/**
- * Tests put and get a record with a nested record (not recursive)
- * The webpage of an Employee
+ * Tests put and get of an {@link org.apache.gora.examples.generated.Employee}
+ * record with a nested {@link org.apache.gora.examples.generated.WegPage} record (not recursive)
+ * the webpage of an Employee.
* @throws IOException
* @throws Exception
*/
http://git-wip-us.apache.org/repos/asf/gora/blob/136fc595/gora-core/src/test/java/org/apache/gora/store/DataStoreTestUtil.java
----------------------------------------------------------------------
diff --git a/gora-core/src/test/java/org/apache/gora/store/DataStoreTestUtil.java b/gora-core/src/test/java/org/apache/gora/store/DataStoreTestUtil.java
index d093b24..7a39d17 100644
--- a/gora-core/src/test/java/org/apache/gora/store/DataStoreTestUtil.java
+++ b/gora-core/src/test/java/org/apache/gora/store/DataStoreTestUtil.java
@@ -42,7 +42,7 @@ import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertTrue;
import static org.junit.Assert.assertNull;
-import org.apache.avro.generic.GenericArray;
+import org.apache.avro.Schema.Field;
import org.apache.avro.util.Utf8;
import org.apache.gora.examples.WebPageDataCreator;
import org.apache.gora.examples.generated.Employee;
@@ -53,9 +53,9 @@ import org.apache.gora.persistency.impl.BeanFactoryImpl;
import org.apache.gora.query.PartitionQuery;
import org.apache.gora.query.Query;
import org.apache.gora.query.Result;
+import org.apache.gora.util.AvroUtils;
import org.apache.gora.util.ByteUtils;
import org.apache.gora.util.StringUtils;
-import org.junit.Test;
/**
* Test utilities for DataStores. This utility class provides everything
@@ -89,7 +89,7 @@ public class DataStoreTestUtil {
public static <K> Employee createEmployee(
DataStore<K, Employee> dataStore) throws IOException, Exception {
- Employee employee = dataStore.newPersistent();
+ Employee employee = Employee.newBuilder().build();
employee.setName(new Utf8("Random Joe"));
employee.setDateOfBirth( System.currentTimeMillis() - 20L * YEAR_IN_MS );
employee.setSalary(100000);
@@ -97,12 +97,22 @@ public class DataStoreTestUtil {
return employee;
}
- public static <K> Employee createBoss(
- DataStore<K, Employee> dataStore) throws IOException, Exception {
+ private static <K> WebPage createWebPage(DataStore<K, Employee> dataStore) {
+ WebPage webpage = WebPage.newBuilder().build();
+ webpage.setUrl(new Utf8("url.."));
+ webpage.setContent(ByteBuffer.wrap("test content".getBytes()));
+ webpage.setParsedContent(new ArrayList<CharSequence>());
+ Metadata metadata = Metadata.newBuilder().build();
+ webpage.setMetadata(metadata);
+ return webpage;
+ }
- Employee employee = dataStore.newPersistent();
+ public static <K> Employee createBoss(DataStore<K, Employee> dataStore)
+ throws IOException, Exception {
+
+ Employee employee = Employee.newBuilder().build();
employee.setName(new Utf8("Random boss"));
- employee.setDateOfBirth( System.currentTimeMillis() - 22L * YEAR_IN_MS );
+ employee.setDateOfBirth(System.currentTimeMillis() - 22L * YEAR_IN_MS);
employee.setSalary(1000000);
employee.setSsn(new Utf8("202020202020"));
return employee;
@@ -159,25 +169,23 @@ public class DataStoreTestUtil {
dataStore.put(ssn, employee);
dataStore.flush();
- Employee after = dataStore.get(ssn, Employee._ALL_FIELDS);
+ Employee after = dataStore.get(ssn, AvroUtils.getSchemaFieldNames(Employee.SCHEMA$));
- assertEquals(employee, after);
+ assertEqualEmployeeObjects(employee, after);
}
-
public static void testGetEmployeeRecursive(DataStore<String, Employee> dataStore)
throws IOException, Exception {
Employee employee = DataStoreTestUtil.createEmployee(dataStore);
Employee boss = DataStoreTestUtil.createBoss(dataStore);
- employee.setBoss(boss) ;
+ employee.setBoss(boss);
String ssn = employee.getSsn().toString();
dataStore.put(ssn, employee);
dataStore.flush();
- Employee after = dataStore.get(ssn, Employee._ALL_FIELDS);
- assertEquals(employee, after);
- assertEquals(boss, after.getBoss()) ;
+ Employee after = dataStore.get(ssn, AvroUtils.getSchemaFieldNames(Employee.SCHEMA$));
+ assertEqualEmployeeObjects(employee, after);
}
public static void testGetEmployeeDoubleRecursive(DataStore<String, Employee> dataStore)
@@ -193,10 +201,8 @@ public class DataStoreTestUtil {
String ssn = employee.getSsn().toString();
dataStore.put(ssn, employee);
dataStore.flush();
- Employee after = dataStore.get(ssn, Employee._ALL_FIELDS);
- assertEquals(employee, after);
- assertEquals(boss, after.getBoss()) ;
- assertEquals(uberBoss, ((Employee)after.getBoss()).getBoss()) ;
+ Employee after = dataStore.get(ssn, AvroUtils.getSchemaFieldNames(Employee.SCHEMA$));
+ assertEqualEmployeeObjects(employee, after);
}
public static void testGetEmployeeNested(DataStore<String, Employee> dataStore)
@@ -207,7 +213,8 @@ public class DataStoreTestUtil {
webpage.setUrl(new Utf8("url..")) ;
webpage.setContent(ByteBuffer.wrap("test content".getBytes())) ;
- Metadata metadata = new BeanFactoryImpl<String,Metadata>(String.class,Metadata.class).newPersistent() ;
+ webpage.setParsedContent(new ArrayList<CharSequence>());
+ Metadata metadata = new BeanFactoryImpl<String,Metadata>(String.class,Metadata.class).newPersistent();
webpage.setMetadata(metadata) ;
employee.setWebpage(webpage) ;
@@ -215,9 +222,9 @@ public class DataStoreTestUtil {
dataStore.put(ssn, employee);
dataStore.flush();
- Employee after = dataStore.get(ssn, Employee._ALL_FIELDS);
- assertEquals(employee, after);
- assertEquals(webpage, after.getWebpage()) ;
+ Employee after = dataStore.get(ssn, AvroUtils.getSchemaFieldNames(Employee.SCHEMA$));
+ assertEqualEmployeeObjects(employee, after);
+ assertEqualWebPageObjects(webpage, after.getWebpage());
}
public static void testGetEmployee3UnionField(DataStore<String, Employee> dataStore)
@@ -225,12 +232,12 @@ public class DataStoreTestUtil {
Employee employee = DataStoreTestUtil.createEmployee(dataStore);
employee.setBoss(new Utf8("Real boss")) ;
-
+
String ssn = employee.getSsn().toString();
dataStore.put(ssn, employee);
dataStore.flush();
- Employee after = dataStore.get(ssn, Employee._ALL_FIELDS);
- assertEquals(employee, after);
+ Employee after = dataStore.get(ssn, AvroUtils.getSchemaFieldNames(Employee.SCHEMA$));
+ assertEqualEmployeeObjects(employee, after);
assertEquals("Real boss", ((Utf8)after.getBoss()).toString()) ;
}
@@ -243,32 +250,139 @@ public class DataStoreTestUtil {
public static void testGetEmployeeWithFields(DataStore<String, Employee> dataStore)
throws IOException, Exception {
Employee employee = DataStoreTestUtil.createEmployee(dataStore);
+ WebPage webpage = createWebPage(dataStore);
+ employee.setWebpage(webpage);
+ Employee boss = createBoss(dataStore);
+ employee.setBoss(boss);
String ssn = employee.getSsn().toString();
dataStore.put(ssn, employee);
dataStore.flush();
- // XXX See GORA-216: special case until later reviewed.
- // Like in K-V stores, if retrieved column does not exists ([webpage] case),
- // get() must return 'null'.
- // We prepare an actual weird synthetic test.
-
- // String[] fields = employee.getFields();
- String[] fields = {"name","dateOfBirth","ssn","salary"} ;
-
+ String[] fields = AvroUtils.getPersistentFieldNames(employee);
for(Set<String> subset : StringUtils.powerset(fields)) {
if(subset.isEmpty())
continue;
Employee after = dataStore.get(ssn, subset.toArray(new String[subset.size()]));
- Employee expected = new Employee();
+ Employee expected = Employee.newBuilder().build();
for(String field:subset) {
- int index = expected.getFieldIndex(field);
+ int index = expected.getSchema().getField(field).pos();
expected.put(index, employee.get(index));
}
- assertEquals(expected, after);
+ assertEqualEmployeeObjects(expected, after);
+ }
+ }
+
+ /**
+ * Simple function which iterates through a before (put) and after (get) object
+ * in an attempt to verify if the same field's and values have been obtained.
+ * Within the original employee object we iterate from 1 instead of 0 due to the
+ * removal of the '__g__' field at position 0 when we put objects into the datastore.
+ * This field is used to identify whether fields within the object, and
+ * consequently the object itself, are/is dirty however this field is not
+ * required when persisting the object.
+ * We explicitly get values from each field as this makes it easier to debug
+ * if tests go wrong.
+ * @param employee
+ * @param after
+ */
+ private static void assertEqualEmployeeObjects(Employee employee, Employee after) {
+ //for (int i = 1; i < employee.SCHEMA$.getFields().size(); i++) {
+ // for (int j = 1; j < after.SCHEMA$.getFields().size(); j++) {
+ // assertEquals(employee.SCHEMA$.getFields().get(i), after.SCHEMA$.getFields().get(j));
+ // }
+ //}
+ //check name field
+ CharSequence beforeName = employee.getName();
+ CharSequence afterName = after.getName();
+ assertEquals(beforeName, afterName);
+ //check dateOfBirth field
+ Long beforeDOB = employee.getDateOfBirth();
+ Long afterDOB = after.getDateOfBirth();
+ assertEquals(beforeDOB, afterDOB);
+ //check ssn field
+ CharSequence beforeSsn = employee.getSsn();
+ CharSequence afterSsn = after.getSsn();
+ assertEquals(beforeSsn, afterSsn);
+ //check salary field
+ Integer beforeSalary = employee.getSalary();
+ Integer afterSalary = after.getSalary();
+ assertEquals(beforeSalary, afterSalary);
+ //check boss field
+ if (employee.getBoss() != null) {
+ if (employee.getBoss() instanceof Utf8) {
+ String beforeBoss = employee.getBoss().toString();
+ String afterBoss = after.getBoss().toString();
+ assertEquals("Boss String field values in UNION should be the same",
+ beforeBoss, afterBoss);
+ } else {
+ Employee beforeBoss = (Employee) employee.getBoss();
+ Employee afterBoss = (Employee) after.getBoss();
+ assertEqualEmployeeObjects(beforeBoss, afterBoss);
+ }
+ }
+ //check webpage field
+ if (employee.getWebpage() != null) {
+ WebPage beforeWebPage = employee.getWebpage();
+ WebPage afterWebPage = after.getWebpage();
+ assertEqualWebPageObjects(beforeWebPage, afterWebPage);
}
}
+ /**
+ * Mimics {@link org.apache.gora.store.DataStoreTestUtil#assertEqualEmployeeObjects(Employee, Employee)}
+ * in that we pick our way through fields within before and after
+ * {@link org.apache.gora.examples.generated.WebPage} objects comparing field values.
+ * @param beforeWebPage
+ * @param afterWebPage
+ */
+ private static void assertEqualWebPageObjects(WebPage beforeWebPage, WebPage afterWebPage) {
+ //check url field
+ CharSequence beforeUrl = beforeWebPage.getUrl();
+ CharSequence afterUrl = afterWebPage.getUrl();
+ assertEquals(beforeUrl, afterUrl);
+ //check content field
+ ByteBuffer beforeContent = beforeWebPage.getContent();
+ ByteBuffer afterContent = afterWebPage.getContent();
+ assertEquals(beforeContent, afterContent);
+ //check parsedContent field
+ List<CharSequence> beforeParsedContent =
+ (List<CharSequence>) beforeWebPage.getParsedContent();
+ List<CharSequence> afterParsedContent =
+ (List<CharSequence>) afterWebPage.getParsedContent();
+ assertEquals(beforeParsedContent, afterParsedContent);
+ //check outlinks field
+ Map<CharSequence, CharSequence> beforeOutlinks =
+ (Map<java.lang.CharSequence,java.lang.CharSequence>) beforeWebPage.getOutlinks();
+ Map<CharSequence, CharSequence> afterOutlinks =
+ (Map<java.lang.CharSequence,java.lang.CharSequence>) afterWebPage.getOutlinks();
+ assertEquals(beforeOutlinks, afterOutlinks);
+ //check metadata field
+ if (beforeWebPage.get(5) != null) {
+ Metadata beforeMetadata = beforeWebPage.getMetadata();
+ Metadata afterMetadata = afterWebPage.getMetadata();
+ assertEqualMetadataObjects(beforeMetadata, afterMetadata);
+ }
+ }
+
+ /**
+ * Mimics {@link org.apache.gora.store.DataStoreTestUtil#assertEqualEmployeeObjects(Employee, Employee)}
+ * in that we pick our way through fields within before and after
+ * {@link org.apache.gora.examples.generated.Metadata} objects comparing field values.
+ * @param beforeMetadata
+ * @param afterMetadata
+ */
+ private static void assertEqualMetadataObjects(Metadata beforeMetadata, Metadata afterMetadata) {
+ //check version field
+ int beforeVersion = beforeMetadata.getVersion();
+ int afterVersion = afterMetadata.getVersion();
+ assertEquals(beforeVersion, afterVersion);
+ //check data field
+ Map<CharSequence, CharSequence> beforeData = beforeMetadata.getData();
+ Map<CharSequence, CharSequence> afterData = afterMetadata.getData();
+ assertEquals(beforeData, afterData);
+ }
+
public static Employee testPutEmployee(DataStore<String, Employee> dataStore)
throws IOException, Exception {
dataStore.createSchema();
@@ -306,6 +420,16 @@ public class DataStoreTestUtil {
assertNull(employee);
}
+ /**
+ * Here we create 5 {@link org.apache.gora.examples.generated.Employee} objects
+ * before populating fields with data and flushing them to the datastore.
+ * We then update the 1st of the {@link org.apache.gora.examples.generated.Employee}'s
+ * with more data and flush this data. Assertions are then made over the updated
+ * {@link org.apache.gora.examples.generated.Employee} object.
+ * @param dataStore
+ * @throws IOException
+ * @throws Exception
+ */
public static void testUpdateEmployee(DataStore<String, Employee> dataStore)
throws IOException, Exception {
dataStore.createSchema();
@@ -313,7 +437,7 @@ public class DataStoreTestUtil {
long now = System.currentTimeMillis();
for (int i = 0; i < 5; i++) {
- Employee employee = dataStore.newPersistent();
+ Employee employee = Employee.newBuilder().build();
employee.setName(new Utf8("John Doe " + i));
employee.setDateOfBirth(now - 20L * YEAR_IN_MS);
employee.setSalary(100000);
@@ -324,9 +448,9 @@ public class DataStoreTestUtil {
dataStore.flush();
for (int i = 0; i < 1; i++) {
- Employee employee = dataStore.newPersistent();
+ Employee employee = Employee.newBuilder().build();
employee.setName(new Utf8("John Doe " + (i + 5)));
- employee.setDateOfBirth(now - 18L * YEAR_IN_MS);
+ employee.setDateOfBirth(now - 18L * YEAR_IN_MS);
employee.setSalary(120000);
employee.setSsn(new Utf8(Long.toString(ssn + i)));
dataStore.put(employee.getSsn().toString(), employee);
@@ -337,33 +461,42 @@ public class DataStoreTestUtil {
for (int i = 0; i < 1; i++) {
String key = Long.toString(ssn + i);
Employee employee = dataStore.get(key);
- assertEquals(now - 18L * YEAR_IN_MS, employee.getDateOfBirth());
+ assertEquals(now - 18L * YEAR_IN_MS, employee.getDateOfBirth().longValue());
assertEquals("John Doe " + (i + 5), employee.getName().toString());
- assertEquals(120000, employee.getSalary());
+ assertEquals(120000, employee.getSalary().intValue());
}
}
- public static void testUpdateWebPage(DataStore<String, WebPage> dataStore)
+ /**
+ * Here we create 7 {@link org.apache.gora.examples.generated.WebPage}
+ * objects and populate field data before flushing the objects to the
+ * datastore. We then get the objects, adding data to the 'content' and
+ * 'parsedContent' fields before clearing the 'outlinks' field and
+ * re-populating it. This data is then flushed to the datastore.
+ * Finally we get the {@link org.apache.gora.examples.generated.WebPage}
+ * objects and make various assertions over verious fields. This tests
+ * that we can update fields and that data can be written and read correctly.
+ * @param dataStore
+ * @throws IOException
+ * @throws Exception
+ */
+ public static void testUpdateWebPagePutToArray(DataStore<String, WebPage> dataStore)
throws IOException, Exception {
dataStore.createSchema();
String[] urls = {"http://a.com/a", "http://b.com/b", "http://c.com/c",
- "http://d.com/d", "http://e.com/e", "http://f.com/f", "http://g.com/g"};
+ "http://d.com/d", "http://e.com/e", "http://f.com/f", "http://g.com/g" };
String content = "content";
String parsedContent = "parsedContent";
- String anchor = "anchor";
int parsedContentCount = 0;
for (int i = 0; i < urls.length; i++) {
- WebPage webPage = dataStore.newPersistent();
+ WebPage webPage = WebPage.newBuilder().build();
webPage.setUrl(new Utf8(urls[i]));
for (parsedContentCount = 0; parsedContentCount < 5; parsedContentCount++) {
- webPage.addToParsedContent(new Utf8(parsedContent + i + "," + parsedContentCount));
- }
- for (int j = 0; j < urls.length; j += 2) {
- webPage.putToOutlinks(new Utf8(anchor + j), new Utf8(urls[j]));
+ webPage.getParsedContent().add(new Utf8(parsedContent + i + "," + parsedContentCount));
}
dataStore.put(webPage.getUrl().toString(), webPage);
}
@@ -374,15 +507,7 @@ public class DataStoreTestUtil {
WebPage webPage = dataStore.get(urls[i]);
webPage.setContent(ByteBuffer.wrap(ByteUtils.toBytes(content + i)));
for (parsedContentCount = 5; parsedContentCount < 10; parsedContentCount++) {
- webPage.addToParsedContent(new Utf8(parsedContent + i + "," + parsedContentCount));
- }
- webPage.getOutlinks().clear();
- for (int j = 1; j < urls.length; j += 2) {
- webPage.putToOutlinks(new Utf8(anchor + j), new Utf8(urls[j]));
- }
- //test for double put of same entries
- for (int j = 1; j < urls.length; j += 2) {
- webPage.putToOutlinks(new Utf8(anchor + j), new Utf8(urls[j]));
+ webPage.getParsedContent().add(new Utf8(parsedContent + i + "," + parsedContentCount));
}
dataStore.put(webPage.getUrl().toString(), webPage);
}
@@ -394,24 +519,90 @@ public class DataStoreTestUtil {
assertEquals(content + i, ByteUtils.toString( toByteArray(webPage.getContent()) ));
assertEquals(10, webPage.getParsedContent().size());
int j = 0;
- for (Utf8 pc : webPage.getParsedContent()) {
+ for (CharSequence pc : webPage.getParsedContent()) {
assertEquals(parsedContent + i + "," + j, pc.toString());
j++;
}
+ }
+ }
+
+ public static void testUpdateWebPagePutToNotNullableMap(DataStore<String, WebPage> dataStore)
+ throws IOException, Exception {
+ dataStore.createSchema();
+
+ String[] urls = {"http://a.com/a", "http://b.com/b", "http://c.com/c",
+ "http://d.com/d", "http://e.com/e", "http://f.com/f", "http://g.com/g" };
+ String anchor = "anchor";
+
+ // putting evens
+ for (int i = 0; i < urls.length; i++) {
+ WebPage webPage = WebPage.newBuilder().build();
+ webPage.setUrl(new Utf8(urls[i]));
+ for (int j = 0; j < urls.length; j += 2) {
+ webPage.getOutlinks().put(new Utf8(anchor + j), new Utf8(urls[j]));
+ }
+ dataStore.put(webPage.getUrl().toString(), webPage);
+ }
+ dataStore.flush();
+
+ // putting odds
+ for (int i = 0; i < urls.length; i++) {
+ WebPage webPage = dataStore.get(urls[i]);
+ webPage.getOutlinks().clear();
+ for (int j = 1; j < urls.length; j += 2) {
+ webPage.getOutlinks().put(new Utf8(anchor + j), new Utf8(urls[j]));
+ }
+ // test for double put of same entries
+ for (int j = 1; j < urls.length; j += 2) {
+ webPage.getOutlinks().put(new Utf8(anchor + j), new Utf8(urls[j]));
+ }
+ dataStore.put(webPage.getUrl().toString(), webPage);
+ }
+ dataStore.flush();
+
+ for (int i = 0; i < urls.length; i++) {
+ WebPage webPage = dataStore.get(urls[i]);
int count = 0;
- for (j = 1; j < urls.length; j += 2) {
- Utf8 link = webPage.getOutlinks().get(new Utf8(anchor + j));
+ for (int j = 1; j < urls.length; j += 2) {
+ CharSequence link = webPage.getOutlinks().get(new Utf8(anchor + j));
assertNotNull(link);
assertEquals(urls[j], link.toString());
count++;
}
assertEquals(count, webPage.getOutlinks().size());
}
+ }
+
+ public static void testUpdateWebPagePutToNullableMap(DataStore<String, WebPage> dataStore)
+ throws IOException, Exception {
+ dataStore.createSchema();
+
+ String[] urls = {"http://a.com/a", "http://b.com/b", "http://c.com/c",
+ "http://d.com/d", "http://e.com/e", "http://f.com/f", "http://g.com/g" };
+ String header = "header";
+ String[] headers = { "firstHeader", "secondHeader", "thirdHeader",
+ "fourthHeader", "fifthHeader", "sixthHeader" };
+
+ for (int i = 0; i < urls.length; i++) {
+ WebPage webPage = WebPage.newBuilder().build();
+ webPage.setUrl(new Utf8(urls[i]));
+ //test put for nullable map field
+ // we put data to the 'headers' field which is a Map with default value of 'null'
+ webPage.setHeaders(new HashMap<CharSequence, CharSequence>());
+ for (int j = 0; j < headers.length; j += 2) {
+ webPage.getHeaders().put(new Utf8(header + j), new Utf8(headers[j]));
+ }
+ dataStore.put(webPage.getUrl().toString(), webPage);
+ }
+
+ dataStore.flush();
for (int i = 0; i < urls.length; i++) {
WebPage webPage = dataStore.get(urls[i]);
- for (int j = 0; j < urls.length; j += 2) {
- webPage.putToOutlinks(new Utf8(anchor + j), new Utf8(urls[j]));
+ //webPage.getHeaders().clear(); //TODO clear method does not work
+ webPage.setHeaders(new HashMap<CharSequence, CharSequence>());
+ for (int j = 1; j < headers.length; j += 2) {
+ webPage.getHeaders().put(new Utf8(header + j), new Utf8(headers[j]));
}
dataStore.put(webPage.getUrl().toString(), webPage);
}
@@ -421,12 +612,93 @@ public class DataStoreTestUtil {
for (int i = 0; i < urls.length; i++) {
WebPage webPage = dataStore.get(urls[i]);
int count = 0;
+ for (int j = 1; j < headers.length; j += 2) {
+ CharSequence headerSample = webPage.getHeaders().get(new Utf8(header + j));
+ assertNotNull(headerSample);
+ assertEquals(headers[j], headerSample.toString());
+ count++;
+ }
+ assertEquals(count, webPage.getHeaders().size());
+ }
+ }
+
+ public static void testUpdateWebPageRemoveMapEntry(DataStore<String, WebPage> dataStore)
+ throws IOException, Exception {
+ dataStore.createSchema();
+
+ String[] urls = {"http://a.com/a", "http://b.com/b", "http://c.com/c",
+ "http://d.com/d", "http://e.com/e", "http://f.com/f", "http://g.com/g" };
+ String anchor = "anchor";
+
+ for (int i = 0; i < urls.length; i++) {
+ WebPage webPage = WebPage.newBuilder().build();
+ webPage.setUrl(new Utf8(urls[i]));
for (int j = 0; j < urls.length; j++) {
- Utf8 link = webPage.getOutlinks().get(new Utf8(anchor + j));
- assertNotNull(link);
- assertEquals(urls[j], link.toString());
+ webPage.getOutlinks().put(new Utf8(anchor + j), new Utf8(urls[j]));
+ }
+ dataStore.put(webPage.getUrl().toString(), webPage);
+ }
+
+ dataStore.flush();
+
+ // map entry removal test
+ for (int i = 0; i < urls.length; i++) {
+ WebPage webPage = dataStore.get(urls[i]);
+ for (int j = 1; j < urls.length; j += 2) {
+ webPage.getOutlinks().remove(new Utf8(anchor + j));
+ }
+ dataStore.put(webPage.getUrl().toString(), webPage);
+ }
+
+ dataStore.flush();
+
+ for (int i = 0; i < urls.length; i++) {
+ int count = 0;
+ WebPage webPage = dataStore.get(urls[i]);
+ for (int j = 1; j < urls.length; j += 2) {
+ CharSequence link = webPage.getOutlinks().get(new Utf8(anchor + j));
+ assertNull(link);
+ //assertEquals(urls[j], link.toString());
count++;
}
+ assertEquals(urls.length - count, webPage.getOutlinks().size());
+ }
+ }
+
+ public static void testUpdateWebPageRemoveField(DataStore<String, WebPage> dataStore)
+ throws IOException, Exception {
+ dataStore.createSchema();
+
+ String[] urls = {"http://a.com/a", "http://b.com/b", "http://c.com/c",
+ "http://d.com/d", "http://e.com/e", "http://f.com/f", "http://g.com/g" };
+ String header = "header";
+ String[] headers = { "firstHeader", "secondHeader", "thirdHeader",
+ "fourthHeader", "fifthHeader", "sixthHeader" };
+
+ for (int i = 0; i < urls.length; i++) {
+ WebPage webPage = WebPage.newBuilder().build();
+ webPage.setUrl(new Utf8(urls[i]));
+ webPage.setHeaders(new HashMap<CharSequence, CharSequence>());
+ for (int j = 0; j < headers.length; j++) {
+ webPage.getHeaders().put(new Utf8(header + j), new Utf8(headers[j]));
+ }
+ dataStore.put(webPage.getUrl().toString(), webPage);
+ }
+
+ dataStore.flush();
+
+ // nullable map field removal test
+ for (int i = 0; i < urls.length; i++) {
+ WebPage webPage = dataStore.get(urls[i]);
+ webPage.setHeaders(null);
+ dataStore.put(webPage.getUrl().toString(), webPage);
+ }
+
+ dataStore.flush();
+
+ for (int i = 0; i < urls.length; i++) {
+ WebPage webPage = dataStore.get(urls[i]);
+ assertNull(webPage.getHeaders());
}
}
@@ -440,19 +712,20 @@ public class DataStoreTestUtil {
" actual=" + CONTENTS[i] + " i=" + i
, Arrays.equals( toByteArray(page.getContent() )
, CONTENTS[i].getBytes()));
- GenericArray<Utf8> parsedContent = page.getParsedContent();
+
+ List<CharSequence> parsedContent = page.getParsedContent();
assertNotNull(parsedContent);
assertTrue(parsedContent.size() > 0);
int j=0;
String[] tokens = CONTENTS[i].split(" ");
- for(Utf8 token : parsedContent) {
+ for(CharSequence token : parsedContent) {
assertEquals(tokens[j++], token.toString());
}
} else {
// when page.getContent() is null
assertTrue(CONTENTS[i] == null) ;
- GenericArray<Utf8> parsedContent = page.getParsedContent();
+ List<CharSequence> parsedContent = page.getParsedContent();
assertNotNull(parsedContent);
assertTrue(parsedContent.size() == 0);
}
@@ -462,7 +735,7 @@ public class DataStoreTestUtil {
assertTrue(page.getOutlinks().size() > 0);
for(int k=0; k<LINKS[i].length; k++) {
assertEquals(ANCHORS[i][k],
- page.getFromOutlinks(new Utf8(URLS[LINKS[i][k]])).toString());
+ page.getOutlinks().get(new Utf8(URLS[LINKS[i][k]])).toString());
}
} else {
assertTrue(page.getOutlinks() == null || page.getOutlinks().isEmpty());
@@ -480,7 +753,7 @@ public class DataStoreTestUtil {
}
public static void testGetWebPage(DataStore<String, WebPage> store) throws IOException, Exception {
- testGetWebPage(store, WebPage._ALL_FIELDS);
+ testGetWebPage(store, getFields(WebPage.SCHEMA$.getFields()));
}
public static void testGetWebPageDefaultFields(DataStore<String, WebPage> store)
@@ -507,7 +780,7 @@ public class DataStoreTestUtil {
public static void testQueryWebPageSingleKey(DataStore<String, WebPage> store)
throws IOException, Exception {
- testQueryWebPageSingleKey(store, WebPage._ALL_FIELDS);
+ testQueryWebPageSingleKey(store, getFields(WebPage.SCHEMA$.getFields()));
}
public static void testQueryWebPageSingleKeyDefaultFields(
@@ -714,7 +987,7 @@ public class DataStoreTestUtil {
WebPageDataCreator.createWebPageData(store);
query = store.newQuery();
- query.setFields(WebPage._ALL_FIELDS);
+ query.setFields(AvroUtils.getSchemaFieldNames(WebPage.SCHEMA$));
assertNumResults(store.newQuery(), URLS.length);
store.deleteByQuery(query);
@@ -757,8 +1030,8 @@ public class DataStoreTestUtil {
WebPageDataCreator.createWebPageData(store);
query = store.newQuery();
- query.setFields(WebPage.Field.OUTLINKS.getName()
- , WebPage.Field.PARSED_CONTENT.getName(), WebPage.Field.CONTENT.getName());
+ query.setFields("outlinks"
+ , "parsedContent", "content");
assertNumResults(store.newQuery(), URLS.length);
store.deleteByQuery(query);
@@ -776,7 +1049,8 @@ public class DataStoreTestUtil {
assertNotNull(page.getUrl());
assertEquals(page.getUrl().toString(), SORTED_URLS[i]);
- assertEquals(0, page.getOutlinks().size());
+ assertEquals("Map of Outlinks should have a size of '0' as the deleteByQuery "
+ + "not only removes the data but also the data structure.", 0, page.getOutlinks().size());
assertEquals(0, page.getParsedContent().size());
if(page.getContent() != null) {
System.out.println("url:" + page.getUrl().toString());
@@ -790,7 +1064,7 @@ public class DataStoreTestUtil {
WebPageDataCreator.createWebPageData(store);
query = store.newQuery();
- query.setFields(WebPage.Field.URL.getName());
+ query.setFields("url");
String startKey = SORTED_URLS[NUM_KEYS];
String endKey = SORTED_URLS[SORTED_URLS.length - NUM_KEYS];
query.setStartKey(startKey);
@@ -832,10 +1106,10 @@ public class DataStoreTestUtil {
String url = "http://foo.com/";
store.createSchema();
- WebPage page = store.newPersistent();
- Metadata metadata = new Metadata();
+ WebPage page = WebPage.newBuilder().build();
+ Metadata metadata = Metadata.newBuilder().build();
metadata.setVersion(1);
- metadata.putToData(new Utf8("foo"), new Utf8("baz"));
+ metadata.getData().put(new Utf8("foo"), new Utf8("baz"));
page.setMetadata(metadata);
page.setUrl(new Utf8(url));
@@ -846,30 +1120,31 @@ public class DataStoreTestUtil {
page = store.get(revUrl);
metadata = page.getMetadata();
assertNotNull(metadata);
- assertEquals(1, metadata.getVersion());
+ assertEquals(1, metadata.getVersion().intValue());
assertEquals(new Utf8("baz"), metadata.getData().get(new Utf8("foo")));
}
public static void testPutArray(DataStore<String, WebPage> store)
throws IOException, Exception {
store.createSchema();
- WebPage page = store.newPersistent();
+ WebPage page = WebPage.newBuilder().build();
String[] tokens = {"example", "content", "in", "example.com"};
-
+ page.setParsedContent(new ArrayList<CharSequence>());
for(String token: tokens) {
- page.addToParsedContent(new Utf8(token));
+ page.getParsedContent().add(new Utf8(token));
}
store.put("com.example/http", page);
store.close();
+
}
public static byte[] testPutBytes(DataStore<String, WebPage> store)
throws IOException, Exception {
store.createSchema();
- WebPage page = store.newPersistent();
+ WebPage page = WebPage.newBuilder().build();
page.setUrl(new Utf8("http://example.com"));
byte[] contentBytes = "example content in example.com".getBytes();
ByteBuffer buff = ByteBuffer.wrap(contentBytes);
@@ -886,12 +1161,12 @@ public class DataStoreTestUtil {
store.createSchema();
- WebPage page = store.newPersistent();
+ WebPage page = WebPage.newBuilder().build();
page.setUrl(new Utf8("http://example.com"));
- page.putToOutlinks(new Utf8("http://example2.com"), new Utf8("anchor2"));
- page.putToOutlinks(new Utf8("http://example3.com"), new Utf8("anchor3"));
- page.putToOutlinks(new Utf8("http://example3.com"), new Utf8("anchor4"));
+ page.getOutlinks().put(new Utf8("http://example2.com"), new Utf8("anchor2"));
+ page.getOutlinks().put(new Utf8("http://example3.com"), new Utf8("anchor3"));
+ page.getOutlinks().put(new Utf8("http://example3.com"), new Utf8("anchor4"));
store.put("com.example/http", page);
store.close();
}
@@ -905,5 +1180,23 @@ public class DataStoreTestUtil {
}
return bytes;
}
-
+
+ public static String[] getFields(List<Field> schemaFields) {
+
+ List<Field> list = new ArrayList<Field>();
+ for (Field field : schemaFields) {
+ if (!Persistent.DIRTY_BYTES_FIELD_NAME.equalsIgnoreCase(field.name())) {
+ list.add(field);
+ }
+ }
+ schemaFields = list;
+
+ String[] fieldNames = new String[schemaFields.size()];
+ for(int i = 0; i<fieldNames.length; i++ ){
+ fieldNames[i] = schemaFields.get(i).name();
+ }
+
+ return fieldNames;
+ }
+
}
http://git-wip-us.apache.org/repos/asf/gora/blob/136fc595/gora-core/src/test/java/org/apache/gora/store/TestDataStoreFactory.java
----------------------------------------------------------------------
diff --git a/gora-core/src/test/java/org/apache/gora/store/TestDataStoreFactory.java b/gora-core/src/test/java/org/apache/gora/store/TestDataStoreFactory.java
index 5f3436c..a50fd71 100644
--- a/gora-core/src/test/java/org/apache/gora/store/TestDataStoreFactory.java
+++ b/gora-core/src/test/java/org/apache/gora/store/TestDataStoreFactory.java
@@ -20,9 +20,9 @@ package org.apache.gora.store;
import java.util.Properties;
-import static junit.framework.Assert.assertNotNull;
-import static junit.framework.Assert.assertEquals;
-import static junit.framework.Assert.assertNotSame;
+import static org.junit.Assert.assertNotNull;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNotSame;
import org.apache.gora.avro.store.DataFileAvroStore;
import org.apache.gora.mock.persistency.MockPersistent;
http://git-wip-us.apache.org/repos/asf/gora/blob/136fc595/gora-core/src/test/java/org/apache/gora/util/TestIOUtils.java
----------------------------------------------------------------------
diff --git a/gora-core/src/test/java/org/apache/gora/util/TestIOUtils.java b/gora-core/src/test/java/org/apache/gora/util/TestIOUtils.java
index 30efd8c..381d366 100644
--- a/gora-core/src/test/java/org/apache/gora/util/TestIOUtils.java
+++ b/gora-core/src/test/java/org/apache/gora/util/TestIOUtils.java
@@ -27,11 +27,10 @@ import java.io.EOFException;
import java.io.IOException;
import java.util.Arrays;
-import org.apache.avro.ipc.ByteBufferInputStream;
-import org.apache.avro.ipc.ByteBufferOutputStream;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
+import org.apache.avro.util.ByteBufferInputStream;
+import org.apache.avro.util.ByteBufferOutputStream;
import org.apache.gora.mapreduce.GoraMapReduceUtils;
+import org.apache.gora.util.IOUtils;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.io.DataInputBuffer;
import org.apache.hadoop.io.DataOutputBuffer;
@@ -39,6 +38,8 @@ import org.apache.hadoop.io.Text;
import org.apache.hadoop.io.Writable;
import org.junit.Test;
import static org.junit.Assert.assertEquals;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
/**
* Test case for {@link IOUtils} class.
http://git-wip-us.apache.org/repos/asf/gora/blob/136fc595/gora-dynamodb/src/main/java/org/apache/gora/dynamodb/store/DynamoDBStore.java
----------------------------------------------------------------------
diff --git a/gora-dynamodb/src/main/java/org/apache/gora/dynamodb/store/DynamoDBStore.java b/gora-dynamodb/src/main/java/org/apache/gora/dynamodb/store/DynamoDBStore.java
index d84f051..7192c8a 100644
--- a/gora-dynamodb/src/main/java/org/apache/gora/dynamodb/store/DynamoDBStore.java
+++ b/gora-dynamodb/src/main/java/org/apache/gora/dynamodb/store/DynamoDBStore.java
@@ -36,7 +36,6 @@ import org.apache.gora.persistency.Persistent;
import org.apache.gora.query.PartitionQuery;
import org.apache.gora.query.Query;
import org.apache.gora.query.Result;
-import org.apache.gora.store.DataStoreFactory;
import org.apache.gora.store.ws.impl.WSDataStoreBase;
import org.apache.gora.util.GoraException;
import org.slf4j.Logger;
@@ -162,11 +161,11 @@ public class DynamoDBStore<K, T extends Persistent> extends WSDataStoreBase<K, T
LOG.debug("Initializing DynamoDB store");
getCredentials();
setWsProvider(wsProvider);
- preferredSchema = DataStoreFactory.findProperty(properties, this, PREF_SCH_NAME, null);
- dynamoDBClient = getClient(DataStoreFactory.findProperty(properties, this, CLI_TYP_PROP, null),(AWSCredentials)getConf());
- dynamoDBClient.setEndpoint(DataStoreFactory.findProperty(properties, this, ENDPOINT_PROP, null));
+ preferredSchema = properties.getProperty(PREF_SCH_NAME);
+ dynamoDBClient = getClient(properties.getProperty(CLI_TYP_PROP),(AWSCredentials)getConf());
+ dynamoDBClient.setEndpoint(properties.getProperty(ENDPOINT_PROP));
mapping = readMapping();
- consistency = DataStoreFactory.findProperty(properties, this, CONSISTENCY_READS, null);
+ consistency = properties.getProperty(CONSISTENCY_READS);
persistentClass = pPersistentClass;
}
catch (Exception e) {