You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@gora.apache.org by le...@apache.org on 2013/12/02 18:11:39 UTC
svn commit: r1547110 - in /gora/branches/GORA_94: ./
gora-core/src/main/java/org/apache/gora/persistency/impl/
gora-core/src/test/java/org/apache/gora/store/ gora-hbase/
gora-hbase/src/main/java/org/apache/gora/hbase/store/
gora-hbase/src/main/java/org...
Author: lewismc
Date: Mon Dec 2 17:11:38 2013
New Revision: 1547110
URL: http://svn.apache.org/r1547110
Log:
GORA-246v3
Modified:
gora/branches/GORA_94/gora-core/src/main/java/org/apache/gora/persistency/impl/PersistentBase.java
gora/branches/GORA_94/gora-core/src/test/java/org/apache/gora/store/DataStoreTestUtil.java
gora/branches/GORA_94/gora-hbase/pom.xml
gora/branches/GORA_94/gora-hbase/src/main/java/org/apache/gora/hbase/store/HBaseStore.java
gora/branches/GORA_94/gora-hbase/src/main/java/org/apache/gora/hbase/util/HBaseByteInterface.java
gora/branches/GORA_94/gora-hbase/src/test/java/org/apache/gora/hbase/util/TestHBaseByteInterface.java
gora/branches/GORA_94/pom.xml
Modified: gora/branches/GORA_94/gora-core/src/main/java/org/apache/gora/persistency/impl/PersistentBase.java
URL: http://svn.apache.org/viewvc/gora/branches/GORA_94/gora-core/src/main/java/org/apache/gora/persistency/impl/PersistentBase.java?rev=1547110&r1=1547109&r2=1547110&view=diff
==============================================================================
--- gora/branches/GORA_94/gora-core/src/main/java/org/apache/gora/persistency/impl/PersistentBase.java (original)
+++ gora/branches/GORA_94/gora-core/src/main/java/org/apache/gora/persistency/impl/PersistentBase.java Mon Dec 2 17:11:38 2013
@@ -1,25 +1,26 @@
/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
+* Licensed to the Apache Software Foundation (ASF) under one
+* or more contributor license agreements. See the NOTICE file
+* distributed with this work for additional information
+* regarding copyright ownership. The ASF licenses this file
+* to you under the Apache License, Version 2.0 (the
+* "License"); you may not use this file except in compliance
+* with the License. You may obtain a copy of the License at
+*
+* http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+* See the License for the specific language governing permissions and
+* limitations under the License.
+*/
package org.apache.gora.persistency.impl;
import java.nio.ByteBuffer;
import java.util.Collection;
import java.util.List;
+
import org.apache.avro.Schema.Field;
import org.apache.avro.specific.SpecificData;
import org.apache.avro.specific.SpecificRecord;
@@ -28,10 +29,10 @@ import org.apache.gora.persistency.Dirty
import org.apache.gora.persistency.Persistent;
/**
- * Base class implementing common functionality for Persistent
- * classes.
- */
-public abstract class PersistentBase extends SpecificRecordBase implements Persistent {
+* Base classs implementing common functionality for Persistent classes.
+*/
+public abstract class PersistentBase extends SpecificRecordBase implements
+ Persistent {
public static class PersistentData extends SpecificData {
private static final PersistentData INSTANCE = new PersistentData();
@@ -114,14 +115,11 @@ public abstract class PersistentBase ext
case MAP:
case ARRAY:
Object value = get(field.pos());
- return value==null ? false : ((Dirtyable) value).isDirty();
+ return !(value instanceof Dirtyable) || value==null ? false : ((Dirtyable) value).isDirty();
case UNION:
value = get(field.pos());
- if (value instanceof Dirtyable) {
- return ((Dirtyable) value).isDirty();
- }
+ return !(value instanceof Dirtyable) || value==null ? false : ((Dirtyable) value).isDirty();
default:
- // TODO add sufficient logging
break;
}
return false;
@@ -143,7 +141,8 @@ public abstract class PersistentBase ext
public boolean isDirty(String fieldName) {
Field field = getSchema().getField(fieldName);
if(field == null){
- throw new IndexOutOfBoundsException("Field "+ fieldName + " does not exist in this schema.");
+ throw new IndexOutOfBoundsException
+ ("Field "+ fieldName + " does not exist in this schema.");
}
return isDirty(field.pos());
}
@@ -184,10 +183,10 @@ public abstract class PersistentBase ext
if (!unmanagedFields.contains(field))
continue;
/*
-* TODO: Its more in the spirit of Gora's clear method to actually clear
-* data structures, but since avro no-longer defaults to having empty
-* structures the way to do this consistently would be complicated.
-*/
+ * TODO: Its more in the spirit of Gora's clear method to actually clear
+ * data structures, but since avro no-longer defaults to having empty
+ * structures the way to do this consistently would be complicated.
+ */
put(field.pos(), null);
}
clearDirty();
Modified: gora/branches/GORA_94/gora-core/src/test/java/org/apache/gora/store/DataStoreTestUtil.java
URL: http://svn.apache.org/viewvc/gora/branches/GORA_94/gora-core/src/test/java/org/apache/gora/store/DataStoreTestUtil.java?rev=1547110&r1=1547109&r2=1547110&view=diff
==============================================================================
--- gora/branches/GORA_94/gora-core/src/test/java/org/apache/gora/store/DataStoreTestUtil.java (original)
+++ gora/branches/GORA_94/gora-core/src/test/java/org/apache/gora/store/DataStoreTestUtil.java Mon Dec 2 17:11:38 2013
@@ -89,7 +89,7 @@ public class DataStoreTestUtil {
public static <K> Employee createEmployee(
DataStore<K, Employee> dataStore) throws IOException, Exception {
- Employee employee = dataStore.newPersistent();
+ Employee employee = Employee.newBuilder().build();
employee.setName(new Utf8("Random Joe"));
employee.setDateOfBirth( System.currentTimeMillis() - 20L * YEAR_IN_MS );
employee.setSalary(100000);
@@ -100,7 +100,7 @@ public class DataStoreTestUtil {
public static <K> Employee createBoss(
DataStore<K, Employee> dataStore) throws IOException, Exception {
- Employee employee = dataStore.newPersistent();
+ Employee employee = Employee.newBuilder().build();
employee.setName(new Utf8("Random boss"));
employee.setDateOfBirth( System.currentTimeMillis() - 22L * YEAR_IN_MS );
employee.setSalary(1000000);
@@ -306,7 +306,7 @@ public class DataStoreTestUtil {
long now = System.currentTimeMillis();
for (int i = 0; i < 5; i++) {
- Employee employee = dataStore.newPersistent();
+ Employee employee = Employee.newBuilder().build();
employee.setName(new Utf8("John Doe " + i));
employee.setDateOfBirth(now - 20L * YEAR_IN_MS);
employee.setSalary(100000);
@@ -317,7 +317,7 @@ public class DataStoreTestUtil {
dataStore.flush();
for (int i = 0; i < 1; i++) {
- Employee employee = dataStore.newPersistent();
+ Employee employee = Employee.newBuilder().build();
employee.setName(new Utf8("John Doe " + (i + 5)));
employee.setDateOfBirth(now - 18L * YEAR_IN_MS);
employee.setSalary(120000);
@@ -350,11 +350,13 @@ public class DataStoreTestUtil {
for (int i = 0; i < urls.length; i++) {
- WebPage webPage = dataStore.newPersistent();
+ WebPage webPage = WebPage.newBuilder().build();
webPage.setUrl(new Utf8(urls[i]));
+ webPage.setParsedContent(new ArrayList<CharSequence>());
for (parsedContentCount = 0; parsedContentCount < 5; parsedContentCount++) {
webPage.getParsedContent().add(new Utf8(parsedContent + i + "," + parsedContentCount));
}
+ webPage.setOutlinks(new HashMap<CharSequence, CharSequence>());
for (int j = 0; j < urls.length; j += 2) {
webPage.getOutlinks().put(new Utf8(anchor + j), new Utf8(urls[j]));
}
@@ -843,23 +845,24 @@ public class DataStoreTestUtil {
public static void testPutArray(DataStore<String, WebPage> store)
throws IOException, Exception {
store.createSchema();
- WebPage page = store.newPersistent();
+ WebPage page = WebPage.newBuilder().build();
String[] tokens = {"example", "content", "in", "example.com"};
-
+ page.setParsedContent(new ArrayList<CharSequence>());
for(String token: tokens) {
page.getParsedContent().add(new Utf8(token));
}
store.put("com.example/http", page);
store.close();
+
}
public static byte[] testPutBytes(DataStore<String, WebPage> store)
throws IOException, Exception {
store.createSchema();
- WebPage page = store.newPersistent();
+ WebPage page = WebPage.newBuilder().build();
page.setUrl(new Utf8("http://example.com"));
byte[] contentBytes = "example content in example.com".getBytes();
ByteBuffer buff = ByteBuffer.wrap(contentBytes);
@@ -876,9 +879,10 @@ public class DataStoreTestUtil {
store.createSchema();
- WebPage page = store.newPersistent();
+ WebPage page = WebPage.newBuilder().build();
page.setUrl(new Utf8("http://example.com"));
+ page.setOutlinks(new HashMap<CharSequence, CharSequence>());
page.getOutlinks().put(new Utf8("http://example2.com"), new Utf8("anchor2"));
page.getOutlinks().put(new Utf8("http://example3.com"), new Utf8("anchor3"));
page.getOutlinks().put(new Utf8("http://example3.com"), new Utf8("anchor4"));
@@ -913,5 +917,5 @@ public class DataStoreTestUtil {
return fieldNames;
}
-
+
}
Modified: gora/branches/GORA_94/gora-hbase/pom.xml
URL: http://svn.apache.org/viewvc/gora/branches/GORA_94/gora-hbase/pom.xml?rev=1547110&r1=1547109&r2=1547110&view=diff
==============================================================================
--- gora/branches/GORA_94/gora-hbase/pom.xml (original)
+++ gora/branches/GORA_94/gora-hbase/pom.xml Mon Dec 2 17:11:38 2013
@@ -100,6 +100,7 @@
<dependency>
<groupId>org.apache.gora</groupId>
<artifactId>gora-core</artifactId>
+ <scope>compile</scope>
</dependency>
<dependency>
@@ -108,45 +109,44 @@
<type>test-jar</type>
<scope>test</scope>
</dependency>
+ <!-- END of Gora Internal Dependencies -->
- <!-- Hadoop Dependencies -->
<dependency>
<groupId>org.apache.hbase</groupId>
<artifactId>hbase</artifactId>
+ <scope>compile</scope>
</dependency>
<dependency>
- <groupId>org.apache.hbase</groupId>
- <artifactId>hbase</artifactId>
- <type>test-jar</type>
- </dependency>
-
-
- <dependency>
<groupId>org.apache.avro</groupId>
<artifactId>avro</artifactId>
+ <scope>compile</scope>
</dependency>
<!-- Misc Dependencies -->
<dependency>
<groupId>org.jdom</groupId>
<artifactId>jdom</artifactId>
+ <scope>compile</scope>
</dependency>
<!-- Logging Dependencies -->
<dependency>
<groupId>org.slf4j</groupId>
<artifactId>slf4j-api</artifactId>
+ <scope>compile</scope>
</dependency>
<dependency>
<groupId>org.slf4j</groupId>
<artifactId>slf4j-simple</artifactId>
+ <scope>compile</scope>
</dependency>
<dependency>
<groupId>log4j</groupId>
<artifactId>log4j</artifactId>
+ <scope>runtime</scope>
<exclusions>
<exclusion>
<groupId>javax.jms</groupId>
@@ -154,18 +154,28 @@
</exclusion>
</exclusions>
</dependency>
+ <!-- END of Logging Dependencies -->
<!-- Testing Dependencies -->
<dependency>
<groupId>junit</groupId>
<artifactId>junit</artifactId>
+ <scope>test</scope>
</dependency>
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-test</artifactId>
+ <scope>test</scope>
</dependency>
-
+
+ <dependency>
+ <groupId>org.apache.hbase</groupId>
+ <artifactId>hbase</artifactId>
+ <type>test-jar</type>
+ <scope>test</scope>
+ </dependency>
+ <!-- END of Testing Dependencies -->
</dependencies>
</project>
Modified: gora/branches/GORA_94/gora-hbase/src/main/java/org/apache/gora/hbase/store/HBaseStore.java
URL: http://svn.apache.org/viewvc/gora/branches/GORA_94/gora-hbase/src/main/java/org/apache/gora/hbase/store/HBaseStore.java?rev=1547110&r1=1547109&r2=1547110&view=diff
==============================================================================
--- gora/branches/GORA_94/gora-hbase/src/main/java/org/apache/gora/hbase/store/HBaseStore.java (original)
+++ gora/branches/GORA_94/gora-hbase/src/main/java/org/apache/gora/hbase/store/HBaseStore.java Mon Dec 2 17:11:38 2013
@@ -25,7 +25,6 @@ import java.io.IOException;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.HashMap;
-import java.util.Iterator;
import java.util.List;
import java.util.Map;
import java.util.Map.Entry;
@@ -36,10 +35,8 @@ import java.util.Set;
import org.apache.avro.Schema;
import org.apache.avro.Schema.Field;
import org.apache.avro.Schema.Type;
-import org.apache.avro.generic.GenericArray;
import org.apache.avro.util.Utf8;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
+
import org.apache.gora.hbase.query.HBaseGetResult;
import org.apache.gora.hbase.query.HBaseQuery;
import org.apache.gora.hbase.query.HBaseScannerResult;
@@ -52,6 +49,7 @@ import org.apache.gora.query.PartitionQu
import org.apache.gora.query.Query;
import org.apache.gora.query.impl.PartitionQueryImpl;
import org.apache.gora.store.impl.DataStoreBase;
+
import org.apache.hadoop.conf.Configurable;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hbase.HBaseConfiguration;
@@ -66,10 +64,14 @@ import org.apache.hadoop.hbase.client.Re
import org.apache.hadoop.hbase.client.Scan;
import org.apache.hadoop.hbase.util.Bytes;
import org.apache.hadoop.hbase.util.Pair;
+
import org.jdom.Document;
import org.jdom.Element;
import org.jdom.input.SAXBuilder;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
/**
* DataStore for HBase. Thread safe.
*
@@ -218,9 +220,9 @@ implements Configurable {
Delete delete = new Delete(keyRaw);
boolean hasPuts = false;
boolean hasDeletes = false;
- Iterator<Field> iter = schema.getFields().iterator();
- for (int i = 0; iter.hasNext(); i++) {
- Field field = iter.next();
+ List<Field> fields = schema.getFields();
+ for (int i = 1; i<fields.size(); i++) {
+ Field field = fields.get(i);
if (i==0 || !persistent.isDirty(i)) {
continue;
}
@@ -237,7 +239,7 @@ implements Configurable {
for(Entry entry: set) {
byte[] qual = toBytes(entry.getKey());
byte[] val = toBytes(entry.getValue(), field.schema().getValueType());
- // XXX - Gora 207: Top-most record level ["null","type"] must be saved raw. "null"=>delete
+ // Gora 207: Top-most record level ["null","type"] must be saved raw. "null"=>delete
if (val == null) { // value == null => must delete the column
delete.deleteColumn(hcol.getFamily(), qual);
hasDeletes = true;
@@ -248,24 +250,24 @@ implements Configurable {
}
break;
case ARRAY:
- if(o instanceof GenericArray) {
- GenericArray arr = (GenericArray) o;
- int j=0;
- for(Object item : arr) {
- byte[] val = toBytes(item, field.schema().getElementType());
- // XXX - Gora 207: Top-most record level ["null","type"] must be saved raw. "null"=>delete
- if (val == null) { // value == null => must delete the column
- delete.deleteColumn(hcol.getFamily(), Bytes.toBytes(j++));
- hasDeletes = true;
- } else {
- put.add(hcol.getFamily(), Bytes.toBytes(j++), val);
- hasPuts = true;
- }
+ List<?> array = (List<?>) o;
+ int j=0;
+ for(Object item : array) {
+ byte[] val = toBytes(item);
+ // Gora 207: Top-most record level ["null","type"]
+ // must be saved raw. "null"=>delete
+ if (val == null) { // value == null => must delete the column
+ delete.deleteColumn(hcol.getFamily(), Bytes.toBytes(j++));
+ hasDeletes = true;
+ } else {
+ put.add(hcol.getFamily(), Bytes.toBytes(j++), val);
+ hasPuts = true;
}
}
break;
default:
- // XXX - Gora 207: Top-most record level ["null","type"] must be saved raw. "null"=>delete
+ // Gora 207: Top-most record level ["null","type"]
+ // must be saved raw. "null"=>delete
byte[] serializedBytes = toBytes(o, field.schema()) ;
if (serializedBytes == null) { // value == null => must delete the column
delete.deleteColumn(hcol.getFamily(), hcol.getQualifier());
@@ -589,7 +591,7 @@ implements Configurable {
persistent.put(field.pos(), fromBytes(field.schema(), val));
}
- @SuppressWarnings("rawtypes")
+ @SuppressWarnings({ "rawtypes", "unchecked" })
private void setField(T persistent, Field field, List list) {
persistent.put(field.pos(), new DirtyListWrapper(list));
}
Modified: gora/branches/GORA_94/gora-hbase/src/main/java/org/apache/gora/hbase/util/HBaseByteInterface.java
URL: http://svn.apache.org/viewvc/gora/branches/GORA_94/gora-hbase/src/main/java/org/apache/gora/hbase/util/HBaseByteInterface.java?rev=1547110&r1=1547109&r2=1547110&view=diff
==============================================================================
--- gora/branches/GORA_94/gora-hbase/src/main/java/org/apache/gora/hbase/util/HBaseByteInterface.java (original)
+++ gora/branches/GORA_94/gora-hbase/src/main/java/org/apache/gora/hbase/util/HBaseByteInterface.java Mon Dec 2 17:11:38 2013
@@ -20,12 +20,7 @@ package org.apache.gora.hbase.util;
import java.io.ByteArrayOutputStream;
import java.io.IOException;
-import java.io.OutputStream;
-import java.lang.reflect.InvocationTargetException;
import java.nio.ByteBuffer;
-import java.util.HashMap;
-import java.util.Map;
-import java.util.WeakHashMap;
import java.util.concurrent.ConcurrentHashMap;
import org.apache.avro.Schema;
@@ -33,17 +28,14 @@ import org.apache.avro.Schema.Type;
import org.apache.avro.generic.GenericData;
import org.apache.avro.io.BinaryDecoder;
import org.apache.avro.io.BinaryEncoder;
-import org.apache.avro.io.Decoder;
import org.apache.avro.io.DecoderFactory;
import org.apache.avro.io.EncoderFactory;
-import org.apache.avro.io.ResolvingDecoder;
import org.apache.avro.specific.SpecificDatumReader;
import org.apache.avro.specific.SpecificDatumWriter;
import org.apache.avro.util.Utf8;
-import org.apache.gora.persistency.Persistent;
+
import org.apache.gora.util.AvroUtils;
-import org.apache.gora.util.ClassLoadingUtils;
-import org.apache.gora.util.ReflectionUtils;
+
import org.apache.hadoop.hbase.util.Bytes;
/**
@@ -68,13 +60,15 @@ public class HBaseByteInterface {
* writer pair for every schema, instead of one for every thread.
*/
- public static final ConcurrentHashMap<String, SpecificDatumReader<?>> readerMap = new ConcurrentHashMap<String, SpecificDatumReader<?>>();
+ public static final ConcurrentHashMap<String, SpecificDatumReader<?>> readerMap =
+ new ConcurrentHashMap<String, SpecificDatumReader<?>>();
- public static final ConcurrentHashMap<String, SpecificDatumWriter<?>> writerMap = new ConcurrentHashMap<String, SpecificDatumWriter<?>>();
+ public static final ConcurrentHashMap<String, SpecificDatumWriter<?>> writerMap =
+ new ConcurrentHashMap<String, SpecificDatumWriter<?>>();
/**
- * Deserializes an array of bytes matching the given schema to the proper basic (enum, Utf8,...) or
- * complex type (Persistent/Record).
+ * Deserializes an array of bytes matching the given schema to the proper basic
+ * (enum, Utf8,...) or complex type (Persistent/Record).
*
* Does not handle <code>arrays/maps</code> if not inside a <code>record</code> type.
*
@@ -83,7 +77,7 @@ public class HBaseByteInterface {
* @return Enum|Utf8|ByteBuffer|Integer|Long|Float|Double|Boolean|Persistent|Null
* @throws IOException
*/
- @SuppressWarnings({ "rawtypes", "unchecked" })
+ @SuppressWarnings({ "rawtypes" })
public static Object fromBytes(Schema schema, byte[] val) throws IOException {
Type type = schema.getType();
switch (type) {
Modified: gora/branches/GORA_94/gora-hbase/src/test/java/org/apache/gora/hbase/util/TestHBaseByteInterface.java
URL: http://svn.apache.org/viewvc/gora/branches/GORA_94/gora-hbase/src/test/java/org/apache/gora/hbase/util/TestHBaseByteInterface.java?rev=1547110&r1=1547109&r2=1547110&view=diff
==============================================================================
--- gora/branches/GORA_94/gora-hbase/src/test/java/org/apache/gora/hbase/util/TestHBaseByteInterface.java (original)
+++ gora/branches/GORA_94/gora-hbase/src/test/java/org/apache/gora/hbase/util/TestHBaseByteInterface.java Mon Dec 2 17:11:38 2013
@@ -29,6 +29,7 @@ import java.util.concurrent.Executors;
import java.util.concurrent.Future;
import org.apache.avro.util.Utf8;
+
import org.apache.gora.examples.generated.Employee;
import org.apache.gora.examples.generated.Metadata;
@@ -38,7 +39,7 @@ import org.junit.Test;
public class TestHBaseByteInterface {
- private static final Random RANDOM = new Random();
+ private static final Random RANDOM = new Random(0);
@Test
public void testEncodingDecoding() throws Exception {
Modified: gora/branches/GORA_94/pom.xml
URL: http://svn.apache.org/viewvc/gora/branches/GORA_94/pom.xml?rev=1547110&r1=1547109&r2=1547110&view=diff
==============================================================================
--- gora/branches/GORA_94/pom.xml (original)
+++ gora/branches/GORA_94/pom.xml Mon Dec 2 17:11:38 2013
@@ -592,7 +592,7 @@
<!-- Hadoop Dependencies -->
<hadoop.version>1.0.1</hadoop.version>
<hadoop.test.version>1.0.1</hadoop.test.version>
- <hbase.version>0.94.9</hbase.version>
+ <hbase.version>0.94.14</hbase.version>
<avro.version>1.7.4</avro.version>
<cxf-rt-frontend-jaxrs.version>2.5.2</cxf-rt-frontend-jaxrs.version>
<!-- Amazon Dependencies -->