You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@gora.apache.org by al...@apache.org on 2013/03/12 18:07:03 UTC
svn commit: r1455623 - in /gora/branches/GORA_174/gora-core/src:
examples/avro/ examples/java/org/apache/gora/examples/
examples/java/org/apache/gora/examples/generated/
examples/java/org/apache/gora/examples/mapreduce/
main/java/org/apache/gora/compil...
Author: alfonsonishikawa
Date: Tue Mar 12 17:07:03 2013
New Revision: 1455623
URL: http://svn.apache.org/r1455623
Log:
Applied working patch GORA-174 [0] version 12/Mar/13 17:55
[0] https://issues.apache.org/jira/browse/GORA-174
Modified:
gora/branches/GORA_174/gora-core/src/examples/avro/webpage.json
gora/branches/GORA_174/gora-core/src/examples/java/org/apache/gora/examples/WebPageDataCreator.java
gora/branches/GORA_174/gora-core/src/examples/java/org/apache/gora/examples/generated/Employee.java
gora/branches/GORA_174/gora-core/src/examples/java/org/apache/gora/examples/generated/Metadata.java
gora/branches/GORA_174/gora-core/src/examples/java/org/apache/gora/examples/generated/TokenDatum.java
gora/branches/GORA_174/gora-core/src/examples/java/org/apache/gora/examples/generated/WebPage.java
gora/branches/GORA_174/gora-core/src/examples/java/org/apache/gora/examples/mapreduce/WordCount.java
gora/branches/GORA_174/gora-core/src/main/java/org/apache/gora/compiler/GoraCompiler.java
gora/branches/GORA_174/gora-core/src/main/java/org/apache/gora/persistency/impl/PersistentBase.java
gora/branches/GORA_174/gora-core/src/test/java/org/apache/gora/mapreduce/MapReduceTestUtils.java
gora/branches/GORA_174/gora-core/src/test/java/org/apache/gora/store/DataStoreTestUtil.java
Modified: gora/branches/GORA_174/gora-core/src/examples/avro/webpage.json
URL: http://svn.apache.org/viewvc/gora/branches/GORA_174/gora-core/src/examples/avro/webpage.json?rev=1455623&r1=1455622&r2=1455623&view=diff
==============================================================================
--- gora/branches/GORA_174/gora-core/src/examples/avro/webpage.json (original)
+++ gora/branches/GORA_174/gora-core/src/examples/avro/webpage.json Tue Mar 12 17:07:03 2013
@@ -4,7 +4,7 @@
"namespace": "org.apache.gora.examples.generated",
"fields" : [
{"name": "url", "type": "string"},
- {"name": "content", "type": "bytes"},
+ {"name": "content", "type": ["null","bytes"]},
{"name": "parsedContent", "type": {"type":"array", "items": "string"}},
{"name": "outlinks", "type": {"type":"map", "values":"string"}},
{"name": "metadata", "type": {
Modified: gora/branches/GORA_174/gora-core/src/examples/java/org/apache/gora/examples/WebPageDataCreator.java
URL: http://svn.apache.org/viewvc/gora/branches/GORA_174/gora-core/src/examples/java/org/apache/gora/examples/WebPageDataCreator.java?rev=1455623&r1=1455622&r2=1455623&view=diff
==============================================================================
--- gora/branches/GORA_174/gora-core/src/examples/java/org/apache/gora/examples/WebPageDataCreator.java (original)
+++ gora/branches/GORA_174/gora-core/src/examples/java/org/apache/gora/examples/WebPageDataCreator.java Tue Mar 12 17:07:03 2013
@@ -61,7 +61,7 @@ public class WebPageDataCreator {
}
public static final String[] CONTENTS = {
- "foo baz bar",
+ null,
"foo",
"foo1 bar1 baz1",
"a b c d e",
@@ -116,11 +116,12 @@ public class WebPageDataCreator {
for(int i=0; i<URLS.length; i++) {
page = new WebPage();
page.setUrl(new Utf8(URLS[i]));
- page.setContent(ByteBuffer.wrap(CONTENTS[i].getBytes()));
- for(String token : CONTENTS[i].split(" ")) {
- page.addToParsedContent(new Utf8(token));
- }
-
+ if (CONTENTS[i]!=null){
+ page.setContent(ByteBuffer.wrap(CONTENTS[i].getBytes()));
+ for(String token : CONTENTS[i].split(" ")) {
+ page.addToParsedContent(new Utf8(token));
+ }
+ }
for(int j=0; j<LINKS[i].length; j++) {
page.putToOutlinks(new Utf8(URLS[LINKS[i][j]]), new Utf8(ANCHORS[i][j]));
}
Modified: gora/branches/GORA_174/gora-core/src/examples/java/org/apache/gora/examples/generated/Employee.java
URL: http://svn.apache.org/viewvc/gora/branches/GORA_174/gora-core/src/examples/java/org/apache/gora/examples/generated/Employee.java?rev=1455623&r1=1455622&r2=1455623&view=diff
==============================================================================
--- gora/branches/GORA_174/gora-core/src/examples/java/org/apache/gora/examples/generated/Employee.java (original)
+++ gora/branches/GORA_174/gora-core/src/examples/java/org/apache/gora/examples/generated/Employee.java Tue Mar 12 17:07:03 2013
@@ -1,3 +1,21 @@
+/**
+ *Licensed to the Apache Software Foundation (ASF) under one
+ *or more contributor license agreements. See the NOTICE file
+ *distributed with this work for additional information
+ *regarding copyright ownership. The ASF licenses this file
+ *to you under the Apache License, Version 2.0 (the"
+ *License"); you may not use this file except in compliance
+ *with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *Unless required by applicable law or agreed to in writing, software
+ *distributed under the License is distributed on an "AS IS" BASIS,
+ *WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *See the License for the specific language governing permissions and
+ *limitations under the License.
+ */
+
package org.apache.gora.examples.generated;
import java.nio.ByteBuffer;
@@ -10,6 +28,7 @@ import org.apache.avro.Protocol;
import org.apache.avro.util.Utf8;
import org.apache.avro.ipc.AvroRemoteException;
import org.apache.avro.generic.GenericArray;
+import org.apache.avro.specific.FixedSize;
import org.apache.avro.specific.SpecificExceptionBase;
import org.apache.avro.specific.SpecificRecordBase;
import org.apache.avro.specific.SpecificRecord;
Modified: gora/branches/GORA_174/gora-core/src/examples/java/org/apache/gora/examples/generated/Metadata.java
URL: http://svn.apache.org/viewvc/gora/branches/GORA_174/gora-core/src/examples/java/org/apache/gora/examples/generated/Metadata.java?rev=1455623&r1=1455622&r2=1455623&view=diff
==============================================================================
--- gora/branches/GORA_174/gora-core/src/examples/java/org/apache/gora/examples/generated/Metadata.java (original)
+++ gora/branches/GORA_174/gora-core/src/examples/java/org/apache/gora/examples/generated/Metadata.java Tue Mar 12 17:07:03 2013
@@ -1,3 +1,21 @@
+/**
+ *Licensed to the Apache Software Foundation (ASF) under one
+ *or more contributor license agreements. See the NOTICE file
+ *distributed with this work for additional information
+ *regarding copyright ownership. The ASF licenses this file
+ *to you under the Apache License, Version 2.0 (the"
+ *License"); you may not use this file except in compliance
+ *with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *Unless required by applicable law or agreed to in writing, software
+ *distributed under the License is distributed on an "AS IS" BASIS,
+ *WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *See the License for the specific language governing permissions and
+ *limitations under the License.
+ */
+
package org.apache.gora.examples.generated;
import java.nio.ByteBuffer;
@@ -10,6 +28,7 @@ import org.apache.avro.Protocol;
import org.apache.avro.util.Utf8;
import org.apache.avro.ipc.AvroRemoteException;
import org.apache.avro.generic.GenericArray;
+import org.apache.avro.specific.FixedSize;
import org.apache.avro.specific.SpecificExceptionBase;
import org.apache.avro.specific.SpecificRecordBase;
import org.apache.avro.specific.SpecificRecord;
Modified: gora/branches/GORA_174/gora-core/src/examples/java/org/apache/gora/examples/generated/TokenDatum.java
URL: http://svn.apache.org/viewvc/gora/branches/GORA_174/gora-core/src/examples/java/org/apache/gora/examples/generated/TokenDatum.java?rev=1455623&r1=1455622&r2=1455623&view=diff
==============================================================================
--- gora/branches/GORA_174/gora-core/src/examples/java/org/apache/gora/examples/generated/TokenDatum.java (original)
+++ gora/branches/GORA_174/gora-core/src/examples/java/org/apache/gora/examples/generated/TokenDatum.java Tue Mar 12 17:07:03 2013
@@ -1,3 +1,21 @@
+/**
+ *Licensed to the Apache Software Foundation (ASF) under one
+ *or more contributor license agreements. See the NOTICE file
+ *distributed with this work for additional information
+ *regarding copyright ownership. The ASF licenses this file
+ *to you under the Apache License, Version 2.0 (the"
+ *License"); you may not use this file except in compliance
+ *with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *Unless required by applicable law or agreed to in writing, software
+ *distributed under the License is distributed on an "AS IS" BASIS,
+ *WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *See the License for the specific language governing permissions and
+ *limitations under the License.
+ */
+
package org.apache.gora.examples.generated;
import java.nio.ByteBuffer;
@@ -10,6 +28,7 @@ import org.apache.avro.Protocol;
import org.apache.avro.util.Utf8;
import org.apache.avro.ipc.AvroRemoteException;
import org.apache.avro.generic.GenericArray;
+import org.apache.avro.specific.FixedSize;
import org.apache.avro.specific.SpecificExceptionBase;
import org.apache.avro.specific.SpecificRecordBase;
import org.apache.avro.specific.SpecificRecord;
Modified: gora/branches/GORA_174/gora-core/src/examples/java/org/apache/gora/examples/generated/WebPage.java
URL: http://svn.apache.org/viewvc/gora/branches/GORA_174/gora-core/src/examples/java/org/apache/gora/examples/generated/WebPage.java?rev=1455623&r1=1455622&r2=1455623&view=diff
==============================================================================
--- gora/branches/GORA_174/gora-core/src/examples/java/org/apache/gora/examples/generated/WebPage.java (original)
+++ gora/branches/GORA_174/gora-core/src/examples/java/org/apache/gora/examples/generated/WebPage.java Tue Mar 12 17:07:03 2013
@@ -1,3 +1,21 @@
+/**
+ *Licensed to the Apache Software Foundation (ASF) under one
+ *or more contributor license agreements. See the NOTICE file
+ *distributed with this work for additional information
+ *regarding copyright ownership. The ASF licenses this file
+ *to you under the Apache License, Version 2.0 (the"
+ *License"); you may not use this file except in compliance
+ *with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *Unless required by applicable law or agreed to in writing, software
+ *distributed under the License is distributed on an "AS IS" BASIS,
+ *WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *See the License for the specific language governing permissions and
+ *limitations under the License.
+ */
+
package org.apache.gora.examples.generated;
import java.nio.ByteBuffer;
@@ -10,6 +28,7 @@ import org.apache.avro.Protocol;
import org.apache.avro.util.Utf8;
import org.apache.avro.ipc.AvroRemoteException;
import org.apache.avro.generic.GenericArray;
+import org.apache.avro.specific.FixedSize;
import org.apache.avro.specific.SpecificExceptionBase;
import org.apache.avro.specific.SpecificRecordBase;
import org.apache.avro.specific.SpecificRecord;
@@ -22,7 +41,7 @@ import org.apache.gora.persistency.ListG
@SuppressWarnings("all")
public class WebPage extends PersistentBase {
- public static final Schema _SCHEMA = Schema.parse("{\"type\":\"record\",\"name\":\"WebPage\",\"namespace\":\"org.apache.gora.examples.generated\",\"fields\":[{\"name\":\"url\",\"type\":\"string\"},{\"name\":\"content\",\"type\":\"bytes\"},{\"name\":\"parsedContent\",\"type\":{\"type\":\"array\",\"items\":\"string\"}},{\"name\":\"outlinks\",\"type\":{\"type\":\"map\",\"values\":\"string\"}},{\"name\":\"metadata\",\"type\":{\"type\":\"record\",\"name\":\"Metadata\",\"fields\":[{\"name\":\"version\",\"type\":\"int\"},{\"name\":\"data\",\"type\":{\"type\":\"map\",\"values\":\"string\"}}]}}]}");
+ public static final Schema _SCHEMA = Schema.parse("{\"type\":\"record\",\"name\":\"WebPage\",\"namespace\":\"org.apache.gora.examples.generated\",\"fields\":[{\"name\":\"url\",\"type\":\"string\"},{\"name\":\"content\",\"type\":[\"null\",\"bytes\"]},{\"name\":\"parsedContent\",\"type\":{\"type\":\"array\",\"items\":\"string\"}},{\"name\":\"outlinks\",\"type\":{\"type\":\"map\",\"values\":\"string\"}},{\"name\":\"metadata\",\"type\":{\"type\":\"record\",\"name\":\"Metadata\",\"fields\":[{\"name\":\"version\",\"type\":\"int\"},{\"name\":\"data\",\"type\":{\"type\":\"map\",\"values\":\"string\"}}]}}]}");
public static enum Field {
URL(0,"url"),
CONTENT(1,"content"),
Modified: gora/branches/GORA_174/gora-core/src/examples/java/org/apache/gora/examples/mapreduce/WordCount.java
URL: http://svn.apache.org/viewvc/gora/branches/GORA_174/gora-core/src/examples/java/org/apache/gora/examples/mapreduce/WordCount.java?rev=1455623&r1=1455622&r2=1455623&view=diff
==============================================================================
--- gora/branches/GORA_174/gora-core/src/examples/java/org/apache/gora/examples/mapreduce/WordCount.java (original)
+++ gora/branches/GORA_174/gora-core/src/examples/java/org/apache/gora/examples/mapreduce/WordCount.java Tue Mar 12 17:07:03 2013
@@ -65,13 +65,15 @@ public class WordCount extends Configure
protected void map(String key, WebPage page, Context context)
throws IOException ,InterruptedException {
- //Get the content from a WebPage as obtained from the DataStore
- String content = new String(page.getContent().array());
-
- StringTokenizer itr = new StringTokenizer(content);
- while (itr.hasMoreTokens()) {
- word.set(itr.nextToken());
- context.write(word, one);
+ if (page.getContent() != null) {
+ // Get the content from a WebPage as obtained from the DataStore
+ String content = new String(page.getContent().array());
+
+ StringTokenizer itr = new StringTokenizer(content);
+ while (itr.hasMoreTokens()) {
+ word.set(itr.nextToken());
+ context.write(word, one);
+ }
}
};
}
Modified: gora/branches/GORA_174/gora-core/src/main/java/org/apache/gora/compiler/GoraCompiler.java
URL: http://svn.apache.org/viewvc/gora/branches/GORA_174/gora-core/src/main/java/org/apache/gora/compiler/GoraCompiler.java?rev=1455623&r1=1455622&r2=1455623&view=diff
==============================================================================
--- gora/branches/GORA_174/gora-core/src/main/java/org/apache/gora/compiler/GoraCompiler.java (original)
+++ gora/branches/GORA_174/gora-core/src/main/java/org/apache/gora/compiler/GoraCompiler.java Tue Mar 12 17:07:03 2013
@@ -374,6 +374,27 @@ public class GoraCompiler {
line(2, "getStateManager().setDirty(this, "+i+");");
line(2, "return "+field.name()+".remove(key);");
line(1, "}");
+ break;
+ case UNION:
+ fieldType = type(fieldSchema);
+ //Create get method: public <unbox(field.schema())> get<camelKey>()
+ line(1, "public "+unbox(field.schema())+" get" +camelKey+"() {");
+ line(2, "return ("+unbox(field.schema())+") get("+i+");");
+ line(1, "}");
+
+ //Create set methods: public void set<camelKey>(<subschema.fieldType> value)
+ for (Schema s : fieldSchema.getTypes()) {
+ if (s.getType().equals(Schema.Type.NULL)) continue ;
+ String unionFieldType = type(s);
+ line(1, "public void set"+camelKey+"("+unionFieldType+" value) {");
+ line(2, "put("+i+", value);");
+ line(1, "}");
+ }
+ break;
+ case NULL:
+ throw new RuntimeException("Unexpected NULL field: "+field);
+ default:
+ throw new RuntimeException("Unknown field: "+field);
}
i++;
}
Modified: gora/branches/GORA_174/gora-core/src/main/java/org/apache/gora/persistency/impl/PersistentBase.java
URL: http://svn.apache.org/viewvc/gora/branches/GORA_174/gora-core/src/main/java/org/apache/gora/persistency/impl/PersistentBase.java?rev=1455623&r1=1455622&r2=1455623&view=diff
==============================================================================
--- gora/branches/GORA_174/gora-core/src/main/java/org/apache/gora/persistency/impl/PersistentBase.java (original)
+++ gora/branches/GORA_174/gora-core/src/main/java/org/apache/gora/persistency/impl/PersistentBase.java Tue Mar 12 17:07:03 2013
@@ -24,6 +24,7 @@ import java.util.Map;
import org.apache.avro.Schema.Field;
import org.apache.avro.Schema.Type;
+import org.apache.avro.generic.GenericData;
import org.apache.avro.specific.SpecificRecord;
import org.apache.gora.avro.PersistentDatumReader;
import org.apache.gora.persistency.ListGenericArray;
@@ -253,12 +254,33 @@ public abstract class PersistentBase imp
return result;
}
+ /**
+ * Computes a (record's) field's hash code.
+ * @param i Index of the field in the actual
+ * @param field
+ * @return
+ */
private int getFieldHashCode(int i, Field field) {
Object o = get(i);
if(o == null)
return 0;
- if(field.schema().getType() == Type.BYTES) {
+ // XXX Union special case: in a field being union we have to check the
+ // inner schemas for Type.BYTES special case, but because it is not a
+ // field we check it this way. Too simple case to create another
+ // private method
+ boolean isUnionField = false ;
+ int unionIndex = -1 ;
+
+ if (field.schema().getType() == Type.UNION) {
+ isUnionField = true ;
+ unionIndex = GenericData.get().resolveUnion(field.schema(), o);
+ }
+
+ if(field.schema().getType() == Type.BYTES
+ || (isUnionField
+ && field.schema().getTypes().get(unionIndex).getType() == Type.BYTES)) {
+ // ByteBuffer.hashCode() depends on internal 'position' index, but we must ignore that.
return getByteBufferHashCode((ByteBuffer)o);
}
Modified: gora/branches/GORA_174/gora-core/src/test/java/org/apache/gora/mapreduce/MapReduceTestUtils.java
URL: http://svn.apache.org/viewvc/gora/branches/GORA_174/gora-core/src/test/java/org/apache/gora/mapreduce/MapReduceTestUtils.java?rev=1455623&r1=1455622&r2=1455623&view=diff
==============================================================================
--- gora/branches/GORA_174/gora-core/src/test/java/org/apache/gora/mapreduce/MapReduceTestUtils.java (original)
+++ gora/branches/GORA_174/gora-core/src/test/java/org/apache/gora/mapreduce/MapReduceTestUtils.java Tue Mar 12 17:07:03 2013
@@ -78,11 +78,13 @@ public class MapReduceTestUtils {
//assert results
HashMap<String, Integer> actualCounts = new HashMap<String, Integer>();
for(String content : WebPageDataCreator.CONTENTS) {
- for(String token:content.split(" ")) {
- Integer count = actualCounts.get(token);
- if(count == null)
- count = 0;
- actualCounts.put(token, ++count);
+ if (content != null) {
+ for(String token:content.split(" ")) {
+ Integer count = actualCounts.get(token);
+ if(count == null)
+ count = 0;
+ actualCounts.put(token, ++count);
+ }
}
}
for(Map.Entry<String, Integer> entry:actualCounts.entrySet()) {
Modified: gora/branches/GORA_174/gora-core/src/test/java/org/apache/gora/store/DataStoreTestUtil.java
URL: http://svn.apache.org/viewvc/gora/branches/GORA_174/gora-core/src/test/java/org/apache/gora/store/DataStoreTestUtil.java?rev=1455623&r1=1455622&r2=1455623&view=diff
==============================================================================
--- gora/branches/GORA_174/gora-core/src/test/java/org/apache/gora/store/DataStoreTestUtil.java (original)
+++ gora/branches/GORA_174/gora-core/src/test/java/org/apache/gora/store/DataStoreTestUtil.java Tue Mar 12 17:07:03 2013
@@ -336,27 +336,35 @@ public class DataStoreTestUtil {
Assert.assertNotNull(page);
Assert.assertEquals(URLS[i], page.getUrl().toString());
- Assert.assertTrue("content error:" + new String( toByteArray(page.getContent()) ) +
+ // 'content' is optional
+ if (page.getContent() != null) {
+ Assert.assertTrue("content error:" + new String( toByteArray(page.getContent()) ) +
" actual=" + CONTENTS[i] + " i=" + i
- , Arrays.equals( toByteArray(page.getContent() )
+ , Arrays.equals( toByteArray(page.getContent() )
, CONTENTS[i].getBytes()));
-
- GenericArray<Utf8> parsedContent = page.getParsedContent();
- Assert.assertNotNull(parsedContent);
- Assert.assertTrue(parsedContent.size() > 0);
-
- int j=0;
- String[] tokens = CONTENTS[i].split(" ");
- for(Utf8 token : parsedContent) {
- Assert.assertEquals(tokens[j++], token.toString());
+ GenericArray<Utf8> parsedContent = page.getParsedContent();
+ Assert.assertNotNull(parsedContent);
+ Assert.assertTrue(parsedContent.size() > 0);
+
+ int j=0;
+ String[] tokens = CONTENTS[i].split(" ");
+ for(Utf8 token : parsedContent) {
+ Assert.assertEquals(tokens[j++], token.toString());
+ }
+ } else {
+ // when page.getContent() is null
+ Assert.assertTrue(CONTENTS[i] == null) ;
+ GenericArray<Utf8> parsedContent = page.getParsedContent();
+ Assert.assertNotNull(parsedContent);
+ Assert.assertTrue(parsedContent.size() == 0);
}
if(LINKS[i].length > 0) {
Assert.assertNotNull(page.getOutlinks());
Assert.assertTrue(page.getOutlinks().size() > 0);
- for(j=0; j<LINKS[i].length; j++) {
- Assert.assertEquals(ANCHORS[i][j],
- page.getFromOutlinks(new Utf8(URLS[LINKS[i][j]])).toString());
+ for(int k=0; k<LINKS[i].length; k++) {
+ Assert.assertEquals(ANCHORS[i][k],
+ page.getFromOutlinks(new Utf8(URLS[LINKS[i][k]])).toString());
}
} else {
Assert.assertTrue(page.getOutlinks() == null || page.getOutlinks().isEmpty());