You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@jena.apache.org by rv...@apache.org on 2014/04/08 15:48:18 UTC
svn commit: r1585725 [3/4] - in /jena/Experimental/hadoop-rdf:
hadoop-rdf-mapreduce/src/main/java/com/
hadoop-rdf-mapreduce/src/main/java/org/
hadoop-rdf-mapreduce/src/main/java/org/apache/
hadoop-rdf-mapreduce/src/main/java/org/apache/jena/ hadoop-rdf...
Added: jena/Experimental/hadoop-rdf/hadoop-rdf-mapreduce/src/main/java/org/apache/jena/hadoop/rdf/mapreduce/transform/TriplesToQuadsConstantGraphMapper.java
URL: http://svn.apache.org/viewvc/jena/Experimental/hadoop-rdf/hadoop-rdf-mapreduce/src/main/java/org/apache/jena/hadoop/rdf/mapreduce/transform/TriplesToQuadsConstantGraphMapper.java?rev=1585725&view=auto
==============================================================================
--- jena/Experimental/hadoop-rdf/hadoop-rdf-mapreduce/src/main/java/org/apache/jena/hadoop/rdf/mapreduce/transform/TriplesToQuadsConstantGraphMapper.java (added)
+++ jena/Experimental/hadoop-rdf/hadoop-rdf-mapreduce/src/main/java/org/apache/jena/hadoop/rdf/mapreduce/transform/TriplesToQuadsConstantGraphMapper.java Tue Apr 8 13:48:16 2014
@@ -0,0 +1,75 @@
+/*
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package org.apache.jena.hadoop.rdf.mapreduce.transform;
+
+import java.io.IOException;
+
+import com.hp.hpl.jena.graph.Node;
+import com.hp.hpl.jena.graph.Triple;
+import com.hp.hpl.jena.sparql.core.Quad;
+
+/**
+ * A mapper which converts triples to quads where all triples are placed in the
+ * same graph
+ *
+ * @author rvesse
+ *
+ * @param <TKey>
+ * Key type
+ */
+public class TriplesToQuadsConstantGraphMapper<TKey> extends AbstractTriplesToQuadsMapper<TKey> {
+
+ private Node graphNode;
+
+ @Override
+ protected void setup(Context context) throws IOException, InterruptedException {
+ super.setup(context);
+ this.graphNode = this.getGraphNode();
+ }
+
+ /**
+ * Gets the graph node that will be used for all quads, this will be called
+ * once and only once during the
+ * {@link #setup(org.apache.hadoop.mapreduce.Mapper.Context)} method and the
+ * value returned cached for use throughout the lifetime of this mapper.
+ * <p>
+ * This implementation always used the default graph as the graph for
+ * generated quads. You can override this method in your own derived
+ * implementation to put triples into a different graph than the default
+ * graph.
+ * </p>
+ * <p>
+ * If instead you wanted to select different graphs for each triple you
+ * should extend {@link AbstractTriplesToQuadsMapper} instead and override
+ * the {@link #selectGraph(Triple)} method which is sealed in this
+ * implementation.
+ * </p>
+ *
+ * @return
+ */
+ protected Node getGraphNode() {
+ return Quad.defaultGraphNodeGenerated;
+ }
+
+ @Override
+ protected final Node selectGraph(Triple triple) {
+ return this.graphNode;
+ }
+
+}
Added: jena/Experimental/hadoop-rdf/hadoop-rdf-mapreduce/src/test/java/org/apache/jena/hadoop/rdf/mapreduce/AbstractMapReduceTests.java
URL: http://svn.apache.org/viewvc/jena/Experimental/hadoop-rdf/hadoop-rdf-mapreduce/src/test/java/org/apache/jena/hadoop/rdf/mapreduce/AbstractMapReduceTests.java?rev=1585725&view=auto
==============================================================================
--- jena/Experimental/hadoop-rdf/hadoop-rdf-mapreduce/src/test/java/org/apache/jena/hadoop/rdf/mapreduce/AbstractMapReduceTests.java (added)
+++ jena/Experimental/hadoop-rdf/hadoop-rdf-mapreduce/src/test/java/org/apache/jena/hadoop/rdf/mapreduce/AbstractMapReduceTests.java Tue Apr 8 13:48:16 2014
@@ -0,0 +1,69 @@
+/*
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package org.apache.jena.hadoop.rdf.mapreduce;
+
+import org.apache.hadoop.mapreduce.Mapper;
+import org.apache.hadoop.mapreduce.Reducer;
+import org.apache.hadoop.mrunit.mapreduce.MapReduceDriver;
+
+/**
+ * Abstract tests for mappers
+ *
+ * @author rvesse
+ * @param <TKey>
+ * Mapper input key type
+ * @param <TValue>
+ * Mapper input value type
+ * @param <TIntermediateKey>
+ * Mapper output/Reducer input key type
+ * @param <TIntermediateValue>
+ * Mapper output/Reducer input value type
+ * @param <TReducedKey>
+ * Reducer output key type
+ * @param <TReducedValue>
+ * Reducer output value type
+ *
+ *
+ */
+public abstract class AbstractMapReduceTests<TKey, TValue, TIntermediateKey, TIntermediateValue, TReducedKey, TReducedValue> {
+
+ /**
+ * Gets the mapper instance to test
+ *
+ * @return Mapper instance
+ */
+ protected abstract Mapper<TKey, TValue, TIntermediateKey, TIntermediateValue> getMapperInstance();
+
+ /**
+ * Gets the reducer instance to test
+ *
+ * @return Reducer instance
+ */
+ protected abstract Reducer<TIntermediateKey, TIntermediateValue, TReducedKey, TReducedValue> getReducerInstance();
+
+ /**
+ * Gets a map reduce driver that can be used to create a test case
+ *
+ * @return Map reduce driver
+ */
+ protected MapReduceDriver<TKey, TValue, TIntermediateKey, TIntermediateValue, TReducedKey, TReducedValue> getMapReduceDriver() {
+ return new MapReduceDriver<TKey, TValue, TIntermediateKey, TIntermediateValue, TReducedKey, TReducedValue>(
+ this.getMapperInstance(), this.getReducerInstance());
+ }
+}
Added: jena/Experimental/hadoop-rdf/hadoop-rdf-mapreduce/src/test/java/org/apache/jena/hadoop/rdf/mapreduce/AbstractMapperTests.java
URL: http://svn.apache.org/viewvc/jena/Experimental/hadoop-rdf/hadoop-rdf-mapreduce/src/test/java/org/apache/jena/hadoop/rdf/mapreduce/AbstractMapperTests.java?rev=1585725&view=auto
==============================================================================
--- jena/Experimental/hadoop-rdf/hadoop-rdf-mapreduce/src/test/java/org/apache/jena/hadoop/rdf/mapreduce/AbstractMapperTests.java (added)
+++ jena/Experimental/hadoop-rdf/hadoop-rdf-mapreduce/src/test/java/org/apache/jena/hadoop/rdf/mapreduce/AbstractMapperTests.java Tue Apr 8 13:48:16 2014
@@ -0,0 +1,65 @@
+/*
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package org.apache.jena.hadoop.rdf.mapreduce;
+
+import org.apache.hadoop.mapreduce.Mapper;
+import org.apache.hadoop.mrunit.mapreduce.MapDriver;
+
+/**
+ * Abstract tests for mappers
+ *
+ * @author rvesse
+ * @param <TKeyIn>
+ * Input key type
+ * @param <TValueIn>
+ * Input value type
+ * @param <TKeyOut>
+ * Output key type
+ * @param <TValueOut>
+ * Output value type
+ *
+ */
+public abstract class AbstractMapperTests<TKeyIn, TValueIn, TKeyOut, TValueOut> {
+
+ /**
+ * Gets the mapper instance to test
+ *
+ * @return Mapper instance
+ */
+ protected abstract Mapper<TKeyIn, TValueIn, TKeyOut, TValueOut> getInstance();
+
+ /**
+ * Gets a map driver that can be used to create a test case
+ *
+ * @return Map driver
+ */
+ protected MapDriver<TKeyIn, TValueIn, TKeyOut, TValueOut> getMapDriver() {
+ MapDriver<TKeyIn, TValueIn, TKeyOut, TValueOut> driver = new MapDriver<TKeyIn, TValueIn, TKeyOut, TValueOut>(this.getInstance());
+ this.configureDriver(driver);
+ return driver;
+ }
+
+ /**
+ * Method that may be overridden by test harnesses which need to configure the driver in more detail e.g. add configuration keys
+ * @param driver Driver
+ */
+ protected void configureDriver(MapDriver<TKeyIn, TValueIn, TKeyOut, TValueOut> driver) {
+ // Does nothing
+ }
+}
Added: jena/Experimental/hadoop-rdf/hadoop-rdf-mapreduce/src/test/java/org/apache/jena/hadoop/rdf/mapreduce/characteristics/AbstractCharacteristicSetGeneratingReducerTests.java
URL: http://svn.apache.org/viewvc/jena/Experimental/hadoop-rdf/hadoop-rdf-mapreduce/src/test/java/org/apache/jena/hadoop/rdf/mapreduce/characteristics/AbstractCharacteristicSetGeneratingReducerTests.java?rev=1585725&view=auto
==============================================================================
--- jena/Experimental/hadoop-rdf/hadoop-rdf-mapreduce/src/test/java/org/apache/jena/hadoop/rdf/mapreduce/characteristics/AbstractCharacteristicSetGeneratingReducerTests.java (added)
+++ jena/Experimental/hadoop-rdf/hadoop-rdf-mapreduce/src/test/java/org/apache/jena/hadoop/rdf/mapreduce/characteristics/AbstractCharacteristicSetGeneratingReducerTests.java Tue Apr 8 13:48:16 2014
@@ -0,0 +1,185 @@
+/*
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package org.apache.jena.hadoop.rdf.mapreduce.characteristics;
+
+import java.io.IOException;
+
+import org.apache.hadoop.io.LongWritable;
+import org.apache.hadoop.io.NullWritable;
+import org.apache.hadoop.mrunit.mapreduce.MapReduceDriver;
+import org.apache.jena.hadoop.rdf.mapreduce.AbstractMapReduceTests;
+import org.apache.jena.hadoop.rdf.mapreduce.characteristics.AbstractCharacteristicSetGeneratingReducer;
+import org.apache.jena.hadoop.rdf.types.AbstractNodeTupleWritable;
+import org.apache.jena.hadoop.rdf.types.CharacteristicSetWritable;
+import org.apache.jena.hadoop.rdf.types.CharacteristicWritable;
+import org.apache.jena.hadoop.rdf.types.NodeWritable;
+import org.junit.Test;
+
+import com.hp.hpl.jena.graph.NodeFactory;
+
+/**
+ * Abstract tests for the {@link AbstractCharacteristicSetGeneratingReducer}
+ *
+ * @author rvesse
+ *
+ * @param <TValue>
+ * @param <T>
+ */
+public abstract class AbstractCharacteristicSetGeneratingReducerTests<TValue, T extends AbstractNodeTupleWritable<TValue>>
+ extends AbstractMapReduceTests<LongWritable, T, NodeWritable, T, CharacteristicSetWritable, NullWritable> {
+
+ /**
+ * Create a tuple
+ *
+ * @param i
+ * Key to use in creating the subject
+ * @param predicateUri
+ * Predicate URI string
+ * @return Tuple
+ */
+ protected abstract T createTuple(int i, String predicateUri);
+
+ /**
+ * Creates a set consisting of the given predicates
+ *
+ * @param predicates
+ * Predicates
+ * @return Set
+ */
+ protected CharacteristicSetWritable createSet(MapReduceDriver<LongWritable, T, NodeWritable, T, CharacteristicSetWritable, NullWritable> driver, int occurrences, String... predicates) {
+ CharacteristicSetWritable set = new CharacteristicSetWritable();
+ for (String predicateUri : predicates) {
+ set.add(new CharacteristicWritable(NodeFactory.createURI(predicateUri)));
+ }
+ for (int i = 1; i <= occurrences; i++) {
+ driver.addOutput(set, NullWritable.get());
+ }
+ return set;
+ }
+
+ /**
+ * Test basic characteristic set computation
+ *
+ * @throws IOException
+ */
+ @Test
+ public void characteristic_set_generating_reducer_01() throws IOException {
+ MapReduceDriver<LongWritable, T, NodeWritable, T, CharacteristicSetWritable, NullWritable> driver = this
+ .getMapReduceDriver();
+ T tuple = this.createTuple(1, "http://predicate");
+ driver.addInput(new LongWritable(1), tuple);
+
+ this.createSet(driver, 1, "http://predicate");
+
+ driver.runTest(false);
+ }
+
+ /**
+ * Test basic characteristic set computation
+ *
+ * @throws IOException
+ */
+ @Test
+ public void characteristic_set_generating_reducer_02() throws IOException {
+ MapReduceDriver<LongWritable, T, NodeWritable, T, CharacteristicSetWritable, NullWritable> driver = this
+ .getMapReduceDriver();
+ T tuple = this.createTuple(1, "http://predicate");
+ driver.addInput(new LongWritable(1), tuple);
+ driver.addInput(new LongWritable(1), tuple);
+
+ this.createSet(driver, 1, "http://predicate");
+
+ driver.runTest(false);
+ }
+
+ /**
+ * Test basic characteristic set computation
+ *
+ * @throws IOException
+ */
+ @Test
+ public void characteristic_set_generating_reducer_03() throws IOException {
+ MapReduceDriver<LongWritable, T, NodeWritable, T, CharacteristicSetWritable, NullWritable> driver = this
+ .getMapReduceDriver();
+ T tuple = this.createTuple(1, "http://predicate");
+ driver.addInput(new LongWritable(1), tuple);
+ tuple = this.createTuple(2, "http://predicate");
+ driver.addInput(new LongWritable(2), tuple);
+
+ this.createSet(driver, 2, "http://predicate");
+
+ driver.runTest(false);
+ }
+
+ /**
+ * Test basic characteristic set computation
+ *
+ * @throws IOException
+ */
+ @Test
+ public void characteristic_set_generating_reducer_04() throws IOException {
+ MapReduceDriver<LongWritable, T, NodeWritable, T, CharacteristicSetWritable, NullWritable> driver = this
+ .getMapReduceDriver();
+ T tuple = this.createTuple(1, "http://predicate");
+ driver.addInput(new LongWritable(1), tuple);
+ tuple = this.createTuple(1, "http://other");
+ driver.addInput(new LongWritable(1), tuple);
+
+ // Single entry sets
+ this.createSet(driver, 1, "http://predicate");
+ this.createSet(driver, 1, "http://other");
+
+ // Two entry sets
+ this.createSet(driver, 1, "http://predicate", "http://other");
+
+ driver.runTest(false);
+ }
+
+ /**
+ * Test basic characteristic set computation
+ *
+ * @throws IOException
+ */
+ @Test
+ public void characteristic_set_generating_reducer_05() throws IOException {
+ MapReduceDriver<LongWritable, T, NodeWritable, T, CharacteristicSetWritable, NullWritable> driver = this
+ .getMapReduceDriver();
+ T tuple = this.createTuple(1, "http://predicate");
+ driver.addInput(new LongWritable(1), tuple);
+ tuple = this.createTuple(1, "http://other");
+ driver.addInput(new LongWritable(2), tuple);
+ tuple = this.createTuple(1, "http://third");
+ driver.addInput(new LongWritable(3), tuple);
+
+ // Single entry sets
+ this.createSet(driver, 1, "http://predicate");
+ this.createSet(driver, 1, "http://other");
+ this.createSet(driver, 1, "http://third");
+
+ // Two entry sets
+ this.createSet(driver, 1, "http://predicate", "http://other");
+ this.createSet(driver, 1, "http://predicate", "http://third");
+ this.createSet(driver, 1, "http://other", "http://third");
+
+ // Three entry sets
+ this.createSet(driver, 1, "http://predicate", "http://other", "http://third");
+
+ driver.runTest(false);
+ }
+}
Added: jena/Experimental/hadoop-rdf/hadoop-rdf-mapreduce/src/test/java/org/apache/jena/hadoop/rdf/mapreduce/characteristics/CharacteristicSetReducerTest.java
URL: http://svn.apache.org/viewvc/jena/Experimental/hadoop-rdf/hadoop-rdf-mapreduce/src/test/java/org/apache/jena/hadoop/rdf/mapreduce/characteristics/CharacteristicSetReducerTest.java?rev=1585725&view=auto
==============================================================================
--- jena/Experimental/hadoop-rdf/hadoop-rdf-mapreduce/src/test/java/org/apache/jena/hadoop/rdf/mapreduce/characteristics/CharacteristicSetReducerTest.java (added)
+++ jena/Experimental/hadoop-rdf/hadoop-rdf-mapreduce/src/test/java/org/apache/jena/hadoop/rdf/mapreduce/characteristics/CharacteristicSetReducerTest.java Tue Apr 8 13:48:16 2014
@@ -0,0 +1,192 @@
+/*
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package org.apache.jena.hadoop.rdf.mapreduce.characteristics;
+
+import java.io.IOException;
+import java.util.List;
+
+import org.apache.hadoop.io.NullWritable;
+import org.apache.hadoop.mapreduce.Mapper;
+import org.apache.hadoop.mapreduce.Reducer;
+import org.apache.hadoop.mrunit.mapreduce.MapReduceDriver;
+import org.apache.hadoop.mrunit.types.Pair;
+import org.apache.jena.hadoop.rdf.mapreduce.AbstractMapReduceTests;
+import org.apache.jena.hadoop.rdf.mapreduce.characteristics.CharacteristicSetReducer;
+import org.apache.jena.hadoop.rdf.types.CharacteristicSetWritable;
+import org.apache.jena.hadoop.rdf.types.CharacteristicWritable;
+import org.junit.Assert;
+import org.junit.Test;
+
+import com.hp.hpl.jena.graph.NodeFactory;
+
+/**
+ * Abstract tests for the {@link CharacteristicSetReducer}
+ *
+ * @author rvesse
+ */
+public class CharacteristicSetReducerTest
+ extends
+ AbstractMapReduceTests<CharacteristicSetWritable, CharacteristicSetWritable, CharacteristicSetWritable, CharacteristicSetWritable, CharacteristicSetWritable, NullWritable> {
+
+ @Override
+ protected final Mapper<CharacteristicSetWritable, CharacteristicSetWritable, CharacteristicSetWritable, CharacteristicSetWritable> getMapperInstance() {
+ // Identity mapper
+ return new Mapper<CharacteristicSetWritable, CharacteristicSetWritable, CharacteristicSetWritable, CharacteristicSetWritable>();
+ }
+
+ @Override
+ protected final Reducer<CharacteristicSetWritable, CharacteristicSetWritable, CharacteristicSetWritable, NullWritable> getReducerInstance() {
+ return new CharacteristicSetReducer();
+ }
+
+ /**
+ * Creates a set consisting of the given predicates
+ *
+ * @param predicates
+ * Predicates
+ * @return Set
+ */
+ protected CharacteristicSetWritable createSet(
+ MapReduceDriver<CharacteristicSetWritable, CharacteristicSetWritable, CharacteristicSetWritable, CharacteristicSetWritable, CharacteristicSetWritable, NullWritable> driver,
+ int inputOccurrences, int outputOccurrences, String... predicates) {
+ CharacteristicSetWritable set = new CharacteristicSetWritable();
+ for (String predicateUri : predicates) {
+ set.add(new CharacteristicWritable(NodeFactory.createURI(predicateUri)));
+ }
+ for (int i = 1; i <= inputOccurrences; i++) {
+ driver.addInput(set, set);
+ }
+ for (int i = 1; i <= outputOccurrences; i++) {
+ driver.addOutput(set, NullWritable.get());
+ }
+ return set;
+ }
+
+ /**
+ * Test characteristic set reduction
+ *
+ * @throws IOException
+ */
+ @Test
+ public void characteristic_set_reducer_01() throws IOException {
+ MapReduceDriver<CharacteristicSetWritable, CharacteristicSetWritable, CharacteristicSetWritable, CharacteristicSetWritable, CharacteristicSetWritable, NullWritable> driver = this
+ .getMapReduceDriver();
+
+ this.createSet(driver, 1, 1, "http://predicate");
+
+ driver.runTest(false);
+ }
+
+ /**
+ * Test characteristic set reduction
+ *
+ * @throws IOException
+ */
+ @Test
+ public void characteristic_set_reducer_02() throws IOException {
+ MapReduceDriver<CharacteristicSetWritable, CharacteristicSetWritable, CharacteristicSetWritable, CharacteristicSetWritable, CharacteristicSetWritable, NullWritable> driver = this
+ .getMapReduceDriver();
+
+ this.createSet(driver, 2, 1, "http://predicate");
+
+ driver.runTest(false);
+
+ List<Pair<CharacteristicSetWritable, NullWritable>> results = driver.run();
+ CharacteristicSetWritable cw = results.get(0).getFirst();
+ Assert.assertEquals(2, cw.getCount().get());
+ }
+
+ /**
+ * Test characteristic set reduction
+ *
+ * @throws IOException
+ */
+ @Test
+ public void characteristic_set_reducer_03() throws IOException {
+ MapReduceDriver<CharacteristicSetWritable, CharacteristicSetWritable, CharacteristicSetWritable, CharacteristicSetWritable, CharacteristicSetWritable, NullWritable> driver = this
+ .getMapReduceDriver();
+
+ this.createSet(driver, 1, 1, "http://predicate");
+ this.createSet(driver, 1, 1, "http://other");
+
+ driver.runTest(false);
+ }
+
+ /**
+ * Test characteristic set reduction
+ *
+ * @throws IOException
+ */
+ @Test
+ public void characteristic_set_reducer_04() throws IOException {
+ MapReduceDriver<CharacteristicSetWritable, CharacteristicSetWritable, CharacteristicSetWritable, CharacteristicSetWritable, CharacteristicSetWritable, NullWritable> driver = this
+ .getMapReduceDriver();
+
+ this.createSet(driver, 2, 1, "http://predicate");
+ this.createSet(driver, 1, 1, "http://other");
+
+ driver.runTest(false);
+
+ List<Pair<CharacteristicSetWritable, NullWritable>> results = driver.run();
+ for (Pair<CharacteristicSetWritable, NullWritable> pair : results) {
+ CharacteristicSetWritable cw = pair.getFirst();
+ boolean expectTwo = cw.getCharacteristics().next().getNode().get().hasURI("http://predicate");
+ Assert.assertEquals(expectTwo ? 2 : 1, cw.getCount().get());
+ }
+ }
+
+ /**
+ * Test characteristic set reduction
+ *
+ * @throws IOException
+ */
+ @Test
+ public void characteristic_set_reducer_05() throws IOException {
+ MapReduceDriver<CharacteristicSetWritable, CharacteristicSetWritable, CharacteristicSetWritable, CharacteristicSetWritable, CharacteristicSetWritable, NullWritable> driver = this
+ .getMapReduceDriver();
+
+ this.createSet(driver, 1, 1, "http://predicate", "http://other");
+ this.createSet(driver, 1, 1, "http://other");
+
+ driver.runTest(false);
+ }
+
+ /**
+ * Test characteristic set reduction
+ *
+ * @throws IOException
+ */
+ @Test
+ public void characteristic_set_reducer_06() throws IOException {
+ MapReduceDriver<CharacteristicSetWritable, CharacteristicSetWritable, CharacteristicSetWritable, CharacteristicSetWritable, CharacteristicSetWritable, NullWritable> driver = this
+ .getMapReduceDriver();
+
+ this.createSet(driver, 2, 1, "http://predicate", "http://other");
+ this.createSet(driver, 1, 1, "http://other");
+
+ driver.runTest(false);
+
+ List<Pair<CharacteristicSetWritable, NullWritable>> results = driver.run();
+ for (Pair<CharacteristicSetWritable, NullWritable> pair : results) {
+ CharacteristicSetWritable cw = pair.getFirst();
+ boolean expectTwo = cw.hasCharacteristic("http://predicate");
+ Assert.assertEquals(expectTwo ? 2 : 1, cw.getCount().get());
+ }
+ }
+}
Added: jena/Experimental/hadoop-rdf/hadoop-rdf-mapreduce/src/test/java/org/apache/jena/hadoop/rdf/mapreduce/characteristics/TripleCharacteristicSetGeneratingReducerTest.java
URL: http://svn.apache.org/viewvc/jena/Experimental/hadoop-rdf/hadoop-rdf-mapreduce/src/test/java/org/apache/jena/hadoop/rdf/mapreduce/characteristics/TripleCharacteristicSetGeneratingReducerTest.java?rev=1585725&view=auto
==============================================================================
--- jena/Experimental/hadoop-rdf/hadoop-rdf-mapreduce/src/test/java/org/apache/jena/hadoop/rdf/mapreduce/characteristics/TripleCharacteristicSetGeneratingReducerTest.java (added)
+++ jena/Experimental/hadoop-rdf/hadoop-rdf-mapreduce/src/test/java/org/apache/jena/hadoop/rdf/mapreduce/characteristics/TripleCharacteristicSetGeneratingReducerTest.java Tue Apr 8 13:48:16 2014
@@ -0,0 +1,59 @@
+/*
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package org.apache.jena.hadoop.rdf.mapreduce.characteristics;
+
+import org.apache.hadoop.io.LongWritable;
+import org.apache.hadoop.io.NullWritable;
+import org.apache.hadoop.mapreduce.Mapper;
+import org.apache.hadoop.mapreduce.Reducer;
+import org.apache.jena.hadoop.rdf.mapreduce.characteristics.TripleCharacteristicSetGeneratingReducer;
+import org.apache.jena.hadoop.rdf.mapreduce.group.TripleGroupBySubjectMapper;
+import org.apache.jena.hadoop.rdf.types.CharacteristicSetWritable;
+import org.apache.jena.hadoop.rdf.types.NodeWritable;
+import org.apache.jena.hadoop.rdf.types.TripleWritable;
+
+import com.hp.hpl.jena.datatypes.xsd.XSDDatatype;
+import com.hp.hpl.jena.graph.NodeFactory;
+import com.hp.hpl.jena.graph.Triple;
+
+/**
+ * Tests for the {@link TripleCharacteristicSetGeneratingReducer}
+ *
+ * @author rvesse
+ *
+ */
+public class TripleCharacteristicSetGeneratingReducerTest extends AbstractCharacteristicSetGeneratingReducerTests<Triple, TripleWritable> {
+
+ @Override
+ protected Mapper<LongWritable, TripleWritable, NodeWritable, TripleWritable> getMapperInstance() {
+ return new TripleGroupBySubjectMapper<LongWritable>();
+ }
+
+ @Override
+ protected Reducer<NodeWritable, TripleWritable, CharacteristicSetWritable, NullWritable> getReducerInstance() {
+ return new TripleCharacteristicSetGeneratingReducer();
+ }
+
+ @Override
+ protected TripleWritable createTuple(int i, String predicateUri) {
+ return new TripleWritable(new Triple(NodeFactory.createURI("http://subjects/" + i), NodeFactory.createURI(predicateUri),
+ NodeFactory.createLiteral(Integer.toString(i), XSDDatatype.XSDinteger)));
+ }
+
+}
Added: jena/Experimental/hadoop-rdf/hadoop-rdf-mapreduce/src/test/java/org/apache/jena/hadoop/rdf/mapreduce/count/AbstractNodeTupleNodeCountReducedTests.java
URL: http://svn.apache.org/viewvc/jena/Experimental/hadoop-rdf/hadoop-rdf-mapreduce/src/test/java/org/apache/jena/hadoop/rdf/mapreduce/count/AbstractNodeTupleNodeCountReducedTests.java?rev=1585725&view=auto
==============================================================================
--- jena/Experimental/hadoop-rdf/hadoop-rdf-mapreduce/src/test/java/org/apache/jena/hadoop/rdf/mapreduce/count/AbstractNodeTupleNodeCountReducedTests.java (added)
+++ jena/Experimental/hadoop-rdf/hadoop-rdf-mapreduce/src/test/java/org/apache/jena/hadoop/rdf/mapreduce/count/AbstractNodeTupleNodeCountReducedTests.java Tue Apr 8 13:48:16 2014
@@ -0,0 +1,149 @@
+/*
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package org.apache.jena.hadoop.rdf.mapreduce.count;
+
+import java.io.IOException;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.Map.Entry;
+
+import org.apache.hadoop.io.LongWritable;
+import org.apache.hadoop.mrunit.mapreduce.MapReduceDriver;
+import org.apache.jena.hadoop.rdf.mapreduce.AbstractMapReduceTests;
+import org.apache.jena.hadoop.rdf.mapreduce.count.AbstractNodeTupleNodeCountMapper;
+import org.apache.jena.hadoop.rdf.types.AbstractNodeTupleWritable;
+import org.apache.jena.hadoop.rdf.types.NodeWritable;
+import org.junit.Test;
+
+
+/**
+ * Abstract tests for mappers derived from
+ * {@link AbstractNodeTupleNodeCountMapper}
+ *
+ * @author rvesse
+ *
+ * @param <TValue>
+ * Tuple type
+ * @param <T>
+ * Writable tuple type
+ */
+public abstract class AbstractNodeTupleNodeCountReducedTests<TValue, T extends AbstractNodeTupleWritable<TValue>> extends
+ AbstractMapReduceTests<LongWritable, T, NodeWritable, LongWritable, NodeWritable, LongWritable> {
+
+ /**
+ * Generates tuples for the tests
+ *
+ * @param driver
+ * Driver
+ * @param num
+ * Number of tuples to generate
+ */
+ protected void generateData(MapReduceDriver<LongWritable, T, NodeWritable, LongWritable, NodeWritable, LongWritable> driver, int num) {
+ Map<NodeWritable, Long> counts = new HashMap<NodeWritable, Long>();
+ for (int i = 0; i < num; i++) {
+ LongWritable key = new LongWritable(i);
+ T value = this.createValue(i);
+ NodeWritable[] nodes = this.getNodes(value);
+
+ driver.addInput(key, value);
+ for (NodeWritable n : nodes) {
+ if (counts.containsKey(n)) {
+ counts.put(n, counts.get(n) + 1);
+ } else {
+ counts.put(n, 1l);
+ }
+ }
+ }
+
+ for (Entry<NodeWritable, Long> kvp : counts.entrySet()) {
+ driver.addOutput(kvp.getKey(), new LongWritable(kvp.getValue()));
+ }
+ }
+
+ /**
+ * Creates a tuple value
+ *
+ * @param i
+ * Index
+ * @return Tuple value
+ */
+ protected abstract T createValue(int i);
+
+ /**
+ * Splits the tuple value into its constituent nodes
+ *
+ * @param tuple
+ * Tuple value
+ * @return Nodes
+ */
+ protected abstract NodeWritable[] getNodes(T tuple);
+
+ /**
+ * Runs a node count test
+ *
+ * @param num
+ * Number of tuples to generate
+ * @throws IOException
+ */
+ protected void testNodeCount(int num) throws IOException {
+ MapReduceDriver<LongWritable, T, NodeWritable, LongWritable, NodeWritable, LongWritable> driver = this.getMapReduceDriver();
+ this.generateData(driver, num);
+ driver.runTest(false);
+ }
+
+ /**
+ * Tests node counting
+ *
+ * @throws IOException
+ */
+ @Test
+ public void node_count_01() throws IOException {
+ this.testNodeCount(1);
+ }
+
+ /**
+ * Tests node counting
+ *
+ * @throws IOException
+ */
+ @Test
+ public void node_count_02() throws IOException {
+ this.testNodeCount(100);
+ }
+
+ /**
+ * Tests node counting
+ *
+ * @throws IOException
+ */
+ @Test
+ public void node_count_03() throws IOException {
+ this.testNodeCount(1000);
+ }
+
+ /**
+ * Tests node counting
+ *
+ * @throws IOException
+ */
+ @Test
+ public void node_count_04() throws IOException {
+ this.testNodeCount(2500);
+ }
+}
Added: jena/Experimental/hadoop-rdf/hadoop-rdf-mapreduce/src/test/java/org/apache/jena/hadoop/rdf/mapreduce/count/AbstractNodeTupleNodeCountTests.java
URL: http://svn.apache.org/viewvc/jena/Experimental/hadoop-rdf/hadoop-rdf-mapreduce/src/test/java/org/apache/jena/hadoop/rdf/mapreduce/count/AbstractNodeTupleNodeCountTests.java?rev=1585725&view=auto
==============================================================================
--- jena/Experimental/hadoop-rdf/hadoop-rdf-mapreduce/src/test/java/org/apache/jena/hadoop/rdf/mapreduce/count/AbstractNodeTupleNodeCountTests.java (added)
+++ jena/Experimental/hadoop-rdf/hadoop-rdf-mapreduce/src/test/java/org/apache/jena/hadoop/rdf/mapreduce/count/AbstractNodeTupleNodeCountTests.java Tue Apr 8 13:48:16 2014
@@ -0,0 +1,138 @@
+/*
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package org.apache.jena.hadoop.rdf.mapreduce.count;
+
+import java.io.IOException;
+
+import org.apache.hadoop.io.LongWritable;
+import org.apache.hadoop.mrunit.mapreduce.MapDriver;
+import org.apache.jena.hadoop.rdf.mapreduce.AbstractMapperTests;
+import org.apache.jena.hadoop.rdf.mapreduce.count.AbstractNodeTupleNodeCountMapper;
+import org.apache.jena.hadoop.rdf.types.AbstractNodeTupleWritable;
+import org.apache.jena.hadoop.rdf.types.NodeWritable;
+import org.junit.Test;
+
+
+/**
+ * Abstract tests for mappers derived from
+ * {@link AbstractNodeTupleNodeCountMapper}
+ *
+ * @author rvesse
+ *
+ * @param <TValue>
+ * Tuple type
+ * @param <T>
+ * Writable tuple type
+ */
+public abstract class AbstractNodeTupleNodeCountTests<TValue, T extends AbstractNodeTupleWritable<TValue>> extends
+ AbstractMapperTests<LongWritable, T, NodeWritable, LongWritable> {
+
+ /**
+ * Generates tuples for the tests
+ *
+ * @param driver
+ * Driver
+ * @param num
+ * Number of tuples to generate
+ */
+ protected void generateData(MapDriver<LongWritable, T, NodeWritable, LongWritable> driver, int num) {
+ LongWritable expectedCount = new LongWritable(1);
+ for (int i = 0; i < num; i++) {
+ LongWritable key = new LongWritable(i);
+ T value = this.createValue(i);
+ NodeWritable[] nodes = this.getNodes(value);
+
+ driver.addInput(key, value);
+ for (NodeWritable n : nodes) {
+ driver.addOutput(n, expectedCount);
+ }
+ }
+ }
+
+ /**
+ * Creates a tuple value
+ *
+ * @param i
+ * Index
+ * @return Tuple value
+ */
+ protected abstract T createValue(int i);
+
+ /**
+ * Splits the tuple value into its constituent nodes
+ *
+ * @param tuple
+ * Tuple value
+ * @return Nodes
+ */
+ protected abstract NodeWritable[] getNodes(T tuple);
+
+ /**
+ * Runs a node count test
+ *
+ * @param num
+ * Number of tuples to generate
+ * @throws IOException
+ */
+ protected void testNodeCount(int num) throws IOException {
+ MapDriver<LongWritable, T, NodeWritable, LongWritable> driver = this.getMapDriver();
+ this.generateData(driver, num);
+ driver.runTest();
+ }
+
+ /**
+ * Tests node counting
+ *
+ * @throws IOException
+ */
+ @Test
+ public void node_count_01() throws IOException {
+ this.testNodeCount(1);
+ }
+
+ /**
+ * Tests node counting
+ *
+ * @throws IOException
+ */
+ @Test
+ public void node_count_02() throws IOException {
+ this.testNodeCount(100);
+ }
+
+ /**
+ * Tests node counting
+ *
+ * @throws IOException
+ */
+ @Test
+ public void node_count_03() throws IOException {
+ this.testNodeCount(1000);
+ }
+
+ /**
+ * Tests node counting
+ *
+ * @throws IOException
+ */
+ @Test
+ public void node_count_04() throws IOException {
+ this.testNodeCount(2500);
+ }
+}
Added: jena/Experimental/hadoop-rdf/hadoop-rdf-mapreduce/src/test/java/org/apache/jena/hadoop/rdf/mapreduce/count/QuadNodeCountMapReduceTest.java
URL: http://svn.apache.org/viewvc/jena/Experimental/hadoop-rdf/hadoop-rdf-mapreduce/src/test/java/org/apache/jena/hadoop/rdf/mapreduce/count/QuadNodeCountMapReduceTest.java?rev=1585725&view=auto
==============================================================================
--- jena/Experimental/hadoop-rdf/hadoop-rdf-mapreduce/src/test/java/org/apache/jena/hadoop/rdf/mapreduce/count/QuadNodeCountMapReduceTest.java (added)
+++ jena/Experimental/hadoop-rdf/hadoop-rdf-mapreduce/src/test/java/org/apache/jena/hadoop/rdf/mapreduce/count/QuadNodeCountMapReduceTest.java Tue Apr 8 13:48:16 2014
@@ -0,0 +1,67 @@
+/*
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package org.apache.jena.hadoop.rdf.mapreduce.count;
+
+import org.apache.hadoop.io.LongWritable;
+import org.apache.hadoop.mapreduce.Mapper;
+import org.apache.hadoop.mapreduce.Reducer;
+import org.apache.jena.hadoop.rdf.mapreduce.count.NodeCountReducer;
+import org.apache.jena.hadoop.rdf.mapreduce.count.QuadNodeCountMapper;
+import org.apache.jena.hadoop.rdf.mapreduce.count.TripleNodeCountMapper;
+import org.apache.jena.hadoop.rdf.types.NodeWritable;
+import org.apache.jena.hadoop.rdf.types.QuadWritable;
+
+import com.hp.hpl.jena.datatypes.xsd.XSDDatatype;
+import com.hp.hpl.jena.graph.NodeFactory;
+import com.hp.hpl.jena.graph.Triple;
+import com.hp.hpl.jena.sparql.core.Quad;
+
+/**
+ * Tests for the {@link TripleNodeCountMapper} used in conjunction with the
+ * {@link NodeCountReducer}
+ *
+ * @author rvesse
+ *
+ */
+public class QuadNodeCountMapReduceTest extends AbstractNodeTupleNodeCountReducedTests<Quad, QuadWritable> {
+
+ @Override
+ protected Mapper<LongWritable, QuadWritable, NodeWritable, LongWritable> getMapperInstance() {
+ return new QuadNodeCountMapper<LongWritable>();
+ }
+
+ @Override
+ protected Reducer<NodeWritable, LongWritable, NodeWritable, LongWritable> getReducerInstance() {
+ return new NodeCountReducer();
+ }
+
+ @Override
+ protected QuadWritable createValue(int i) {
+ return new QuadWritable(new Quad(Quad.defaultGraphNodeGenerated, new Triple(
+ NodeFactory.createURI("http://subjects/" + i), NodeFactory.createURI("http://predicate"),
+ NodeFactory.createLiteral(Integer.toString(i), XSDDatatype.XSDinteger))));
+ }
+
+ @Override
+ protected NodeWritable[] getNodes(QuadWritable tuple) {
+ Quad q = tuple.get();
+ return new NodeWritable[] { new NodeWritable(q.getGraph()), new NodeWritable(q.getSubject()),
+ new NodeWritable(q.getPredicate()), new NodeWritable(q.getObject()) };
+ }
+}
Added: jena/Experimental/hadoop-rdf/hadoop-rdf-mapreduce/src/test/java/org/apache/jena/hadoop/rdf/mapreduce/count/QuadNodeCountMapperTest.java
URL: http://svn.apache.org/viewvc/jena/Experimental/hadoop-rdf/hadoop-rdf-mapreduce/src/test/java/org/apache/jena/hadoop/rdf/mapreduce/count/QuadNodeCountMapperTest.java?rev=1585725&view=auto
==============================================================================
--- jena/Experimental/hadoop-rdf/hadoop-rdf-mapreduce/src/test/java/org/apache/jena/hadoop/rdf/mapreduce/count/QuadNodeCountMapperTest.java (added)
+++ jena/Experimental/hadoop-rdf/hadoop-rdf-mapreduce/src/test/java/org/apache/jena/hadoop/rdf/mapreduce/count/QuadNodeCountMapperTest.java Tue Apr 8 13:48:16 2014
@@ -0,0 +1,59 @@
+/*
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package org.apache.jena.hadoop.rdf.mapreduce.count;
+
+import org.apache.hadoop.io.LongWritable;
+import org.apache.hadoop.mapreduce.Mapper;
+import org.apache.jena.hadoop.rdf.mapreduce.count.QuadNodeCountMapper;
+import org.apache.jena.hadoop.rdf.types.NodeWritable;
+import org.apache.jena.hadoop.rdf.types.QuadWritable;
+
+import com.hp.hpl.jena.datatypes.xsd.XSDDatatype;
+import com.hp.hpl.jena.graph.NodeFactory;
+import com.hp.hpl.jena.graph.Triple;
+import com.hp.hpl.jena.sparql.core.Quad;
+
+/**
+ * Tests for the {@link QuadNodeCountMapper}
+ *
+ * @author rvesse
+ *
+ */
+public class QuadNodeCountMapperTest extends AbstractNodeTupleNodeCountTests<Quad, QuadWritable> {
+
+ @Override
+ protected Mapper<LongWritable, QuadWritable, NodeWritable, LongWritable> getInstance() {
+ return new QuadNodeCountMapper<LongWritable>();
+ }
+
+ @Override
+ protected QuadWritable createValue(int i) {
+ return new QuadWritable(new Quad(Quad.defaultGraphNodeGenerated, new Triple(
+ NodeFactory.createURI("http://subjects/" + i), NodeFactory.createURI("http://predicate"),
+ NodeFactory.createLiteral(Integer.toString(i), XSDDatatype.XSDinteger))));
+ }
+
+ @Override
+ protected NodeWritable[] getNodes(QuadWritable tuple) {
+ Quad q = tuple.get();
+ return new NodeWritable[] { new NodeWritable(q.getGraph()), new NodeWritable(q.getSubject()),
+ new NodeWritable(q.getPredicate()), new NodeWritable(q.getObject()) };
+ }
+
+}
Added: jena/Experimental/hadoop-rdf/hadoop-rdf-mapreduce/src/test/java/org/apache/jena/hadoop/rdf/mapreduce/count/TripleNodeCountMapReduceTest.java
URL: http://svn.apache.org/viewvc/jena/Experimental/hadoop-rdf/hadoop-rdf-mapreduce/src/test/java/org/apache/jena/hadoop/rdf/mapreduce/count/TripleNodeCountMapReduceTest.java?rev=1585725&view=auto
==============================================================================
--- jena/Experimental/hadoop-rdf/hadoop-rdf-mapreduce/src/test/java/org/apache/jena/hadoop/rdf/mapreduce/count/TripleNodeCountMapReduceTest.java (added)
+++ jena/Experimental/hadoop-rdf/hadoop-rdf-mapreduce/src/test/java/org/apache/jena/hadoop/rdf/mapreduce/count/TripleNodeCountMapReduceTest.java Tue Apr 8 13:48:16 2014
@@ -0,0 +1,66 @@
+/*
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package org.apache.jena.hadoop.rdf.mapreduce.count;
+
+import org.apache.hadoop.io.LongWritable;
+import org.apache.hadoop.mapreduce.Mapper;
+import org.apache.hadoop.mapreduce.Reducer;
+import org.apache.jena.hadoop.rdf.mapreduce.count.NodeCountReducer;
+import org.apache.jena.hadoop.rdf.mapreduce.count.TripleNodeCountMapper;
+import org.apache.jena.hadoop.rdf.types.NodeWritable;
+import org.apache.jena.hadoop.rdf.types.TripleWritable;
+
+import com.hp.hpl.jena.datatypes.xsd.XSDDatatype;
+import com.hp.hpl.jena.graph.NodeFactory;
+import com.hp.hpl.jena.graph.Triple;
+
+/**
+ * Tests for the {@link TripleNodeCountMapper} used in conjunction with the
+ * {@link NodeCountReducer}
+ *
+ * @author rvesse
+ *
+ */
+public class TripleNodeCountMapReduceTest extends AbstractNodeTupleNodeCountReducedTests<Triple, TripleWritable> {
+
+ @Override
+ protected Mapper<LongWritable, TripleWritable, NodeWritable, LongWritable> getMapperInstance() {
+ return new TripleNodeCountMapper<LongWritable>();
+ }
+
+
+ @Override
+ protected Reducer<NodeWritable, LongWritable, NodeWritable, LongWritable> getReducerInstance() {
+ return new NodeCountReducer();
+ }
+
+ @Override
+ protected TripleWritable createValue(int i) {
+ return new TripleWritable(
+ new Triple(NodeFactory.createURI("http://subjects/" + i), NodeFactory.createURI("http://predicate"),
+ NodeFactory.createLiteral(Integer.toString(i), XSDDatatype.XSDinteger)));
+ }
+
+ @Override
+ protected NodeWritable[] getNodes(TripleWritable tuple) {
+ Triple t = tuple.get();
+ return new NodeWritable[] { new NodeWritable(t.getSubject()), new NodeWritable(t.getPredicate()),
+ new NodeWritable(t.getObject()) };
+ }
+}
Added: jena/Experimental/hadoop-rdf/hadoop-rdf-mapreduce/src/test/java/org/apache/jena/hadoop/rdf/mapreduce/count/TripleNodeCountMapperTest.java
URL: http://svn.apache.org/viewvc/jena/Experimental/hadoop-rdf/hadoop-rdf-mapreduce/src/test/java/org/apache/jena/hadoop/rdf/mapreduce/count/TripleNodeCountMapperTest.java?rev=1585725&view=auto
==============================================================================
--- jena/Experimental/hadoop-rdf/hadoop-rdf-mapreduce/src/test/java/org/apache/jena/hadoop/rdf/mapreduce/count/TripleNodeCountMapperTest.java (added)
+++ jena/Experimental/hadoop-rdf/hadoop-rdf-mapreduce/src/test/java/org/apache/jena/hadoop/rdf/mapreduce/count/TripleNodeCountMapperTest.java Tue Apr 8 13:48:16 2014
@@ -0,0 +1,58 @@
+/*
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package org.apache.jena.hadoop.rdf.mapreduce.count;
+
+import org.apache.hadoop.io.LongWritable;
+import org.apache.hadoop.mapreduce.Mapper;
+import org.apache.jena.hadoop.rdf.mapreduce.count.TripleNodeCountMapper;
+import org.apache.jena.hadoop.rdf.types.NodeWritable;
+import org.apache.jena.hadoop.rdf.types.TripleWritable;
+
+import com.hp.hpl.jena.datatypes.xsd.XSDDatatype;
+import com.hp.hpl.jena.graph.NodeFactory;
+import com.hp.hpl.jena.graph.Triple;
+
+/**
+ * Tests for the {@link TripleNodeCountMapper}
+ *
+ * @author rvesse
+ *
+ */
+public class TripleNodeCountMapperTest extends AbstractNodeTupleNodeCountTests<Triple, TripleWritable> {
+
+ @Override
+ protected Mapper<LongWritable, TripleWritable, NodeWritable, LongWritable> getInstance() {
+ return new TripleNodeCountMapper<LongWritable>();
+ }
+
+ @Override
+ protected TripleWritable createValue(int i) {
+ return new TripleWritable(
+ new Triple(NodeFactory.createURI("http://subjects/" + i), NodeFactory.createURI("http://predicate"),
+ NodeFactory.createLiteral(Integer.toString(i), XSDDatatype.XSDinteger)));
+ }
+
+ @Override
+ protected NodeWritable[] getNodes(TripleWritable tuple) {
+ Triple t = tuple.get();
+ return new NodeWritable[] { new NodeWritable(t.getSubject()), new NodeWritable(t.getPredicate()),
+ new NodeWritable(t.getObject()) };
+ }
+
+}
Added: jena/Experimental/hadoop-rdf/hadoop-rdf-mapreduce/src/test/java/org/apache/jena/hadoop/rdf/mapreduce/filter/AbstractNodeTupleFilterTests.java
URL: http://svn.apache.org/viewvc/jena/Experimental/hadoop-rdf/hadoop-rdf-mapreduce/src/test/java/org/apache/jena/hadoop/rdf/mapreduce/filter/AbstractNodeTupleFilterTests.java?rev=1585725&view=auto
==============================================================================
--- jena/Experimental/hadoop-rdf/hadoop-rdf-mapreduce/src/test/java/org/apache/jena/hadoop/rdf/mapreduce/filter/AbstractNodeTupleFilterTests.java (added)
+++ jena/Experimental/hadoop-rdf/hadoop-rdf-mapreduce/src/test/java/org/apache/jena/hadoop/rdf/mapreduce/filter/AbstractNodeTupleFilterTests.java Tue Apr 8 13:48:16 2014
@@ -0,0 +1,146 @@
+/*
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package org.apache.jena.hadoop.rdf.mapreduce.filter;
+
+import java.io.IOException;
+
+import org.apache.hadoop.io.LongWritable;
+import org.apache.hadoop.mrunit.mapreduce.MapDriver;
+import org.apache.jena.hadoop.rdf.mapreduce.AbstractMapperTests;
+import org.apache.jena.hadoop.rdf.mapreduce.filter.AbstractNodeTupleFilterMapper;
+import org.apache.jena.hadoop.rdf.types.AbstractNodeTupleWritable;
+import org.junit.Test;
+
+
+/**
+ * Abstract tests for {@link AbstractNodeTupleFilterMapper} implementations
+ * which filter based on the validity of tuples
+ *
+ * @author rvesse
+ *
+ * @param <TValue>
+ * Tuple type
+ * @param <T>
+ * Writable tuple type
+ */
+public abstract class AbstractNodeTupleFilterTests<TValue, T extends AbstractNodeTupleWritable<TValue>> extends
+ AbstractMapperTests<LongWritable, T, LongWritable, T> {
+
+ protected final void generateData(MapDriver<LongWritable, T, LongWritable, T> driver, int num) {
+ for (int i = 0; i < num; i++) {
+ LongWritable key = new LongWritable(i);
+ if (i % 2 == 0 && !this.noValidInputs()) {
+ T value = this.createValidValue(i);
+ driver.addInput(key, value);
+ if (!this.isInverted())
+ driver.addOutput(key, value);
+ } else {
+ T value = this.createInvalidValue(i);
+ driver.addInput(key, value);
+ if (this.isInverted())
+ driver.addOutput(key, value);
+ }
+ }
+ }
+
+ /**
+ * Method that may be overridden for testing filters where all the generated
+ * data will be rejected as invalid
+ *
+ * @return True if there are no valid inputs, false otherwise (default)
+ */
+ protected boolean noValidInputs() {
+ return false;
+ }
+
+ /**
+ * Method that may be overridden for testing filters with inverted mode
+ * enabled i.e. where normally valid input is considered invalid and vice
+ * versa
+ *
+ * @return True if inverted, false otherwise (default)
+ */
+ protected boolean isInverted() {
+ return false;
+ }
+
+ /**
+ * Creates an invalid value
+ *
+ * @param i
+ * Key
+ * @return Invalid value
+ */
+ protected abstract T createInvalidValue(int i);
+
+ /**
+ * Creates a valid value
+ *
+ * @param i
+ * Key
+ * @return Valid value
+ */
+ protected abstract T createValidValue(int i);
+
+ protected final void testFilterValid(int num) throws IOException {
+ MapDriver<LongWritable, T, LongWritable, T> driver = this.getMapDriver();
+ this.generateData(driver, num);
+ driver.runTest();
+ }
+
+ /**
+ * Test splitting tuples into their constituent nodes
+ *
+ * @throws IOException
+ */
+ @Test
+ public final void filter_valid_01() throws IOException {
+ this.testFilterValid(1);
+ }
+
+ /**
+ * Test splitting tuples into their constituent nodes
+ *
+ * @throws IOException
+ */
+ @Test
+ public final void filter_valid_02() throws IOException {
+ this.testFilterValid(100);
+ }
+
+ /**
+ * Test splitting tuples into their constituent nodes
+ *
+ * @throws IOException
+ */
+ @Test
+ public final void filter_valid_03() throws IOException {
+ this.testFilterValid(1000);
+ }
+
+ /**
+ * Test splitting tuples into their constituent nodes
+ *
+ * @throws IOException
+ */
+ @Test
+ public final void filter_valid_04() throws IOException {
+ this.testFilterValid(2500);
+ }
+}
Added: jena/Experimental/hadoop-rdf/hadoop-rdf-mapreduce/src/test/java/org/apache/jena/hadoop/rdf/mapreduce/filter/AbstractQuadValidityFilterTests.java
URL: http://svn.apache.org/viewvc/jena/Experimental/hadoop-rdf/hadoop-rdf-mapreduce/src/test/java/org/apache/jena/hadoop/rdf/mapreduce/filter/AbstractQuadValidityFilterTests.java?rev=1585725&view=auto
==============================================================================
--- jena/Experimental/hadoop-rdf/hadoop-rdf-mapreduce/src/test/java/org/apache/jena/hadoop/rdf/mapreduce/filter/AbstractQuadValidityFilterTests.java (added)
+++ jena/Experimental/hadoop-rdf/hadoop-rdf-mapreduce/src/test/java/org/apache/jena/hadoop/rdf/mapreduce/filter/AbstractQuadValidityFilterTests.java Tue Apr 8 13:48:16 2014
@@ -0,0 +1,86 @@
+/*
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package org.apache.jena.hadoop.rdf.mapreduce.filter;
+
+import org.apache.jena.hadoop.rdf.types.QuadWritable;
+
+import com.hp.hpl.jena.datatypes.xsd.XSDDatatype;
+import com.hp.hpl.jena.graph.NodeFactory;
+import com.hp.hpl.jena.sparql.core.Quad;
+
+/**
+ * Abstract tests for triple filter mappers that check triple validity
+ *
+ * @author rvesse
+ *
+ */
+public abstract class AbstractQuadValidityFilterTests extends AbstractNodeTupleFilterTests<Quad, QuadWritable> {
+
+ @Override
+ protected QuadWritable createValidValue(int i) {
+ return new QuadWritable(
+ new Quad(Quad.defaultGraphNodeGenerated, NodeFactory.createURI("http://subjects/" + i),
+ NodeFactory.createURI("http://predicate"), NodeFactory.createLiteral(Integer.toString(i),
+ XSDDatatype.XSDinteger)));
+ }
+
+ @Override
+ protected QuadWritable createInvalidValue(int i) {
+ switch (i % 8) {
+ case 0:
+ // Invalid to use Literal as Graph
+ return new QuadWritable(new Quad(NodeFactory.createLiteral("invalid"), NodeFactory.createURI("http://subjects/" + i),
+ NodeFactory.createURI("http://predicate"), NodeFactory.createLiteral(Integer.toString(i),
+ XSDDatatype.XSDinteger)));
+ case 1:
+ // Invalid to use Variable as Graph
+ return new QuadWritable(new Quad(NodeFactory.createVariable("invalid"),
+ NodeFactory.createURI("http://subjects/" + i), NodeFactory.createURI("http://predicate"),
+ NodeFactory.createLiteral(Integer.toString(i), XSDDatatype.XSDinteger)));
+ case 2:
+ // Invalid to use Literal as Subject
+ return new QuadWritable(new Quad(Quad.defaultGraphNodeGenerated, NodeFactory.createLiteral("invalid"),
+ NodeFactory.createURI("http://predicate"), NodeFactory.createLiteral(Integer.toString(i),
+ XSDDatatype.XSDinteger)));
+ case 3:
+ // Invalid to use Variable as Subject
+ return new QuadWritable(new Quad(Quad.defaultGraphNodeGenerated, NodeFactory.createVariable("invalid"),
+ NodeFactory.createURI("http://predicate"), NodeFactory.createLiteral(Integer.toString(i),
+ XSDDatatype.XSDinteger)));
+ case 4:
+ // Invalid to use Blank Node as Predicate
+ return new QuadWritable(new Quad(Quad.defaultGraphNodeGenerated, NodeFactory.createURI("http://subjects/" + i),
+ NodeFactory.createAnon(), NodeFactory.createLiteral(Integer.toString(i), XSDDatatype.XSDinteger)));
+ case 5:
+ // Invalid to use Literal as Predicate
+ return new QuadWritable(new Quad(Quad.defaultGraphNodeGenerated, NodeFactory.createURI("http://subjects/" + i),
+ NodeFactory.createLiteral("invalid"), NodeFactory.createLiteral(Integer.toString(i), XSDDatatype.XSDinteger)));
+ case 6:
+ // Invalid to use Variable as Predicate
+ return new QuadWritable(
+ new Quad(Quad.defaultGraphNodeGenerated, NodeFactory.createURI("http://subjects/" + i),
+ NodeFactory.createVariable("invalid"), NodeFactory.createLiteral(Integer.toString(i),
+ XSDDatatype.XSDinteger)));
+ default:
+ // Invalid to use Variable as Object
+ return new QuadWritable(new Quad(Quad.defaultGraphNodeGenerated, NodeFactory.createURI("http://subjects/" + i),
+ NodeFactory.createURI("http://predicate"), NodeFactory.createVariable("invalid")));
+ }
+ }
+}
Added: jena/Experimental/hadoop-rdf/hadoop-rdf-mapreduce/src/test/java/org/apache/jena/hadoop/rdf/mapreduce/filter/AbstractTripleValidityFilterTests.java
URL: http://svn.apache.org/viewvc/jena/Experimental/hadoop-rdf/hadoop-rdf-mapreduce/src/test/java/org/apache/jena/hadoop/rdf/mapreduce/filter/AbstractTripleValidityFilterTests.java?rev=1585725&view=auto
==============================================================================
--- jena/Experimental/hadoop-rdf/hadoop-rdf-mapreduce/src/test/java/org/apache/jena/hadoop/rdf/mapreduce/filter/AbstractTripleValidityFilterTests.java (added)
+++ jena/Experimental/hadoop-rdf/hadoop-rdf-mapreduce/src/test/java/org/apache/jena/hadoop/rdf/mapreduce/filter/AbstractTripleValidityFilterTests.java Tue Apr 8 13:48:16 2014
@@ -0,0 +1,73 @@
+/*
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package org.apache.jena.hadoop.rdf.mapreduce.filter;
+
+import org.apache.jena.hadoop.rdf.types.TripleWritable;
+
+import com.hp.hpl.jena.datatypes.xsd.XSDDatatype;
+import com.hp.hpl.jena.graph.NodeFactory;
+import com.hp.hpl.jena.graph.Triple;
+
+/**
+ * Abstract tests for triple filter mappers that check triple validity
+ *
+ * @author rvesse
+ *
+ */
+public abstract class AbstractTripleValidityFilterTests extends AbstractNodeTupleFilterTests<Triple, TripleWritable> {
+
+ @Override
+ protected TripleWritable createValidValue(int i) {
+ return new TripleWritable(
+ new Triple(NodeFactory.createURI("http://subjects/" + i), NodeFactory.createURI("http://predicate"),
+ NodeFactory.createLiteral(Integer.toString(i), XSDDatatype.XSDinteger)));
+ }
+
+ @Override
+ protected TripleWritable createInvalidValue(int i) {
+ switch (i % 6) {
+ case 0:
+ // Invalid to use Literal as Subject
+ return new TripleWritable(new Triple(NodeFactory.createLiteral("invalid"), NodeFactory.createURI("http://predicate"),
+ NodeFactory.createLiteral(Integer.toString(i), XSDDatatype.XSDinteger)));
+ case 1:
+ // Invalid to use Variable as Subject
+ return new TripleWritable(new Triple(NodeFactory.createVariable("invalid"),
+ NodeFactory.createURI("http://predicate"), NodeFactory.createLiteral(Integer.toString(i),
+ XSDDatatype.XSDinteger)));
+ case 2:
+ // Invalid to use Blank Node as Predicate
+ return new TripleWritable(new Triple(NodeFactory.createURI("http://subjects/" + i), NodeFactory.createAnon(),
+ NodeFactory.createLiteral(Integer.toString(i), XSDDatatype.XSDinteger)));
+ case 3:
+ // Invalid to use Literal as Predicate
+ return new TripleWritable(new Triple(NodeFactory.createURI("http://subjects/" + i),
+ NodeFactory.createLiteral("invalid"), NodeFactory.createLiteral(Integer.toString(i), XSDDatatype.XSDinteger)));
+ case 4:
+ // Invalid to use Variable as Predicate
+ return new TripleWritable(
+ new Triple(NodeFactory.createURI("http://subjects/" + i), NodeFactory.createVariable("invalid"),
+ NodeFactory.createLiteral(Integer.toString(i), XSDDatatype.XSDinteger)));
+ default:
+ // Invalid to use Variable as Object
+ return new TripleWritable(new Triple(NodeFactory.createURI("http://subjects/" + i),
+ NodeFactory.createURI("http://predicate"), NodeFactory.createVariable("invalid")));
+ }
+ }
+}
Added: jena/Experimental/hadoop-rdf/hadoop-rdf-mapreduce/src/test/java/org/apache/jena/hadoop/rdf/mapreduce/filter/TripleFilterByNoPredicateMapperTest.java
URL: http://svn.apache.org/viewvc/jena/Experimental/hadoop-rdf/hadoop-rdf-mapreduce/src/test/java/org/apache/jena/hadoop/rdf/mapreduce/filter/TripleFilterByNoPredicateMapperTest.java?rev=1585725&view=auto
==============================================================================
--- jena/Experimental/hadoop-rdf/hadoop-rdf-mapreduce/src/test/java/org/apache/jena/hadoop/rdf/mapreduce/filter/TripleFilterByNoPredicateMapperTest.java (added)
+++ jena/Experimental/hadoop-rdf/hadoop-rdf-mapreduce/src/test/java/org/apache/jena/hadoop/rdf/mapreduce/filter/TripleFilterByNoPredicateMapperTest.java Tue Apr 8 13:48:16 2014
@@ -0,0 +1,49 @@
+/*
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package org.apache.jena.hadoop.rdf.mapreduce.filter;
+
+import org.apache.jena.hadoop.rdf.mapreduce.filter.positional.TripleFilterByPredicateUriMapper;
+
+/**
+ * Tests for the {@link TripleFilterByPredicateUriMapper} where there are no
+ * predicates and thus all data must be invalid
+ *
+ * @author rvesse
+ *
+ */
+public class TripleFilterByNoPredicateMapperTest extends TripleFilterByPredicateMapperTest {
+
+ private static final String[] EMPTY_PREDICATE_POOL = new String[0];
+
+ /**
+ * Gets the pool of predicates considered valid
+ *
+ * @return Predicate pool
+ */
+ @Override
+ protected String[] getPredicatePool() {
+ return EMPTY_PREDICATE_POOL;
+ }
+
+ @Override
+ protected boolean noValidInputs() {
+ return true;
+ }
+
+}
Added: jena/Experimental/hadoop-rdf/hadoop-rdf-mapreduce/src/test/java/org/apache/jena/hadoop/rdf/mapreduce/filter/TripleFilterByPredicateMapperTest.java
URL: http://svn.apache.org/viewvc/jena/Experimental/hadoop-rdf/hadoop-rdf-mapreduce/src/test/java/org/apache/jena/hadoop/rdf/mapreduce/filter/TripleFilterByPredicateMapperTest.java?rev=1585725&view=auto
==============================================================================
--- jena/Experimental/hadoop-rdf/hadoop-rdf-mapreduce/src/test/java/org/apache/jena/hadoop/rdf/mapreduce/filter/TripleFilterByPredicateMapperTest.java (added)
+++ jena/Experimental/hadoop-rdf/hadoop-rdf-mapreduce/src/test/java/org/apache/jena/hadoop/rdf/mapreduce/filter/TripleFilterByPredicateMapperTest.java Tue Apr 8 13:48:16 2014
@@ -0,0 +1,80 @@
+/*
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package org.apache.jena.hadoop.rdf.mapreduce.filter;
+
+import org.apache.hadoop.io.LongWritable;
+import org.apache.hadoop.mapreduce.Mapper;
+import org.apache.hadoop.mrunit.mapreduce.MapDriver;
+import org.apache.jena.hadoop.rdf.mapreduce.RdfMapReduceConstants;
+import org.apache.jena.hadoop.rdf.mapreduce.filter.positional.TripleFilterByPredicateUriMapper;
+import org.apache.jena.hadoop.rdf.types.TripleWritable;
+
+import com.hp.hpl.jena.datatypes.xsd.XSDDatatype;
+import com.hp.hpl.jena.graph.NodeFactory;
+import com.hp.hpl.jena.graph.Triple;
+import com.hp.hpl.jena.vocabulary.RDF;
+import com.hp.hpl.jena.vocabulary.RDFS;
+
+/**
+ * Tests for the {@link TripleFilterByPredicateUriMapper}
+ *
+ * @author rvesse
+ *
+ */
+public class TripleFilterByPredicateMapperTest extends AbstractNodeTupleFilterTests<Triple, TripleWritable> {
+
+ private static final String[] DEFAULT_PREDICATE_POOL = new String[] { RDF.type.getURI(), RDFS.range.getURI(),
+ RDFS.domain.getURI() };
+
+ @Override
+ protected Mapper<LongWritable, TripleWritable, LongWritable, TripleWritable> getInstance() {
+ return new TripleFilterByPredicateUriMapper<LongWritable>();
+ }
+
+ @Override
+ protected void configureDriver(MapDriver<LongWritable, TripleWritable, LongWritable, TripleWritable> driver) {
+ super.configureDriver(driver);
+ driver.getContext().getConfiguration().setStrings(RdfMapReduceConstants.FILTER_PREDICATE_URIS, this.getPredicatePool());
+ }
+
+ /**
+ * Gets the pool of predicates considered valid
+ *
+ * @return Predicate pool
+ */
+ protected String[] getPredicatePool() {
+ return DEFAULT_PREDICATE_POOL;
+ }
+
+ @Override
+ protected TripleWritable createInvalidValue(int i) {
+ return new TripleWritable(
+ new Triple(NodeFactory.createURI("http://subjects/" + i), NodeFactory.createURI("http://predicate"),
+ NodeFactory.createLiteral(Integer.toString(i), XSDDatatype.XSDinteger)));
+ }
+
+ @Override
+ protected TripleWritable createValidValue(int i) {
+ String[] predicates = this.getPredicatePool();
+ if (predicates.length == 0) return this.createInvalidValue(i);
+ return new TripleWritable(new Triple(NodeFactory.createURI("http://subjects/" + i), NodeFactory.createURI(predicates[i
+ % predicates.length]), NodeFactory.createLiteral(Integer.toString(i), XSDDatatype.XSDinteger)));
+ }
+
+}
Added: jena/Experimental/hadoop-rdf/hadoop-rdf-mapreduce/src/test/java/org/apache/jena/hadoop/rdf/mapreduce/filter/TripleInvertedFilterByNoPredicateMapperTest.java
URL: http://svn.apache.org/viewvc/jena/Experimental/hadoop-rdf/hadoop-rdf-mapreduce/src/test/java/org/apache/jena/hadoop/rdf/mapreduce/filter/TripleInvertedFilterByNoPredicateMapperTest.java?rev=1585725&view=auto
==============================================================================
--- jena/Experimental/hadoop-rdf/hadoop-rdf-mapreduce/src/test/java/org/apache/jena/hadoop/rdf/mapreduce/filter/TripleInvertedFilterByNoPredicateMapperTest.java (added)
+++ jena/Experimental/hadoop-rdf/hadoop-rdf-mapreduce/src/test/java/org/apache/jena/hadoop/rdf/mapreduce/filter/TripleInvertedFilterByNoPredicateMapperTest.java Tue Apr 8 13:48:16 2014
@@ -0,0 +1,54 @@
+/*
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package org.apache.jena.hadoop.rdf.mapreduce.filter;
+
+import org.apache.jena.hadoop.rdf.mapreduce.filter.positional.TripleFilterByPredicateUriMapper;
+
+/**
+ * Tests for the {@link TripleFilterByPredicateUriMapper} where there are no
+ * predicates and thus all data must be invalid
+ *
+ * @author rvesse
+ *
+ */
+public class TripleInvertedFilterByNoPredicateMapperTest extends TripleInvertedFilterByPredicateMapperTest {
+
+ private static final String[] EMPTY_PREDICATE_POOL = new String[0];
+
+ /**
+ * Gets the pool of predicates considered valid
+ *
+ * @return Predicate pool
+ */
+ @Override
+ protected String[] getPredicatePool() {
+ return EMPTY_PREDICATE_POOL;
+ }
+
+ @Override
+ protected boolean noValidInputs() {
+ return true;
+ }
+
+ @Override
+ protected boolean isInverted() {
+ return true;
+ }
+
+}
Added: jena/Experimental/hadoop-rdf/hadoop-rdf-mapreduce/src/test/java/org/apache/jena/hadoop/rdf/mapreduce/filter/TripleInvertedFilterByPredicateMapperTest.java
URL: http://svn.apache.org/viewvc/jena/Experimental/hadoop-rdf/hadoop-rdf-mapreduce/src/test/java/org/apache/jena/hadoop/rdf/mapreduce/filter/TripleInvertedFilterByPredicateMapperTest.java?rev=1585725&view=auto
==============================================================================
--- jena/Experimental/hadoop-rdf/hadoop-rdf-mapreduce/src/test/java/org/apache/jena/hadoop/rdf/mapreduce/filter/TripleInvertedFilterByPredicateMapperTest.java (added)
+++ jena/Experimental/hadoop-rdf/hadoop-rdf-mapreduce/src/test/java/org/apache/jena/hadoop/rdf/mapreduce/filter/TripleInvertedFilterByPredicateMapperTest.java Tue Apr 8 13:48:16 2014
@@ -0,0 +1,87 @@
+/*
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package org.apache.jena.hadoop.rdf.mapreduce.filter;
+
+import org.apache.hadoop.io.LongWritable;
+import org.apache.hadoop.mapreduce.Mapper;
+import org.apache.hadoop.mrunit.mapreduce.MapDriver;
+import org.apache.jena.hadoop.rdf.mapreduce.RdfMapReduceConstants;
+import org.apache.jena.hadoop.rdf.mapreduce.filter.positional.TripleFilterByPredicateUriMapper;
+import org.apache.jena.hadoop.rdf.types.TripleWritable;
+
+import com.hp.hpl.jena.datatypes.xsd.XSDDatatype;
+import com.hp.hpl.jena.graph.NodeFactory;
+import com.hp.hpl.jena.graph.Triple;
+import com.hp.hpl.jena.vocabulary.RDF;
+import com.hp.hpl.jena.vocabulary.RDFS;
+
+/**
+ * Tests for the {@link TripleFilterByPredicateUriMapper}
+ *
+ * @author rvesse
+ *
+ */
+public class TripleInvertedFilterByPredicateMapperTest extends AbstractNodeTupleFilterTests<Triple, TripleWritable> {
+
+ private static final String[] DEFAULT_PREDICATE_POOL = new String[] { RDF.type.getURI(), RDFS.range.getURI(),
+ RDFS.domain.getURI() };
+
+ @Override
+ protected Mapper<LongWritable, TripleWritable, LongWritable, TripleWritable> getInstance() {
+ return new TripleFilterByPredicateUriMapper<LongWritable>();
+ }
+
+ @Override
+ protected void configureDriver(MapDriver<LongWritable, TripleWritable, LongWritable, TripleWritable> driver) {
+ super.configureDriver(driver);
+ driver.getContext().getConfiguration().setStrings(RdfMapReduceConstants.FILTER_PREDICATE_URIS, this.getPredicatePool());
+ driver.getContext().getConfiguration().setBoolean(RdfMapReduceConstants.FILTER_INVERT, true);
+ }
+
+ @Override
+ protected boolean isInverted() {
+ return true;
+ }
+
+ /**
+ * Gets the pool of predicates considered valid
+ *
+ * @return Predicate pool
+ */
+ protected String[] getPredicatePool() {
+ return DEFAULT_PREDICATE_POOL;
+ }
+
+ @Override
+ protected TripleWritable createInvalidValue(int i) {
+ return new TripleWritable(
+ new Triple(NodeFactory.createURI("http://subjects/" + i), NodeFactory.createURI("http://predicate"),
+ NodeFactory.createLiteral(Integer.toString(i), XSDDatatype.XSDinteger)));
+ }
+
+ @Override
+ protected TripleWritable createValidValue(int i) {
+ String[] predicates = this.getPredicatePool();
+ if (predicates.length == 0)
+ return this.createInvalidValue(i);
+ return new TripleWritable(new Triple(NodeFactory.createURI("http://subjects/" + i), NodeFactory.createURI(predicates[i
+ % predicates.length]), NodeFactory.createLiteral(Integer.toString(i), XSDDatatype.XSDinteger)));
+ }
+
+}
Added: jena/Experimental/hadoop-rdf/hadoop-rdf-mapreduce/src/test/java/org/apache/jena/hadoop/rdf/mapreduce/filter/ValidQuadFilterMapperTest.java
URL: http://svn.apache.org/viewvc/jena/Experimental/hadoop-rdf/hadoop-rdf-mapreduce/src/test/java/org/apache/jena/hadoop/rdf/mapreduce/filter/ValidQuadFilterMapperTest.java?rev=1585725&view=auto
==============================================================================
--- jena/Experimental/hadoop-rdf/hadoop-rdf-mapreduce/src/test/java/org/apache/jena/hadoop/rdf/mapreduce/filter/ValidQuadFilterMapperTest.java (added)
+++ jena/Experimental/hadoop-rdf/hadoop-rdf-mapreduce/src/test/java/org/apache/jena/hadoop/rdf/mapreduce/filter/ValidQuadFilterMapperTest.java Tue Apr 8 13:48:16 2014
@@ -0,0 +1,40 @@
+/*
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package org.apache.jena.hadoop.rdf.mapreduce.filter;
+
+import org.apache.hadoop.io.LongWritable;
+import org.apache.hadoop.mapreduce.Mapper;
+import org.apache.jena.hadoop.rdf.mapreduce.filter.ValidQuadFilterMapper;
+import org.apache.jena.hadoop.rdf.types.QuadWritable;
+
+
+/**
+ * Tests for the {@link ValidQuadFilterMapper}
+ *
+ * @author rvesse
+ *
+ */
+public class ValidQuadFilterMapperTest extends AbstractQuadValidityFilterTests {
+
+ @Override
+ protected Mapper<LongWritable, QuadWritable, LongWritable, QuadWritable> getInstance() {
+ return new ValidQuadFilterMapper<LongWritable>();
+ }
+
+}
Added: jena/Experimental/hadoop-rdf/hadoop-rdf-mapreduce/src/test/java/org/apache/jena/hadoop/rdf/mapreduce/filter/ValidTripleFilterMapperTest.java
URL: http://svn.apache.org/viewvc/jena/Experimental/hadoop-rdf/hadoop-rdf-mapreduce/src/test/java/org/apache/jena/hadoop/rdf/mapreduce/filter/ValidTripleFilterMapperTest.java?rev=1585725&view=auto
==============================================================================
--- jena/Experimental/hadoop-rdf/hadoop-rdf-mapreduce/src/test/java/org/apache/jena/hadoop/rdf/mapreduce/filter/ValidTripleFilterMapperTest.java (added)
+++ jena/Experimental/hadoop-rdf/hadoop-rdf-mapreduce/src/test/java/org/apache/jena/hadoop/rdf/mapreduce/filter/ValidTripleFilterMapperTest.java Tue Apr 8 13:48:16 2014
@@ -0,0 +1,40 @@
+/*
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package org.apache.jena.hadoop.rdf.mapreduce.filter;
+
+import org.apache.hadoop.io.LongWritable;
+import org.apache.hadoop.mapreduce.Mapper;
+import org.apache.jena.hadoop.rdf.mapreduce.filter.ValidTripleFilterMapper;
+import org.apache.jena.hadoop.rdf.types.TripleWritable;
+
+
+/**
+ * Tests for the {@link ValidTripleFilterMapper}
+ *
+ * @author rvesse
+ *
+ */
+public class ValidTripleFilterMapperTest extends AbstractTripleValidityFilterTests {
+
+ @Override
+ protected Mapper<LongWritable, TripleWritable, LongWritable, TripleWritable> getInstance() {
+ return new ValidTripleFilterMapper<LongWritable>();
+ }
+
+}
Added: jena/Experimental/hadoop-rdf/hadoop-rdf-mapreduce/src/test/java/org/apache/jena/hadoop/rdf/mapreduce/group/AbstractNodeTupleGroupingTests.java
URL: http://svn.apache.org/viewvc/jena/Experimental/hadoop-rdf/hadoop-rdf-mapreduce/src/test/java/org/apache/jena/hadoop/rdf/mapreduce/group/AbstractNodeTupleGroupingTests.java?rev=1585725&view=auto
==============================================================================
--- jena/Experimental/hadoop-rdf/hadoop-rdf-mapreduce/src/test/java/org/apache/jena/hadoop/rdf/mapreduce/group/AbstractNodeTupleGroupingTests.java (added)
+++ jena/Experimental/hadoop-rdf/hadoop-rdf-mapreduce/src/test/java/org/apache/jena/hadoop/rdf/mapreduce/group/AbstractNodeTupleGroupingTests.java Tue Apr 8 13:48:16 2014
@@ -0,0 +1,114 @@
+/*
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package org.apache.jena.hadoop.rdf.mapreduce.group;
+
+import java.io.IOException;
+
+import org.apache.hadoop.io.LongWritable;
+import org.apache.hadoop.mrunit.mapreduce.MapDriver;
+import org.apache.jena.hadoop.rdf.mapreduce.AbstractMapperTests;
+import org.apache.jena.hadoop.rdf.mapreduce.split.AbstractNodeTupleSplitToNodesMapper;
+import org.apache.jena.hadoop.rdf.types.AbstractNodeTupleWritable;
+import org.apache.jena.hadoop.rdf.types.NodeWritable;
+import org.junit.Test;
+
+
+/**
+ * Abstract tests for {@link AbstractNodeTupleSplitToNodesMapper}
+ * implementations
+ *
+ * @author rvesse
+ *
+ * @param <TValue>
+ * Tuple type
+ * @param <T>
+ * Writable tuple type
+ */
+public abstract class AbstractNodeTupleGroupingTests<TValue, T extends AbstractNodeTupleWritable<TValue>> extends
+ AbstractMapperTests<LongWritable, T, NodeWritable, T> {
+
+ /**
+ * Generates data for use in tests
+ *
+ * @param driver
+ * Driver
+ * @param num
+ * Number of tuples to generate
+ */
+ protected void generateData(MapDriver<LongWritable, T, NodeWritable, T> driver, int num) {
+ for (int i = 0; i < num; i++) {
+ LongWritable inputKey = new LongWritable(i);
+ T value = this.createValue(i);
+ NodeWritable outputKey = this.getOutputKey(value);
+
+ driver.addInput(inputKey, value);
+ driver.addOutput(outputKey, value);
+ }
+ }
+
+ protected abstract T createValue(int i);
+
+ protected abstract NodeWritable getOutputKey(T tuple);
+
+ protected final void testGrouping(int num) throws IOException {
+ MapDriver<LongWritable, T, NodeWritable, T> driver = this.getMapDriver();
+ this.generateData(driver, num);
+ driver.runTest();
+ }
+
+ /**
+ * Test grouping tuples by nodes
+ *
+ * @throws IOException
+ */
+ @Test
+ public final void grouping_01() throws IOException {
+ this.testGrouping(1);
+ }
+
+ /**
+ * Test grouping tuples by nodes
+ *
+ * @throws IOException
+ */
+ @Test
+ public final void grouping_02() throws IOException {
+ this.testGrouping(100);
+ }
+
+ /**
+ * Test grouping tuples by nodes
+ *
+ * @throws IOException
+ */
+ @Test
+ public final void grouping_03() throws IOException {
+ this.testGrouping(1000);
+ }
+
+ /**
+ * Test grouping tuples by nodes
+ *
+ * @throws IOException
+ */
+ @Test
+ public final void grouping_04() throws IOException {
+ this.testGrouping(2500);
+ }
+}