You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@jena.apache.org by rv...@apache.org on 2014/04/02 11:20:58 UTC
svn commit: r1583942 [6/9] - in /jena/Experimental/hadoop-rdf: ./
hadoop-rdf-common/ hadoop-rdf-common/src/ hadoop-rdf-common/src/main/
hadoop-rdf-common/src/main/java/ hadoop-rdf-common/src/main/java/com/
hadoop-rdf-common/src/main/java/com/yarcdata/ ...
Added: jena/Experimental/hadoop-rdf/hadoop-rdf-io/src/test/java/com/yarcdata/urika/hadoop/rdf/io/input/util/AbstractTrackableInputStreamTests.java
URL: http://svn.apache.org/viewvc/jena/Experimental/hadoop-rdf/hadoop-rdf-io/src/test/java/com/yarcdata/urika/hadoop/rdf/io/input/util/AbstractTrackableInputStreamTests.java?rev=1583942&view=auto
==============================================================================
--- jena/Experimental/hadoop-rdf/hadoop-rdf-io/src/test/java/com/yarcdata/urika/hadoop/rdf/io/input/util/AbstractTrackableInputStreamTests.java (added)
+++ jena/Experimental/hadoop-rdf/hadoop-rdf-io/src/test/java/com/yarcdata/urika/hadoop/rdf/io/input/util/AbstractTrackableInputStreamTests.java Wed Apr 2 09:20:51 2014
@@ -0,0 +1,686 @@
+/*
+ * Copyright 2013 YarcData LLC All Rights Reserved.
+ */
+
+package com.yarcdata.urika.hadoop.rdf.io.input.util;
+
+import java.io.ByteArrayInputStream;
+import java.io.ByteArrayOutputStream;
+import java.io.IOException;
+import java.io.InputStream;
+
+import org.junit.Assert;
+import org.junit.Test;
+
+/**
+ * Abstract tests for {@link TrackableInputStream} implementations
+ *
+ * @author rvesse
+ *
+ */
+public abstract class AbstractTrackableInputStreamTests {
+
+ protected static final int KILO = 1024;
+ protected static final int BYTES_PER_KB = KILO;
+ protected static final int BYTES_PER_MB = BYTES_PER_KB * KILO;
+
+ /**
+ * Gets the instance to test using the given input as the stream to track
+ *
+ * @param input
+ * Input Stream
+ * @return Trackable Input Stream
+ */
+ protected abstract TrackableInputStream getInstance(InputStream input);
+
+ /**
+ * Generates an input stream containing the given number of bytes
+ *
+ * @param length
+ * Number of bytes
+ * @return Input stream
+ */
+ protected final InputStream generateData(int length) {
+ ByteArrayOutputStream output = new ByteArrayOutputStream(length);
+ byte b = (byte) 'b';
+ for (int i = 0; i < length; i++) {
+ output.write(b);
+ }
+ return new ByteArrayInputStream(output.toByteArray());
+ }
+
+ protected final void testSingleByteRead(int length) throws IOException {
+ InputStream input = this.generateData(length);
+ TrackableInputStream trackable = this.getInstance(input);
+ long count = 0;
+ while (trackable.read() >= 0) {
+ count++;
+ }
+ Assert.assertEquals(length, count);
+ Assert.assertEquals(length, trackable.getBytesRead());
+ trackable.close();
+ }
+
+ /**
+ * Test reading byte by byte
+ *
+ * @throws IOException
+ */
+ @Test
+ public final void trackable_input_read_single_01() throws IOException {
+ this.testSingleByteRead(0);
+ }
+
+ /**
+ * Test reading byte by byte
+ *
+ * @throws IOException
+ */
+ @Test
+ public final void trackable_input_read_single_02() throws IOException {
+ this.testSingleByteRead(100);
+ }
+
+ /**
+ * Test reading byte by byte
+ *
+ * @throws IOException
+ */
+ @Test
+ public final void trackable_input_read_single_03() throws IOException {
+ // 1KB
+ this.testSingleByteRead(BYTES_PER_KB);
+ }
+
+ /**
+ * Test reading byte by byte
+ *
+ * @throws IOException
+ */
+ @Test
+ public final void trackable_input_read_single_04() throws IOException {
+ // 1 MB
+ this.testSingleByteRead(BYTES_PER_MB);
+ }
+
+ protected final void testMultiByteRead(int length, int bufferSize) throws IOException {
+ if (bufferSize < 1)
+ throw new IllegalArgumentException("bufferSize must be >= 1");
+ InputStream input = this.generateData(length);
+ TrackableInputStream trackable = this.getInstance(input);
+ long count = 0;
+ byte[] buffer = new byte[bufferSize];
+ long read;
+ do {
+ read = trackable.read(buffer);
+ if (read > 0)
+ count += read;
+ } while (read >= 0);
+ Assert.assertEquals(length, count);
+ Assert.assertEquals(length, trackable.getBytesRead());
+ trackable.close();
+ }
+
+ /**
+ * Test reading multiple bytes i.e. calling {@link InputStream#read(byte[])}
+ *
+ * @throws IOException
+ */
+ @Test
+ public final void trackable_input_read_multiple_01() throws IOException {
+ this.testMultiByteRead(0, 1);
+ }
+
+ /**
+ * Test reading multiple bytes i.e. calling {@link InputStream#read(byte[])}
+ *
+ * @throws IOException
+ */
+ @Test
+ public final void trackable_input_read_multiple_02() throws IOException {
+ this.testMultiByteRead(0, 16);
+ }
+
+ /**
+ * Test reading multiple bytes i.e. calling {@link InputStream#read(byte[])}
+ *
+ * @throws IOException
+ */
+ @Test
+ public final void trackable_input_read_multiple_03() throws IOException {
+ this.testMultiByteRead(0, BYTES_PER_KB);
+ }
+
+ /**
+ * Test reading multiple bytes i.e. calling {@link InputStream#read(byte[])}
+ *
+ * @throws IOException
+ */
+ @Test
+ public final void trackable_input_read_multiple_04() throws IOException {
+ // 1KB
+ this.testMultiByteRead(BYTES_PER_KB, 1);
+ }
+
+ /**
+ * Test reading multiple bytes i.e. calling {@link InputStream#read(byte[])}
+ *
+ * @throws IOException
+ */
+ @Test
+ public final void trackable_input_read_multiple_05() throws IOException {
+ // 1KB
+ this.testMultiByteRead(BYTES_PER_KB, 16);
+ }
+
+ /**
+ * Test reading multiple bytes i.e. calling {@link InputStream#read(byte[])}
+ *
+ * @throws IOException
+ */
+ @Test
+ public final void trackable_input_read_multiple_06() throws IOException {
+ // 1KB
+ this.testMultiByteRead(BYTES_PER_KB, BYTES_PER_KB);
+ }
+
+ /**
+ * Test reading multiple bytes i.e. calling {@link InputStream#read(byte[])}
+ *
+ * @throws IOException
+ */
+ @Test
+ public final void trackable_input_read_multiple_07() throws IOException {
+ // 1MB
+ this.testMultiByteRead(BYTES_PER_MB, 1);
+ }
+
+ /**
+ * Test reading multiple bytes i.e. calling {@link InputStream#read(byte[])}
+ *
+ * @throws IOException
+ */
+ @Test
+ public final void trackable_input_read_multiple_08() throws IOException {
+ // 1MB
+ this.testMultiByteRead(BYTES_PER_MB, 16);
+ }
+
+ /**
+ * Test reading multiple bytes i.e. calling {@link InputStream#read(byte[])}
+ *
+ * @throws IOException
+ */
+ @Test
+ public final void trackable_input_read_multiple_09() throws IOException {
+ // 1MB
+ this.testMultiByteRead(BYTES_PER_MB, BYTES_PER_KB);
+ }
+
+ protected final void testMultiByteRead(int length, int bufferSize, int readSize) throws IOException {
+ if (bufferSize < 1)
+ throw new IllegalArgumentException("bufferSize must be >= 1");
+ if (readSize < 1 || readSize > bufferSize)
+ throw new IllegalArgumentException("readSize must be >= 1 and <= bufferSize");
+ InputStream input = this.generateData(length);
+ TrackableInputStream trackable = this.getInstance(input);
+ long count = 0;
+ byte[] buffer = new byte[bufferSize];
+ long read;
+ do {
+ read = trackable.read(buffer, 0, readSize);
+ if (read > 0)
+ count += read;
+ } while (read >= 0);
+ Assert.assertEquals(length, count);
+ Assert.assertEquals(length, trackable.getBytesRead());
+ trackable.close();
+ }
+
+ /**
+ * Test reading multiple bytes while reading less than the buffer size bytes
+ * i.e. calling {@link InputStream#read(byte[], int, int)}
+ *
+ * @throws IOException
+ */
+ @Test
+ public final void trackable_input_read_multiple_partial_01() throws IOException {
+ this.testMultiByteRead(0, 1, 1);
+ }
+
+ /**
+ * Test reading multiple bytes while reading less than the buffer size bytes
+ * i.e. calling {@link InputStream#read(byte[], int, int)}
+ *
+ * @throws IOException
+ */
+ @Test
+ public final void trackable_input_read_multiple_partial_02() throws IOException {
+ this.testMultiByteRead(0, 16, 1);
+ }
+
+ /**
+ * Test reading multiple bytes while reading less than the buffer size bytes
+ * i.e. calling {@link InputStream#read(byte[], int, int)}
+ *
+ * @throws IOException
+ */
+ @Test
+ public final void trackable_input_read_multiple_partial_03() throws IOException {
+ this.testMultiByteRead(0, 16, 16);
+ }
+
+ /**
+ * Test reading multiple bytes while reading less than the buffer size bytes
+ * i.e. calling {@link InputStream#read(byte[], int, int)}
+ *
+ * @throws IOException
+ */
+ @Test
+ public final void trackable_input_read_multiple_partial_04() throws IOException {
+ this.testMultiByteRead(0, BYTES_PER_KB, 1);
+ }
+
+ /**
+ * Test reading multiple bytes while reading less than the buffer size bytes
+ * i.e. calling {@link InputStream#read(byte[], int, int)}
+ *
+ * @throws IOException
+ */
+ @Test
+ public final void trackable_input_read_multiple_partial_05() throws IOException {
+ this.testMultiByteRead(0, BYTES_PER_KB, 16);
+ }
+
+ /**
+ * Test reading multiple bytes while reading less than the buffer size bytes
+ * i.e. calling {@link InputStream#read(byte[], int, int)}
+ *
+ * @throws IOException
+ */
+ @Test
+ public final void trackable_input_read_multiple_partial_06() throws IOException {
+ this.testMultiByteRead(0, BYTES_PER_KB, BYTES_PER_KB);
+ }
+
+ /**
+ * Test reading multiple bytes while reading less than the buffer size bytes
+ * i.e. calling {@link InputStream#read(byte[], int, int)}
+ *
+ * @throws IOException
+ */
+ @Test
+ public final void trackable_input_read_multiple_partial_07() throws IOException {
+ // 1KB
+ this.testMultiByteRead(BYTES_PER_KB, 1, 1);
+ }
+
+ /**
+ * Test reading multiple bytes while reading less than the buffer size bytes
+ * i.e. calling {@link InputStream#read(byte[], int, int)}
+ *
+ * @throws IOException
+ */
+ @Test
+ public final void trackable_input_read_multiple_partial_08() throws IOException {
+ // 1KB
+ this.testMultiByteRead(BYTES_PER_KB, 16, 1);
+ }
+
+ /**
+ * Test reading multiple bytes while reading less than the buffer size bytes
+ * i.e. calling {@link InputStream#read(byte[], int, int)}
+ *
+ * @throws IOException
+ */
+ @Test
+ public final void trackable_input_read_multiple_partial_09() throws IOException {
+ // 1KB
+ this.testMultiByteRead(BYTES_PER_KB, 16, 16);
+ }
+
+ /**
+ * Test reading multiple bytes while reading less than the buffer size bytes
+ * i.e. calling {@link InputStream#read(byte[], int, int)}
+ *
+ * @throws IOException
+ */
+ @Test
+ public final void trackable_input_read_multiple_partial_10() throws IOException {
+ // 1KB
+ this.testMultiByteRead(BYTES_PER_KB, BYTES_PER_KB, 1);
+ }
+
+ /**
+ * Test reading multiple bytes while reading less than the buffer size bytes
+ * i.e. calling {@link InputStream#read(byte[], int, int)}
+ *
+ * @throws IOException
+ */
+ @Test
+ public final void trackable_input_read_multiple_partial_11() throws IOException {
+ // 1KB
+ this.testMultiByteRead(BYTES_PER_KB, BYTES_PER_KB, 16);
+ }
+
+ /**
+ * Test reading multiple bytes while reading less than the buffer size bytes
+ * i.e. calling {@link InputStream#read(byte[], int, int)}
+ *
+ * @throws IOException
+ */
+ @Test
+ public final void trackable_input_read_multiple_partial_12() throws IOException {
+ // 1KB
+ this.testMultiByteRead(BYTES_PER_KB, BYTES_PER_KB, BYTES_PER_KB);
+ }
+
+ /**
+ * Test reading multiple bytes while reading less than the buffer size bytes
+ * i.e. calling {@link InputStream#read(byte[], int, int)}
+ *
+ * @throws IOException
+ */
+ @Test
+ public final void trackable_input_read_multiple_partial_13() throws IOException {
+ // 1MB
+ this.testMultiByteRead(BYTES_PER_MB, 1, 1);
+ }
+
+ /**
+ * Test reading multiple bytes while reading less than the buffer size bytes
+ * i.e. calling {@link InputStream#read(byte[], int, int)}
+ *
+ * @throws IOException
+ */
+ @Test
+ public final void trackable_input_read_multiple_partial_14() throws IOException {
+ // 1MB
+ this.testMultiByteRead(BYTES_PER_MB, 16, 1);
+ }
+
+ /**
+ * Test reading multiple bytes while reading less than the buffer size bytes
+ * i.e. calling {@link InputStream#read(byte[], int, int)}
+ *
+ * @throws IOException
+ */
+ @Test
+ public final void trackable_input_read_multiple_partial_15() throws IOException {
+ // 1MB
+ this.testMultiByteRead(BYTES_PER_MB, 16, 16);
+ }
+
+ /**
+ * Test reading multiple bytes while reading less than the buffer size bytes
+ * i.e. calling {@link InputStream#read(byte[], int, int)}
+ *
+ * @throws IOException
+ */
+ @Test
+ public final void trackable_input_read_multiple_partial_16() throws IOException {
+ // 1MB
+ this.testMultiByteRead(BYTES_PER_MB, BYTES_PER_KB, 1);
+ }
+
+ /**
+ * Test reading multiple bytes while reading less than the buffer size bytes
+ * i.e. calling {@link InputStream#read(byte[], int, int)}
+ *
+ * @throws IOException
+ */
+ @Test
+ public final void trackable_input_read_multiple_partial_17() throws IOException {
+ // 1MB
+ this.testMultiByteRead(BYTES_PER_MB, BYTES_PER_KB, 16);
+ }
+
+ /**
+ * Test reading multiple bytes while reading less than the buffer size bytes
+ * i.e. calling {@link InputStream#read(byte[], int, int)}
+ *
+ * @throws IOException
+ */
+ @Test
+ public final void trackable_input_read_multiple_partial_18() throws IOException {
+ // 1MB
+ this.testMultiByteRead(BYTES_PER_MB, BYTES_PER_KB, BYTES_PER_KB);
+ }
+
+ protected final void testSkip(int length, long skipSize) throws IOException {
+ if (skipSize < 1)
+ throw new IllegalArgumentException("skipSize must be >= 1");
+ InputStream input = this.generateData(length);
+ TrackableInputStream trackable = this.getInstance(input);
+ long count = 0;
+ long skipped;
+ do {
+ skipped = trackable.skip(skipSize);
+ if (skipped > 0)
+ count += skipped;
+ } while (skipped > 0);
+ Assert.assertEquals(length, count);
+ Assert.assertEquals(length, trackable.getBytesRead());
+ trackable.close();
+ }
+
+ /**
+ * Test skipping
+ *
+ * @throws IOException
+ */
+ @Test
+ public final void trackable_input_skip_single_01() throws IOException {
+ this.testSkip(0, 1);
+ }
+
+ /**
+ * Test skipping
+ *
+ * @throws IOException
+ */
+ @Test
+ public final void trackable_input_skip_single_02() throws IOException {
+ this.testSkip(100, 1);
+ }
+
+ /**
+ * Test skipping
+ *
+ * @throws IOException
+ */
+ @Test
+ public final void trackable_input_skip_single_03() throws IOException {
+ this.testSkip(100, 16);
+ }
+
+ /**
+ * Test skipping
+ *
+ * @throws IOException
+ */
+ @Test
+ public final void trackable_input_skip_single_04() throws IOException {
+ this.testSkip(100, BYTES_PER_KB);
+ }
+
+ /**
+ * Test skipping
+ *
+ * @throws IOException
+ */
+ @Test
+ public final void trackable_input_skip_single_05() throws IOException {
+ // 1KB
+ this.testSkip(BYTES_PER_KB, 1);
+ }
+
+ /**
+ * Test skipping
+ *
+ * @throws IOException
+ */
+ @Test
+ public final void trackable_input_skip_single_06() throws IOException {
+ // 1KB
+ this.testSkip(BYTES_PER_KB, 16);
+ }
+
+ /**
+ * Test skipping
+ *
+ * @throws IOException
+ */
+ @Test
+ public final void trackable_input_skip_single_07() throws IOException {
+ // 1KB
+ this.testSkip(BYTES_PER_KB, BYTES_PER_KB);
+ }
+
+ /**
+ * Test skipping
+ *
+ * @throws IOException
+ */
+ @Test
+ public final void trackable_input_skip_single_08() throws IOException {
+ // 1KB
+ this.testSkip(BYTES_PER_KB, BYTES_PER_MB);
+ }
+
+ /**
+ * Test skipping
+ *
+ * @throws IOException
+ */
+ @Test
+ public final void trackable_input_skip_single_09() throws IOException {
+ // 1 MB
+ this.testSkip(BYTES_PER_MB, 1);
+ }
+
+ /**
+ * Test skipping
+ *
+ * @throws IOException
+ */
+ @Test
+ public final void trackable_input_skip_single_10() throws IOException {
+ // 1 MB
+ this.testSkip(BYTES_PER_MB, 16);
+ }
+
+ /**
+ * Test skipping
+ *
+ * @throws IOException
+ */
+ @Test
+ public final void trackable_input_skip_single_11() throws IOException {
+ // 1 MB
+ this.testSkip(BYTES_PER_MB, BYTES_PER_KB);
+ }
+
+ /**
+ * Test skipping
+ *
+ * @throws IOException
+ */
+ @Test
+ public final void trackable_input_skip_single_12() throws IOException {
+ // 1 MB
+ this.testSkip(BYTES_PER_MB, BYTES_PER_MB);
+ }
+
+ /**
+ * Tests behaviour after closing
+ *
+ * @throws IOException
+ */
+ @Test
+ public final void trackable_input_post_close_01() throws IOException {
+ InputStream input = this.generateData(0);
+ TrackableInputStream trackable = this.getInstance(input);
+ trackable.close();
+ Assert.assertEquals(-1, trackable.read());
+ }
+
+ /**
+ * Tests behaviour after closing
+ *
+ * @throws IOException
+ */
+ @Test
+ public final void trackable_input_post_close_02() throws IOException {
+ InputStream input = this.generateData(0);
+ TrackableInputStream trackable = this.getInstance(input);
+ trackable.close();
+ Assert.assertEquals(0, trackable.read(new byte[0]));
+ }
+
+ /**
+ * Tests behaviour after closing
+ *
+ * @throws IOException
+ */
+ @Test
+ public final void trackable_input_post_close_03() throws IOException {
+ InputStream input = this.generateData(0);
+ TrackableInputStream trackable = this.getInstance(input);
+ trackable.close();
+ Assert.assertEquals(-1, trackable.read(new byte[1]));
+ }
+
+ /**
+ * Tests behaviour after closing
+ *
+ * @throws IOException
+ */
+ @Test
+ public final void trackable_input_post_close_04() throws IOException {
+ InputStream input = this.generateData(0);
+ TrackableInputStream trackable = this.getInstance(input);
+ trackable.close();
+ Assert.assertEquals(0, trackable.read(new byte[16], 0, 0));
+ }
+
+ /**
+ * Tests behaviour after closing
+ *
+ * @throws IOException
+ */
+ @Test
+ public final void trackable_input_post_close_05() throws IOException {
+ InputStream input = this.generateData(0);
+ TrackableInputStream trackable = this.getInstance(input);
+ trackable.close();
+ Assert.assertEquals(-1, trackable.read(new byte[16], 0, 8));
+ }
+
+ /**
+ * Tests behaviour after closing
+ *
+ * @throws IOException
+ */
+ @Test
+ public final void trackable_input_post_close_06() throws IOException {
+ InputStream input = this.generateData(0);
+ TrackableInputStream trackable = this.getInstance(input);
+ trackable.close();
+ Assert.assertEquals(0, trackable.skip(0));
+ }
+
+ /**
+ * Tests exceptions are thrown trying to perform actions after closing the
+ * input
+ *
+ * @throws IOException
+ */
+ @Test
+ public final void trackable_input_post_close_07() throws IOException {
+ InputStream input = this.generateData(0);
+ TrackableInputStream trackable = this.getInstance(input);
+ trackable.close();
+ Assert.assertEquals(0, trackable.skip(1));
+ }
+}
Added: jena/Experimental/hadoop-rdf/hadoop-rdf-io/src/test/java/com/yarcdata/urika/hadoop/rdf/io/input/util/BlockInputStreamTest.java
URL: http://svn.apache.org/viewvc/jena/Experimental/hadoop-rdf/hadoop-rdf-io/src/test/java/com/yarcdata/urika/hadoop/rdf/io/input/util/BlockInputStreamTest.java?rev=1583942&view=auto
==============================================================================
--- jena/Experimental/hadoop-rdf/hadoop-rdf-io/src/test/java/com/yarcdata/urika/hadoop/rdf/io/input/util/BlockInputStreamTest.java (added)
+++ jena/Experimental/hadoop-rdf/hadoop-rdf-io/src/test/java/com/yarcdata/urika/hadoop/rdf/io/input/util/BlockInputStreamTest.java Wed Apr 2 09:20:51 2014
@@ -0,0 +1,224 @@
+/*
+ * Copyright 2013 YarcData LLC All Rights Reserved.
+ */
+
+package com.yarcdata.urika.hadoop.rdf.io.input.util;
+
+import java.io.IOException;
+import java.io.InputStream;
+
+import org.junit.Assert;
+import org.junit.Test;
+
+/**
+ * Tests for the {@link BlockInputStream}
+ *
+ * @author rvesse
+ *
+ */
+public class BlockInputStreamTest extends AbstractTrackableInputStreamTests {
+
+ @Override
+ protected TrackableInputStream getInstance(InputStream input) {
+ return new BlockInputStream(input, Long.MAX_VALUE);
+ }
+
+ /**
+ * Gets an instance of a block input stream
+ *
+ * @param input
+ * Underlying input stream
+ * @param limit
+ * Limit on bytes to read
+ * @return Block input stream
+ */
+ protected BlockInputStream getInstance(InputStream input, long limit) {
+ return new BlockInputStream(input, limit);
+ }
+
+ protected final void testSingleByteRead(int length, long limit) throws IOException {
+ InputStream input = this.generateData(length);
+ TrackableInputStream trackable = this.getInstance(input, limit);
+ long count = 0;
+ while (trackable.read() >= 0) {
+ count++;
+ }
+ int expected = (int) Math.min(length, limit);
+ Assert.assertEquals(expected, count);
+ Assert.assertEquals(expected, trackable.getBytesRead());
+ trackable.close();
+ }
+
+ /**
+ * Test reading byte by byte
+ *
+ * @throws IOException
+ */
+ @Test
+ public final void block_input_read_single_01() throws IOException {
+ this.testSingleByteRead(0, 0);
+ }
+
+ /**
+ * Test reading byte by byte
+ *
+ * @throws IOException
+ */
+ @Test
+ public final void block_input_read_single_02() throws IOException {
+ this.testSingleByteRead(100, 0);
+ }
+
+ /**
+ * Test reading byte by byte
+ *
+ * @throws IOException
+ */
+ @Test
+ public final void block_input_read_single_03() throws IOException {
+ this.testSingleByteRead(100, 50);
+ }
+
+ /**
+ * Test reading byte by byte
+ *
+ * @throws IOException
+ */
+ @Test
+ public final void block_input_read_single_04() throws IOException {
+ this.testSingleByteRead(100, 100);
+ }
+
+ /**
+ * Test reading byte by byte
+ *
+ * @throws IOException
+ */
+ @Test
+ public final void block_input_read_single_05() throws IOException {
+ // 1KB
+ this.testSingleByteRead(BYTES_PER_KB, 1);
+ }
+
+ /**
+ * Test reading byte by byte
+ *
+ * @throws IOException
+ */
+ @Test
+ public final void block_input_read_single_06() throws IOException {
+ // 1KB
+ this.testSingleByteRead(BYTES_PER_KB, 100);
+ }
+
+ /**
+ * Test reading byte by byte
+ *
+ * @throws IOException
+ */
+ @Test
+ public final void block_input_read_single_07() throws IOException {
+ // 1KB
+ this.testSingleByteRead(BYTES_PER_KB, BYTES_PER_KB / 2);
+ }
+
+ /**
+ * Test reading byte by byte
+ *
+ * @throws IOException
+ */
+ @Test
+ public final void block_input_read_single_08() throws IOException {
+ // 1KB
+ this.testSingleByteRead(BYTES_PER_KB, BYTES_PER_KB);
+ }
+
+ /**
+ * Test reading byte by byte
+ *
+ * @throws IOException
+ */
+ @Test
+ public final void block_input_read_single_09() throws IOException {
+ // 1KB
+ this.testSingleByteRead(BYTES_PER_KB, BYTES_PER_MB);
+ }
+
+ /**
+ * Test reading byte by byte
+ *
+ * @throws IOException
+ */
+ @Test
+ public final void block_input_read_single_10() throws IOException {
+ // 1KB
+ this.testSingleByteRead(BYTES_PER_KB, BYTES_PER_MB * 10);
+ }
+
+ /**
+ * Test reading byte by byte
+ *
+ * @throws IOException
+ */
+ @Test
+ public final void block_input_read_single_11() throws IOException {
+ // 1MB
+ this.testSingleByteRead(BYTES_PER_MB, 1);
+ }
+
+ /**
+ * Test reading byte by byte
+ *
+ * @throws IOException
+ */
+ @Test
+ public final void block_input_read_single_12() throws IOException {
+ // 1MB
+ this.testSingleByteRead(BYTES_PER_MB, 100);
+ }
+
+ /**
+ * Test reading byte by byte
+ *
+ * @throws IOException
+ */
+ @Test
+ public final void block_input_read_single_13() throws IOException {
+ // 1MB
+ this.testSingleByteRead(BYTES_PER_MB, BYTES_PER_KB);
+ }
+
+ /**
+ * Test reading byte by byte
+ *
+ * @throws IOException
+ */
+ @Test
+ public final void block_input_read_single_14() throws IOException {
+ // 1MB
+ this.testSingleByteRead(BYTES_PER_MB, BYTES_PER_MB / 2);
+ }
+
+ /**
+ * Test reading byte by byte
+ *
+ * @throws IOException
+ */
+ @Test
+ public final void block_input_read_single_15() throws IOException {
+ // 1MB
+ this.testSingleByteRead(BYTES_PER_MB, BYTES_PER_MB);
+ }
+
+ /**
+ * Test reading byte by byte
+ *
+ * @throws IOException
+ */
+ @Test
+ public final void block_input_read_single_16() throws IOException {
+ // 1MB
+ this.testSingleByteRead(BYTES_PER_MB, BYTES_PER_MB * 10);
+ }
+
+}
Added: jena/Experimental/hadoop-rdf/hadoop-rdf-io/src/test/java/com/yarcdata/urika/hadoop/rdf/io/input/util/TrackedInputStreamTest.java
URL: http://svn.apache.org/viewvc/jena/Experimental/hadoop-rdf/hadoop-rdf-io/src/test/java/com/yarcdata/urika/hadoop/rdf/io/input/util/TrackedInputStreamTest.java?rev=1583942&view=auto
==============================================================================
--- jena/Experimental/hadoop-rdf/hadoop-rdf-io/src/test/java/com/yarcdata/urika/hadoop/rdf/io/input/util/TrackedInputStreamTest.java (added)
+++ jena/Experimental/hadoop-rdf/hadoop-rdf-io/src/test/java/com/yarcdata/urika/hadoop/rdf/io/input/util/TrackedInputStreamTest.java Wed Apr 2 09:20:51 2014
@@ -0,0 +1,22 @@
+/*
+ * Copyright 2013 YarcData LLC All Rights Reserved.
+ */
+
+package com.yarcdata.urika.hadoop.rdf.io.input.util;
+
+import java.io.InputStream;
+
+/**
+ * Tests for the {@link TrackedInputStream}
+ *
+ * @author rvesse
+ *
+ */
+public class TrackedInputStreamTest extends AbstractTrackableInputStreamTests {
+
+ @Override
+ protected TrackableInputStream getInstance(InputStream input) {
+ return new TrackedInputStream(input);
+ }
+
+}
Added: jena/Experimental/hadoop-rdf/hadoop-rdf-io/src/test/java/com/yarcdata/urika/hadoop/rdf/io/output/AbstractNodeTupleOutputFormatTests.java
URL: http://svn.apache.org/viewvc/jena/Experimental/hadoop-rdf/hadoop-rdf-io/src/test/java/com/yarcdata/urika/hadoop/rdf/io/output/AbstractNodeTupleOutputFormatTests.java?rev=1583942&view=auto
==============================================================================
--- jena/Experimental/hadoop-rdf/hadoop-rdf-io/src/test/java/com/yarcdata/urika/hadoop/rdf/io/output/AbstractNodeTupleOutputFormatTests.java (added)
+++ jena/Experimental/hadoop-rdf/hadoop-rdf-io/src/test/java/com/yarcdata/urika/hadoop/rdf/io/output/AbstractNodeTupleOutputFormatTests.java Wed Apr 2 09:20:51 2014
@@ -0,0 +1,233 @@
+/*
+ * Copyright 2013 YarcData LLC All Rights Reserved.
+ */
+
+package com.yarcdata.urika.hadoop.rdf.io.output;
+
+import java.io.File;
+import java.io.FileNotFoundException;
+import java.io.IOException;
+import java.util.Iterator;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.LocatedFileStatus;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.RemoteIterator;
+import org.apache.hadoop.io.NullWritable;
+import org.apache.hadoop.mapreduce.Job;
+import org.apache.hadoop.mapreduce.JobContext;
+import org.apache.hadoop.mapreduce.OutputFormat;
+import org.apache.hadoop.mapreduce.RecordWriter;
+import org.apache.hadoop.mapreduce.TaskAttemptContext;
+import org.apache.hadoop.mapreduce.TaskAttemptID;
+import org.apache.hadoop.mapreduce.TaskType;
+import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;
+import org.apache.hadoop.mapreduce.task.JobContextImpl;
+import org.apache.hadoop.mapreduce.task.TaskAttemptContextImpl;
+import org.apache.jena.riot.Lang;
+import org.apache.jena.riot.RDFDataMgr;
+import org.apache.jena.riot.lang.StreamRDFCounting;
+import org.apache.jena.riot.system.StreamRDFLib;
+import org.junit.Assert;
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.rules.TemporaryFolder;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import com.yarcdata.urika.hadoop.rdf.types.AbstractNodeTupleWritable;
+
+/**
+ * Abstract node tuple output format tests
+ *
+ * @author rvesse
+ * @param <TValue>
+ * Tuple type
+ * @param <T>
+ * Writable tuple type
+ *
+ */
+public abstract class AbstractNodeTupleOutputFormatTests<TValue, T extends AbstractNodeTupleWritable<TValue>> {
+
+ @SuppressWarnings("unused")
+ private static final Logger LOG = LoggerFactory.getLogger(AbstractNodeTupleOutputFormatTests.class);
+
+ protected static final int EMPTY_SIZE = 0, SMALL_SIZE = 100, LARGE_SIZE = 10000, VERY_LARGE_SIZE = 100000;
+
+ /**
+ * Temporary folder for the tests
+ */
+ @Rule
+ public TemporaryFolder folder = new TemporaryFolder();
+
+ /**
+ * Prepares a fresh configuration
+ *
+ * @return Configuration
+ */
+ protected Configuration prepareConfiguration() {
+ Configuration config = new Configuration(true);
+ // Nothing else to do
+ return config;
+ }
+
+ /**
+ * Gets the extra file extension to add to the filenames
+ *
+ * @return File extension
+ */
+ protected abstract String getFileExtension();
+
+ /**
+ * Generates tuples to be output for testing
+ *
+ * @param num
+ * Number of tuples to generate
+ * @return Iterator of tuples
+ */
+ protected abstract Iterator<T> generateTuples(int num);
+
+ /**
+ * Counts tuples in the output file
+ *
+ * @param f
+ * Output file
+ * @return Tuple count
+ */
+ protected final long countTuples(File f) {
+ StreamRDFCounting counter = StreamRDFLib.count();
+ RDFDataMgr.parse(counter, f.getAbsolutePath(), this.getRdfLanguage(), null);
+ return counter.count();
+ }
+
+ protected final void checkTuples(File f, long expected) {
+ Assert.assertEquals(expected, this.countTuples(f));
+ }
+
+ /**
+ * Gets the RDF language of the produced output which is used to parse back
+ * in the output to validate the correct amount of output was produced
+ *
+ * @return RDF language
+ */
+ protected abstract Lang getRdfLanguage();
+
+ /**
+ * Gets the output format to test
+ *
+ * @return Output format
+ */
+ protected abstract OutputFormat<NullWritable, T> getOutputFormat();
+
+ /**
+ * Adds an output path to the job configuration
+ *
+ * @param f
+ * File
+ * @param config
+ * Configuration
+ * @param job
+ * Job
+ * @throws IOException
+ */
+ protected void addOutputPath(File f, Configuration config, Job job) throws IOException {
+ FileSystem fs = FileSystem.getLocal(config);
+ Path outputPath = fs.makeQualified(new Path(f.getAbsolutePath()));
+ FileOutputFormat.setOutputPath(job, outputPath);
+ }
+
+ protected File findOutputFile(File dir, JobContext context) throws FileNotFoundException, IOException {
+ Path outputPath = FileOutputFormat.getOutputPath(context);
+ RemoteIterator<LocatedFileStatus> files = outputPath.getFileSystem(context.getConfiguration()).listFiles(outputPath, true);
+ while (files.hasNext()) {
+ LocatedFileStatus status = files.next();
+ if (status.isFile() && !status.getPath().getName().startsWith("_")) {
+ return new File(status.getPath().toUri());
+ }
+ }
+ return null;
+ }
+
+ /**
+ * Tests output
+ *
+ * @param f
+ * File to output to
+ * @param num
+ * Number of tuples to output
+ * @throws IOException
+ * @throws InterruptedException
+ */
+ protected final void testOutput(File f, int num) throws IOException, InterruptedException {
+ // Prepare configuration
+ Configuration config = this.prepareConfiguration();
+
+ // Set up fake job
+ OutputFormat<NullWritable, T> outputFormat = this.getOutputFormat();
+ Job job = Job.getInstance(config);
+ job.setOutputFormatClass(outputFormat.getClass());
+ this.addOutputPath(f, job.getConfiguration(), job);
+ JobContext context = new JobContextImpl(job.getConfiguration(), job.getJobID());
+ Assert.assertNotNull(FileOutputFormat.getOutputPath(context));
+
+ // Output the data
+ TaskAttemptID id = new TaskAttemptID("outputTest", 1, TaskType.MAP, 1, 1);
+ TaskAttemptContext taskContext = new TaskAttemptContextImpl(job.getConfiguration(), id);
+ RecordWriter<NullWritable, T> writer = outputFormat.getRecordWriter(taskContext);
+ Iterator<T> tuples = this.generateTuples(num);
+ while (tuples.hasNext()) {
+ writer.write(NullWritable.get(), tuples.next());
+ }
+ writer.close(taskContext);
+
+ // Check output
+ File outputFile = this.findOutputFile(this.folder.getRoot(), context);
+ Assert.assertNotNull(outputFile);
+ this.checkTuples(outputFile, num);
+ }
+
+ /**
+ * Basic output tests
+ *
+ * @throws IOException
+ * @throws InterruptedException
+ */
+ @Test
+ public final void output_01() throws IOException, InterruptedException {
+ this.testOutput(this.folder.getRoot(), EMPTY_SIZE);
+ }
+
+ /**
+ * Basic output tests
+ *
+ * @throws IOException
+ * @throws InterruptedException
+ */
+ @Test
+ public final void output_02() throws IOException, InterruptedException {
+ this.testOutput(this.folder.getRoot(), SMALL_SIZE);
+ }
+
+ /**
+ * Basic output tests
+ *
+ * @throws IOException
+ * @throws InterruptedException
+ */
+ @Test
+ public final void output_03() throws IOException, InterruptedException {
+ this.testOutput(this.folder.getRoot(), LARGE_SIZE);
+ }
+
+ /**
+ * Basic output tests
+ *
+ * @throws IOException
+ * @throws InterruptedException
+ */
+ @Test
+ public final void output_04() throws IOException, InterruptedException {
+ this.testOutput(this.folder.getRoot(), VERY_LARGE_SIZE);
+ }
+}
Added: jena/Experimental/hadoop-rdf/hadoop-rdf-io/src/test/java/com/yarcdata/urika/hadoop/rdf/io/output/AbstractQuadOutputFormatTests.java
URL: http://svn.apache.org/viewvc/jena/Experimental/hadoop-rdf/hadoop-rdf-io/src/test/java/com/yarcdata/urika/hadoop/rdf/io/output/AbstractQuadOutputFormatTests.java?rev=1583942&view=auto
==============================================================================
--- jena/Experimental/hadoop-rdf/hadoop-rdf-io/src/test/java/com/yarcdata/urika/hadoop/rdf/io/output/AbstractQuadOutputFormatTests.java (added)
+++ jena/Experimental/hadoop-rdf/hadoop-rdf-io/src/test/java/com/yarcdata/urika/hadoop/rdf/io/output/AbstractQuadOutputFormatTests.java Wed Apr 2 09:20:51 2014
@@ -0,0 +1,36 @@
+/*
+ * Copyright 2013 YarcData LLC All Rights Reserved.
+ */
+
+package com.yarcdata.urika.hadoop.rdf.io.output;
+
+import java.util.ArrayList;
+import java.util.Iterator;
+import java.util.List;
+
+import com.hp.hpl.jena.datatypes.xsd.XSDDatatype;
+import com.hp.hpl.jena.graph.NodeFactory;
+import com.hp.hpl.jena.sparql.core.Quad;
+import com.yarcdata.urika.hadoop.rdf.types.QuadWritable;
+
+/**
+ * Abstract tests for quad output formats
+ *
+ * @author rvesse
+ *
+ */
+public abstract class AbstractQuadOutputFormatTests extends AbstractNodeTupleOutputFormatTests<Quad, QuadWritable> {
+
+ @Override
+ protected Iterator<QuadWritable> generateTuples(int num) {
+ List<QuadWritable> qs = new ArrayList<QuadWritable>();
+ for (int i = 0; i < num; i++) {
+ Quad q = new Quad(NodeFactory.createURI("http://example.org/graphs/" + i),
+ NodeFactory.createURI("http://example.org/subjects/" + i),
+ NodeFactory.createURI("http://example.org/predicate"), NodeFactory.createLiteral(Integer.toString(i),
+ XSDDatatype.XSDinteger));
+ qs.add(new QuadWritable(q));
+ }
+ return qs.iterator();
+ }
+}
Added: jena/Experimental/hadoop-rdf/hadoop-rdf-io/src/test/java/com/yarcdata/urika/hadoop/rdf/io/output/AbstractTripleOutputFormatTests.java
URL: http://svn.apache.org/viewvc/jena/Experimental/hadoop-rdf/hadoop-rdf-io/src/test/java/com/yarcdata/urika/hadoop/rdf/io/output/AbstractTripleOutputFormatTests.java?rev=1583942&view=auto
==============================================================================
--- jena/Experimental/hadoop-rdf/hadoop-rdf-io/src/test/java/com/yarcdata/urika/hadoop/rdf/io/output/AbstractTripleOutputFormatTests.java (added)
+++ jena/Experimental/hadoop-rdf/hadoop-rdf-io/src/test/java/com/yarcdata/urika/hadoop/rdf/io/output/AbstractTripleOutputFormatTests.java Wed Apr 2 09:20:51 2014
@@ -0,0 +1,32 @@
+/*
+ * Copyright 2013 YarcData LLC All Rights Reserved.
+ */
+
+package com.yarcdata.urika.hadoop.rdf.io.output;
+
+import java.util.ArrayList;
+import java.util.Iterator;
+import java.util.List;
+
+import com.hp.hpl.jena.datatypes.xsd.XSDDatatype;
+import com.hp.hpl.jena.graph.NodeFactory;
+import com.hp.hpl.jena.graph.Triple;
+import com.yarcdata.urika.hadoop.rdf.types.TripleWritable;
+
+/**
+ * Abstract tests for triple output formats
+ * @author rvesse
+ *
+ */
+public abstract class AbstractTripleOutputFormatTests extends AbstractNodeTupleOutputFormatTests<Triple, TripleWritable> {
+
+ @Override
+ protected Iterator<TripleWritable> generateTuples(int num) {
+ List<TripleWritable> ts = new ArrayList<TripleWritable>();
+ for (int i = 0; i < num; i++) {
+ Triple t = new Triple(NodeFactory.createURI("http://example.org/subjects/" + i), NodeFactory.createURI("http://example.org/predicate"), NodeFactory.createLiteral(Integer.toString(i), XSDDatatype.XSDinteger));
+ ts.add(new TripleWritable(t));
+ }
+ return ts.iterator();
+ }
+}
Added: jena/Experimental/hadoop-rdf/hadoop-rdf-io/src/test/java/com/yarcdata/urika/hadoop/rdf/io/output/NQuadsOutputTest.java
URL: http://svn.apache.org/viewvc/jena/Experimental/hadoop-rdf/hadoop-rdf-io/src/test/java/com/yarcdata/urika/hadoop/rdf/io/output/NQuadsOutputTest.java?rev=1583942&view=auto
==============================================================================
--- jena/Experimental/hadoop-rdf/hadoop-rdf-io/src/test/java/com/yarcdata/urika/hadoop/rdf/io/output/NQuadsOutputTest.java (added)
+++ jena/Experimental/hadoop-rdf/hadoop-rdf-io/src/test/java/com/yarcdata/urika/hadoop/rdf/io/output/NQuadsOutputTest.java Wed Apr 2 09:20:51 2014
@@ -0,0 +1,36 @@
+/*
+ * Copyright 2013 YarcData LLC All Rights Reserved.
+ */
+
+package com.yarcdata.urika.hadoop.rdf.io.output;
+
+import org.apache.hadoop.io.NullWritable;
+import org.apache.hadoop.mapreduce.OutputFormat;
+import org.apache.jena.riot.Lang;
+
+import com.yarcdata.urika.hadoop.rdf.types.QuadWritable;
+
+/**
+ * Tests for NQuads output format
+ *
+ * @author rvesse
+ *
+ */
+public class NQuadsOutputTest extends AbstractQuadOutputFormatTests {
+
+ @Override
+ protected String getFileExtension() {
+ return ".nq";
+ }
+
+ @Override
+ protected Lang getRdfLanguage() {
+ return Lang.NQUADS;
+ }
+
+ @Override
+ protected OutputFormat<NullWritable, QuadWritable> getOutputFormat() {
+ return new NQuadsOutputFormat<NullWritable>();
+ }
+
+}
Added: jena/Experimental/hadoop-rdf/hadoop-rdf-io/src/test/java/com/yarcdata/urika/hadoop/rdf/io/output/NTriplesOutputTest.java
URL: http://svn.apache.org/viewvc/jena/Experimental/hadoop-rdf/hadoop-rdf-io/src/test/java/com/yarcdata/urika/hadoop/rdf/io/output/NTriplesOutputTest.java?rev=1583942&view=auto
==============================================================================
--- jena/Experimental/hadoop-rdf/hadoop-rdf-io/src/test/java/com/yarcdata/urika/hadoop/rdf/io/output/NTriplesOutputTest.java (added)
+++ jena/Experimental/hadoop-rdf/hadoop-rdf-io/src/test/java/com/yarcdata/urika/hadoop/rdf/io/output/NTriplesOutputTest.java Wed Apr 2 09:20:51 2014
@@ -0,0 +1,36 @@
+/*
+ * Copyright 2013 YarcData LLC All Rights Reserved.
+ */
+
+package com.yarcdata.urika.hadoop.rdf.io.output;
+
+import org.apache.hadoop.io.NullWritable;
+import org.apache.hadoop.mapreduce.OutputFormat;
+import org.apache.jena.riot.Lang;
+
+import com.yarcdata.urika.hadoop.rdf.types.TripleWritable;
+
+/**
+ * Tests for NTriples output format
+ *
+ * @author rvesse
+ *
+ */
+public class NTriplesOutputTest extends AbstractTripleOutputFormatTests {
+
+ @Override
+ protected String getFileExtension() {
+ return ".nt";
+ }
+
+ @Override
+ protected Lang getRdfLanguage() {
+ return Lang.NTRIPLES;
+ }
+
+ @Override
+ protected OutputFormat<NullWritable, TripleWritable> getOutputFormat() {
+ return new NTriplesOutputFormat<NullWritable>();
+ }
+
+}
Added: jena/Experimental/hadoop-rdf/hadoop-rdf-io/src/test/java/com/yarcdata/urika/hadoop/rdf/io/output/RdfJsonOutputTest.java
URL: http://svn.apache.org/viewvc/jena/Experimental/hadoop-rdf/hadoop-rdf-io/src/test/java/com/yarcdata/urika/hadoop/rdf/io/output/RdfJsonOutputTest.java?rev=1583942&view=auto
==============================================================================
--- jena/Experimental/hadoop-rdf/hadoop-rdf-io/src/test/java/com/yarcdata/urika/hadoop/rdf/io/output/RdfJsonOutputTest.java (added)
+++ jena/Experimental/hadoop-rdf/hadoop-rdf-io/src/test/java/com/yarcdata/urika/hadoop/rdf/io/output/RdfJsonOutputTest.java Wed Apr 2 09:20:51 2014
@@ -0,0 +1,36 @@
+/*
+ * Copyright 2013 YarcData LLC All Rights Reserved.
+ */
+
+package com.yarcdata.urika.hadoop.rdf.io.output;
+
+import org.apache.hadoop.io.NullWritable;
+import org.apache.hadoop.mapreduce.OutputFormat;
+import org.apache.jena.riot.Lang;
+
+import com.yarcdata.urika.hadoop.rdf.types.TripleWritable;
+
+/**
+ * Tests for RDF/JSON output
+ *
+ * @author rvesse
+ *
+ */
+public class RdfJsonOutputTest extends AbstractTripleOutputFormatTests {
+
+ @Override
+ protected String getFileExtension() {
+ return ".rj";
+ }
+
+ @Override
+ protected Lang getRdfLanguage() {
+ return Lang.RDFJSON;
+ }
+
+ @Override
+ protected OutputFormat<NullWritable, TripleWritable> getOutputFormat() {
+ return new RdfJsonOutputFormat<NullWritable>();
+ }
+
+}
Added: jena/Experimental/hadoop-rdf/hadoop-rdf-io/src/test/java/com/yarcdata/urika/hadoop/rdf/io/output/RdfXmlOutputTest.java
URL: http://svn.apache.org/viewvc/jena/Experimental/hadoop-rdf/hadoop-rdf-io/src/test/java/com/yarcdata/urika/hadoop/rdf/io/output/RdfXmlOutputTest.java?rev=1583942&view=auto
==============================================================================
--- jena/Experimental/hadoop-rdf/hadoop-rdf-io/src/test/java/com/yarcdata/urika/hadoop/rdf/io/output/RdfXmlOutputTest.java (added)
+++ jena/Experimental/hadoop-rdf/hadoop-rdf-io/src/test/java/com/yarcdata/urika/hadoop/rdf/io/output/RdfXmlOutputTest.java Wed Apr 2 09:20:51 2014
@@ -0,0 +1,36 @@
+/*
+ * Copyright 2013 YarcData LLC All Rights Reserved.
+ */
+
+package com.yarcdata.urika.hadoop.rdf.io.output;
+
+import org.apache.hadoop.io.NullWritable;
+import org.apache.hadoop.mapreduce.OutputFormat;
+import org.apache.jena.riot.Lang;
+
+import com.yarcdata.urika.hadoop.rdf.types.TripleWritable;
+
+/**
+ * Tests for RDF/XML output
+ *
+ * @author rvesse
+ *
+ */
+public class RdfXmlOutputTest extends AbstractTripleOutputFormatTests {
+
+ @Override
+ protected String getFileExtension() {
+ return ".rdf";
+ }
+
+ @Override
+ protected Lang getRdfLanguage() {
+ return Lang.RDFXML;
+ }
+
+ @Override
+ protected OutputFormat<NullWritable, TripleWritable> getOutputFormat() {
+ return new RdfXmlOutputFormat<NullWritable>();
+ }
+
+}
Added: jena/Experimental/hadoop-rdf/hadoop-rdf-io/src/test/java/com/yarcdata/urika/hadoop/rdf/io/output/TriGOutputTest.java
URL: http://svn.apache.org/viewvc/jena/Experimental/hadoop-rdf/hadoop-rdf-io/src/test/java/com/yarcdata/urika/hadoop/rdf/io/output/TriGOutputTest.java?rev=1583942&view=auto
==============================================================================
--- jena/Experimental/hadoop-rdf/hadoop-rdf-io/src/test/java/com/yarcdata/urika/hadoop/rdf/io/output/TriGOutputTest.java (added)
+++ jena/Experimental/hadoop-rdf/hadoop-rdf-io/src/test/java/com/yarcdata/urika/hadoop/rdf/io/output/TriGOutputTest.java Wed Apr 2 09:20:51 2014
@@ -0,0 +1,77 @@
+/*
+ * Copyright 2013 YarcData LLC All Rights Reserved.
+ */
+
+package com.yarcdata.urika.hadoop.rdf.io.output;
+
+import java.util.Arrays;
+import java.util.Collection;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.io.NullWritable;
+import org.apache.hadoop.mapreduce.OutputFormat;
+import org.apache.jena.riot.Lang;
+import org.junit.runner.RunWith;
+import org.junit.runners.Parameterized;
+import org.junit.runners.Parameterized.Parameters;
+
+import com.yarcdata.urika.hadoop.rdf.io.RdfIOConstants;
+import com.yarcdata.urika.hadoop.rdf.types.QuadWritable;
+
+/**
+ * Tests for TriG output
+ *
+ * @author rvesse
+ *
+ */
+@RunWith(Parameterized.class)
+public class TriGOutputTest extends AbstractQuadOutputFormatTests {
+
+ static long $bs1 = RdfIOConstants.DEFAULT_OUTPUT_BATCH_SIZE;
+ static long $bs2 = 1000;
+ static long $bs3 = 100;
+ static long $bs4 = 1;
+
+ /**
+ * @return Test parameters
+ */
+ @Parameters
+ public static Collection<Object[]> data() {
+ return Arrays.asList(new Object[][] { { $bs1 }, { $bs2 }, { $bs3 }, { $bs4 } });
+ }
+
+ private final long batchSize;
+
+ /**
+ * Creates new tests
+ *
+ * @param batchSize
+ * Batch size
+ */
+ public TriGOutputTest(long batchSize) {
+ this.batchSize = batchSize;
+ }
+
+ @Override
+ protected String getFileExtension() {
+ return ".trig";
+ }
+
+ @Override
+ protected Lang getRdfLanguage() {
+ return Lang.TRIG;
+ }
+
+ @Override
+ protected Configuration prepareConfiguration() {
+ Configuration config = super.prepareConfiguration();
+ config.setLong(RdfIOConstants.OUTPUT_BATCH_SIZE, this.batchSize);
+ return config;
+ }
+
+ @Override
+ protected OutputFormat<NullWritable, QuadWritable> getOutputFormat() {
+ return new TriGOutputFormat<NullWritable>();
+ }
+
+}
Added: jena/Experimental/hadoop-rdf/hadoop-rdf-io/src/test/java/com/yarcdata/urika/hadoop/rdf/io/output/TurtleOutputTest.java
URL: http://svn.apache.org/viewvc/jena/Experimental/hadoop-rdf/hadoop-rdf-io/src/test/java/com/yarcdata/urika/hadoop/rdf/io/output/TurtleOutputTest.java?rev=1583942&view=auto
==============================================================================
--- jena/Experimental/hadoop-rdf/hadoop-rdf-io/src/test/java/com/yarcdata/urika/hadoop/rdf/io/output/TurtleOutputTest.java (added)
+++ jena/Experimental/hadoop-rdf/hadoop-rdf-io/src/test/java/com/yarcdata/urika/hadoop/rdf/io/output/TurtleOutputTest.java Wed Apr 2 09:20:51 2014
@@ -0,0 +1,77 @@
+/*
+ * Copyright 2013 YarcData LLC All Rights Reserved.
+ */
+
+package com.yarcdata.urika.hadoop.rdf.io.output;
+
+import java.util.Arrays;
+import java.util.Collection;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.io.NullWritable;
+import org.apache.hadoop.mapreduce.OutputFormat;
+import org.apache.jena.riot.Lang;
+import org.junit.runner.RunWith;
+import org.junit.runners.Parameterized;
+import org.junit.runners.Parameterized.Parameters;
+
+import com.yarcdata.urika.hadoop.rdf.io.RdfIOConstants;
+import com.yarcdata.urika.hadoop.rdf.types.TripleWritable;
+
+/**
+ * Tests for Turtle output
+ *
+ * @author rvesse
+ *
+ */
+@RunWith(Parameterized.class)
+public class TurtleOutputTest extends AbstractTripleOutputFormatTests {
+
+ static long $bs1 = RdfIOConstants.DEFAULT_OUTPUT_BATCH_SIZE;
+ static long $bs2 = 1000;
+ static long $bs3 = 100;
+ static long $bs4 = 1;
+
+ /**
+ * @return Test parameters
+ */
+ @Parameters
+ public static Collection<Object[]> data() {
+ return Arrays.asList(new Object[][] { { $bs1 }, { $bs2 }, { $bs3 }, { $bs4 } });
+ }
+
+ private final long batchSize;
+
+ /**
+ * Creates new tests
+ *
+ * @param batchSize
+ * Batch size
+ */
+ public TurtleOutputTest(long batchSize) {
+ this.batchSize = batchSize;
+ }
+
+ @Override
+ protected String getFileExtension() {
+ return ".ttl";
+ }
+
+ @Override
+ protected Lang getRdfLanguage() {
+ return Lang.TURTLE;
+ }
+
+ @Override
+ protected Configuration prepareConfiguration() {
+ Configuration config = super.prepareConfiguration();
+ config.setLong(RdfIOConstants.OUTPUT_BATCH_SIZE, this.batchSize);
+ return config;
+ }
+
+ @Override
+ protected OutputFormat<NullWritable, TripleWritable> getOutputFormat() {
+ return new TurtleOutputFormat<NullWritable>();
+ }
+
+}
Propchange: jena/Experimental/hadoop-rdf/hadoop-rdf-mapreduce/
------------------------------------------------------------------------------
--- svn:ignore (added)
+++ svn:ignore Wed Apr 2 09:20:51 2014
@@ -0,0 +1,4 @@
+.classpath
+.project
+.settings
+target
Added: jena/Experimental/hadoop-rdf/hadoop-rdf-mapreduce/pom.xml
URL: http://svn.apache.org/viewvc/jena/Experimental/hadoop-rdf/hadoop-rdf-mapreduce/pom.xml?rev=1583942&view=auto
==============================================================================
--- jena/Experimental/hadoop-rdf/hadoop-rdf-mapreduce/pom.xml (added)
+++ jena/Experimental/hadoop-rdf/hadoop-rdf-mapreduce/pom.xml Wed Apr 2 09:20:51 2014
@@ -0,0 +1,71 @@
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+ <modelVersion>4.0.0</modelVersion>
+ <parent>
+ <groupId>com.yarcdata.urika</groupId>
+ <artifactId>hadoop-rdf</artifactId>
+ <version>0.0.1-SNAPSHOT</version>
+ </parent>
+ <artifactId>hadoop-rdf-mapreduce</artifactId>
+ <name>Hadoop RDF Map/Reduce</name>
+ <description>Contains some basic Map/Reduce implementations for working with RDF on Hadoop</description>
+
+ <dependencies>
+ <!-- Internal Project Dependencies -->
+ <dependency>
+ <groupId>com.yarcdata.urika</groupId>
+ <artifactId>hadoop-rdf-common</artifactId>
+ <version>${project.version}</version>
+ </dependency>
+
+ <!-- Hadoop Dependencies -->
+ <!-- Note these will be provided on the Hadoop cluster hence the provided
+ scope -->
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-common</artifactId>
+ <scope>provided</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-mapreduce-client-common</artifactId>
+ <scope>provided</scope>
+ </dependency>
+
+ <!-- Jena dependencies -->
+ <dependency>
+ <groupId>org.apache.jena</groupId>
+ <artifactId>jena-arq</artifactId>
+ </dependency>
+
+ <!-- Test Dependencies -->
+ <dependency>
+ <groupId>junit</groupId>
+ <artifactId>junit</artifactId>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.mrunit</groupId>
+ <artifactId>mrunit</artifactId>
+ <scope>test</scope>
+ <classifier>hadoop2</classifier>
+ </dependency>
+ </dependencies>
+
+ <build>
+ <plugins>
+ <!-- JAR plugin to ensure tests jar is built -->
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-jar-plugin</artifactId>
+ <executions>
+ <execution>
+ <goals>
+ <goal>test-jar</goal>
+ </goals>
+ </execution>
+ </executions>
+ </plugin>
+ </plugins>
+ </build>
+</project>
\ No newline at end of file
Added: jena/Experimental/hadoop-rdf/hadoop-rdf-mapreduce/src/main/java/com/yarcdata/urika/hadoop/rdf/mapreduce/KeyMapper.java
URL: http://svn.apache.org/viewvc/jena/Experimental/hadoop-rdf/hadoop-rdf-mapreduce/src/main/java/com/yarcdata/urika/hadoop/rdf/mapreduce/KeyMapper.java?rev=1583942&view=auto
==============================================================================
--- jena/Experimental/hadoop-rdf/hadoop-rdf-mapreduce/src/main/java/com/yarcdata/urika/hadoop/rdf/mapreduce/KeyMapper.java (added)
+++ jena/Experimental/hadoop-rdf/hadoop-rdf-mapreduce/src/main/java/com/yarcdata/urika/hadoop/rdf/mapreduce/KeyMapper.java Wed Apr 2 09:20:51 2014
@@ -0,0 +1,40 @@
+/*
+ * Copyright 2013 YarcData LLC All Rights Reserved.
+ */
+
+package com.yarcdata.urika.hadoop.rdf.mapreduce;
+
+import java.io.IOException;
+
+import org.apache.hadoop.mapreduce.Mapper;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * A mapper which discards the value replacing it with the key
+ * @author rvesse
+ *
+ * @param <TKey> Key type
+ * @param <TValue> Value type
+ */
+public class KeyMapper<TKey, TValue> extends Mapper<TKey, TValue, TKey, TKey> {
+ private static final Logger LOG = LoggerFactory.getLogger(KeyMapper.class);
+
+ private boolean tracing = false;
+
+ @Override
+ protected void setup(Context context) throws IOException, InterruptedException {
+ super.setup(context);
+ this.tracing = LOG.isTraceEnabled();
+ }
+
+ @Override
+ protected void map(TKey key, TValue value, Context context) throws IOException,
+ InterruptedException {
+ if (this.tracing) {
+ LOG.trace("Key = {}", key);
+ }
+ context.write(key, key);
+ }
+
+}
Added: jena/Experimental/hadoop-rdf/hadoop-rdf-mapreduce/src/main/java/com/yarcdata/urika/hadoop/rdf/mapreduce/RdfMapReduceConstants.java
URL: http://svn.apache.org/viewvc/jena/Experimental/hadoop-rdf/hadoop-rdf-mapreduce/src/main/java/com/yarcdata/urika/hadoop/rdf/mapreduce/RdfMapReduceConstants.java?rev=1583942&view=auto
==============================================================================
--- jena/Experimental/hadoop-rdf/hadoop-rdf-mapreduce/src/main/java/com/yarcdata/urika/hadoop/rdf/mapreduce/RdfMapReduceConstants.java (added)
+++ jena/Experimental/hadoop-rdf/hadoop-rdf-mapreduce/src/main/java/com/yarcdata/urika/hadoop/rdf/mapreduce/RdfMapReduceConstants.java Wed Apr 2 09:20:51 2014
@@ -0,0 +1,53 @@
+/*
+ * Copyright 2013 YarcData LLC All Rights Reserved.
+ */
+
+package com.yarcdata.urika.hadoop.rdf.mapreduce;
+
+/**
+ * RDF Map/Reduce related constants
+ *
+ * @author rvesse
+ *
+ */
+public class RdfMapReduceConstants {
+
+ /**
+ * Private constructor prevents instantiation
+ */
+ private RdfMapReduceConstants() {
+
+ }
+
+ /**
+ * Configuration key used to set whether the behaviour of the filter mappers
+ * is inverted. When enabled the filter mappers will invert their selection
+ * i.e. tuples that would normally be accepted will be rejected and vice
+ * versa.
+ */
+ public static final String FILTER_INVERT = "rdf.mapreduce.filter.invert";
+
+ /**
+ * Configuration key used to set a command separated list of predicate URIs
+ * to filter upon
+ */
+ public static final String FILTER_PREDICATE_URIS = "rdf.mapreduce.filter.predicate.uris";
+
+ /**
+ * Configuration key used to set a command separated list of subject URIs to
+ * filter upon
+ */
+ public static final String FILTER_SUBJECT_URIS = "rdf.mapreduce.filter.subject.uris";
+
+ /**
+ * Configuration key used to set a command separated list of object URIs to
+ * filter upon
+ */
+ public static final String FILTER_OBJECT_URIS = "rdf.mapreduce.filter.object.uris";
+
+ /**
+ * Configuration key used to set a command separated list of graph URIs to
+ * filter upon
+ */
+ public static final String FILTER_GRAPH_URIS = "rdf.mapreduce.filter.graph.uris";
+}
Added: jena/Experimental/hadoop-rdf/hadoop-rdf-mapreduce/src/main/java/com/yarcdata/urika/hadoop/rdf/mapreduce/TextCountReducer.java
URL: http://svn.apache.org/viewvc/jena/Experimental/hadoop-rdf/hadoop-rdf-mapreduce/src/main/java/com/yarcdata/urika/hadoop/rdf/mapreduce/TextCountReducer.java?rev=1583942&view=auto
==============================================================================
--- jena/Experimental/hadoop-rdf/hadoop-rdf-mapreduce/src/main/java/com/yarcdata/urika/hadoop/rdf/mapreduce/TextCountReducer.java (added)
+++ jena/Experimental/hadoop-rdf/hadoop-rdf-mapreduce/src/main/java/com/yarcdata/urika/hadoop/rdf/mapreduce/TextCountReducer.java Wed Apr 2 09:20:51 2014
@@ -0,0 +1,35 @@
+/*
+ * Copyright 2013 YarcData LLC All Rights Reserved.
+ */
+
+package com.yarcdata.urika.hadoop.rdf.mapreduce;
+
+import java.io.IOException;
+import java.util.Iterator;
+
+import org.apache.hadoop.io.LongWritable;
+import org.apache.hadoop.io.Text;
+import org.apache.hadoop.mapreduce.Reducer;
+
+/**
+ * A reducer which takes text keys with a sequence of longs representing counts
+ * as the values and sums the counts together into pairs consisting of a node
+ * key and a count value.
+ *
+ * @author rvesse
+ *
+ */
+public class TextCountReducer extends Reducer<Text, LongWritable, Text, LongWritable> {
+
+ @Override
+ protected void reduce(Text key, Iterable<LongWritable> values, Context context) throws IOException,
+ InterruptedException {
+ long count = 0;
+ Iterator<LongWritable> iter = values.iterator();
+ while (iter.hasNext()) {
+ count += iter.next().get();
+ }
+ context.write(key, new LongWritable(count));
+ }
+
+}
Added: jena/Experimental/hadoop-rdf/hadoop-rdf-mapreduce/src/main/java/com/yarcdata/urika/hadoop/rdf/mapreduce/characteristics/AbstractCharacteristicSetGeneratingReducer.java
URL: http://svn.apache.org/viewvc/jena/Experimental/hadoop-rdf/hadoop-rdf-mapreduce/src/main/java/com/yarcdata/urika/hadoop/rdf/mapreduce/characteristics/AbstractCharacteristicSetGeneratingReducer.java?rev=1583942&view=auto
==============================================================================
--- jena/Experimental/hadoop-rdf/hadoop-rdf-mapreduce/src/main/java/com/yarcdata/urika/hadoop/rdf/mapreduce/characteristics/AbstractCharacteristicSetGeneratingReducer.java (added)
+++ jena/Experimental/hadoop-rdf/hadoop-rdf-mapreduce/src/main/java/com/yarcdata/urika/hadoop/rdf/mapreduce/characteristics/AbstractCharacteristicSetGeneratingReducer.java Wed Apr 2 09:20:51 2014
@@ -0,0 +1,165 @@
+/*
+ * Copyright 2013 YarcData LLC All Rights Reserved.
+ */
+
+package com.yarcdata.urika.hadoop.rdf.mapreduce.characteristics;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
+import java.util.TreeMap;
+
+import org.apache.hadoop.io.NullWritable;
+import org.apache.hadoop.mapreduce.Reducer;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import com.yarcdata.urika.hadoop.rdf.types.CharacteristicSetWritable;
+import com.yarcdata.urika.hadoop.rdf.types.CharacteristicWritable;
+import com.yarcdata.urika.hadoop.rdf.types.AbstractNodeTupleWritable;
+import com.yarcdata.urika.hadoop.rdf.types.NodeWritable;
+
+/**
+ * Abstract reducer which takes in tuples grouped by some node and generating
+ * initial characteristic sets.
+ * <p>
+ * This produces the characteristic sets as both the key and value so that in a
+ * subsequent job the characteristic steps may be further combined together to
+ * total up the usage counts appropriately.
+ * </p>
+ * <p>
+ * It is important to note that the output from this mapper can be very large
+ * and since it typically needs to be written to HDFS before being processed by
+ * further jobs it is strongly recommended that you use appropriate output
+ * compression
+ * </p>
+ *
+ * @author rvesse
+ *
+ * @param <TValue>
+ * Tuple type
+ * @param <T>
+ * Writable tuple type
+ */
+public abstract class AbstractCharacteristicSetGeneratingReducer<TValue, T extends AbstractNodeTupleWritable<TValue>> extends
+ Reducer<NodeWritable, T, CharacteristicSetWritable, NullWritable> {
+
+ private static final Logger LOG = LoggerFactory.getLogger(AbstractCharacteristicSetGeneratingReducer.class);
+
+ private boolean tracing = false;
+
+ @Override
+ protected void setup(Context context) throws IOException, InterruptedException {
+ super.setup(context);
+ this.tracing = LOG.isTraceEnabled();
+ }
+
+ @Override
+ protected void reduce(NodeWritable key, Iterable<T> values, Context context) throws IOException, InterruptedException {
+ Map<NodeWritable, CharacteristicWritable> characteristics = new TreeMap<NodeWritable, CharacteristicWritable>();
+
+ // Firstly need to find individual characteristics
+ Iterator<T> iter = values.iterator();
+ while (iter.hasNext()) {
+ T tuple = iter.next();
+ NodeWritable predicate = this.getPredicate(tuple);
+
+ if (characteristics.containsKey(predicate)) {
+ characteristics.get(predicate).increment();
+ } else {
+ characteristics.put(predicate, new CharacteristicWritable(predicate.get()));
+ }
+ }
+
+ // Then we need to produce all the possible characteristic sets based on
+ // this information
+ List<CharacteristicWritable> cs = new ArrayList<CharacteristicWritable>(characteristics.values());
+ if (cs.size() == 0)
+ return;
+ for (int i = 1; i <= cs.size(); i++) {
+ this.outputSets(cs, i, context);
+ }
+ }
+
+ /**
+ * Output all sets of a given size
+ *
+ * @param cs
+ * Characteristics
+ * @param perSet
+ * Set size
+ * @param context
+ * Context to output sets to
+ * @throws IOException
+ * @throws InterruptedException
+ */
+ protected void outputSets(List<CharacteristicWritable> cs, int perSet, Context context) throws IOException,
+ InterruptedException {
+ if (perSet == 1) {
+ for (CharacteristicWritable c : cs) {
+ CharacteristicSetWritable set = new CharacteristicSetWritable(c);
+ context.write(set, NullWritable.get());
+ if (this.tracing) {
+ LOG.trace("Key = {}", set);
+ }
+ }
+ } else if (perSet == cs.size()) {
+ CharacteristicSetWritable set = new CharacteristicSetWritable();
+ for (CharacteristicWritable c : cs) {
+ set.add(c);
+ }
+ context.write(set, NullWritable.get());
+ if (this.tracing) {
+ LOG.trace("Key = {}", set);
+ }
+ } else {
+ CharacteristicWritable[] members = new CharacteristicWritable[perSet];
+ this.combinations(cs, perSet, 0, members, context);
+ }
+ }
+
+ /**
+ * Calculate all available combinations of N elements from the given
+ * characteristics
+ *
+ * @param cs
+ * Characteristics
+ * @param len
+ * Desired number of elements
+ * @param startPosition
+ * Start position
+ * @param result
+ * Result array to fill
+ * @param context
+ * Context to write completed combinations to
+ * @throws IOException
+ * @throws InterruptedException
+ */
+ protected final void combinations(List<CharacteristicWritable> cs, int len, int startPosition,
+ CharacteristicWritable[] result, Context context) throws IOException, InterruptedException {
+ if (len == 0) {
+ CharacteristicSetWritable set = new CharacteristicSetWritable(result);
+ context.write(set, NullWritable.get());
+ if (this.tracing) {
+ LOG.trace("Key = {}", set);
+ }
+ return;
+ }
+ for (int i = startPosition; i <= cs.size() - len; i++) {
+ result[result.length - len] = cs.get(i);
+ combinations(cs, len - 1, i + 1, result, context);
+ }
+ }
+
+ /**
+ * Gets the predicate for the tuple
+ *
+ * @param tuple
+ * Tuple
+ * @return
+ */
+ protected abstract NodeWritable getPredicate(T tuple);
+
+}
Added: jena/Experimental/hadoop-rdf/hadoop-rdf-mapreduce/src/main/java/com/yarcdata/urika/hadoop/rdf/mapreduce/characteristics/CharacteristicSetReducer.java
URL: http://svn.apache.org/viewvc/jena/Experimental/hadoop-rdf/hadoop-rdf-mapreduce/src/main/java/com/yarcdata/urika/hadoop/rdf/mapreduce/characteristics/CharacteristicSetReducer.java?rev=1583942&view=auto
==============================================================================
--- jena/Experimental/hadoop-rdf/hadoop-rdf-mapreduce/src/main/java/com/yarcdata/urika/hadoop/rdf/mapreduce/characteristics/CharacteristicSetReducer.java (added)
+++ jena/Experimental/hadoop-rdf/hadoop-rdf-mapreduce/src/main/java/com/yarcdata/urika/hadoop/rdf/mapreduce/characteristics/CharacteristicSetReducer.java Wed Apr 2 09:20:51 2014
@@ -0,0 +1,54 @@
+/*
+ * Copyright 2013 YarcData LLC All Rights Reserved.
+ */
+
+package com.yarcdata.urika.hadoop.rdf.mapreduce.characteristics;
+
+import java.io.IOException;
+import java.util.Iterator;
+
+import org.apache.hadoop.io.NullWritable;
+import org.apache.hadoop.mapreduce.Reducer;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import com.yarcdata.urika.hadoop.rdf.types.CharacteristicSetWritable;
+
+/**
+ * Reducer which takes in characteristic sets and sums up all their usage counts
+ *
+ * @author rvesse
+ */
+public class CharacteristicSetReducer extends
+ Reducer<CharacteristicSetWritable, CharacteristicSetWritable, CharacteristicSetWritable, NullWritable> {
+
+ private static final Logger LOG = LoggerFactory.getLogger(CharacteristicSetReducer.class);
+ private boolean tracing = false;
+
+ @Override
+ protected void setup(Context context) throws IOException, InterruptedException {
+ super.setup(context);
+ this.tracing = LOG.isTraceEnabled();
+ }
+
+ @Override
+ protected void reduce(CharacteristicSetWritable key, Iterable<CharacteristicSetWritable> values, Context context)
+ throws IOException, InterruptedException {
+ Iterator<CharacteristicSetWritable> iter = values.iterator();
+ CharacteristicSetWritable output = new CharacteristicSetWritable(0);
+
+ if (this.tracing) {
+ LOG.trace("Key = {}", key);
+ }
+
+ while (iter.hasNext()) {
+ CharacteristicSetWritable set = iter.next();
+ if (this.tracing) {
+ LOG.trace("Value = {}", set);
+ }
+ output.add(set);
+ }
+
+ context.write(output, NullWritable.get());
+ }
+}
Added: jena/Experimental/hadoop-rdf/hadoop-rdf-mapreduce/src/main/java/com/yarcdata/urika/hadoop/rdf/mapreduce/characteristics/QuadCharacteristicSetGeneratingReducer.java
URL: http://svn.apache.org/viewvc/jena/Experimental/hadoop-rdf/hadoop-rdf-mapreduce/src/main/java/com/yarcdata/urika/hadoop/rdf/mapreduce/characteristics/QuadCharacteristicSetGeneratingReducer.java?rev=1583942&view=auto
==============================================================================
--- jena/Experimental/hadoop-rdf/hadoop-rdf-mapreduce/src/main/java/com/yarcdata/urika/hadoop/rdf/mapreduce/characteristics/QuadCharacteristicSetGeneratingReducer.java (added)
+++ jena/Experimental/hadoop-rdf/hadoop-rdf-mapreduce/src/main/java/com/yarcdata/urika/hadoop/rdf/mapreduce/characteristics/QuadCharacteristicSetGeneratingReducer.java Wed Apr 2 09:20:51 2014
@@ -0,0 +1,24 @@
+/*
+ * Copyright 2013 YarcData LLC All Rights Reserved.
+ */
+
+package com.yarcdata.urika.hadoop.rdf.mapreduce.characteristics;
+
+import com.hp.hpl.jena.sparql.core.Quad;
+import com.yarcdata.urika.hadoop.rdf.types.NodeWritable;
+import com.yarcdata.urika.hadoop.rdf.types.QuadWritable;
+
+/**
+ * A reducer which converts quads grouped by some node into characteristic sets
+ *
+ * @author rvesse
+ *
+ */
+public class QuadCharacteristicSetGeneratingReducer extends AbstractCharacteristicSetGeneratingReducer<Quad, QuadWritable> {
+
+ @Override
+ protected NodeWritable getPredicate(QuadWritable tuple) {
+ return new NodeWritable(tuple.get().getPredicate());
+ }
+
+}
Added: jena/Experimental/hadoop-rdf/hadoop-rdf-mapreduce/src/main/java/com/yarcdata/urika/hadoop/rdf/mapreduce/characteristics/TripleCharacteristicSetGeneratingReducer.java
URL: http://svn.apache.org/viewvc/jena/Experimental/hadoop-rdf/hadoop-rdf-mapreduce/src/main/java/com/yarcdata/urika/hadoop/rdf/mapreduce/characteristics/TripleCharacteristicSetGeneratingReducer.java?rev=1583942&view=auto
==============================================================================
--- jena/Experimental/hadoop-rdf/hadoop-rdf-mapreduce/src/main/java/com/yarcdata/urika/hadoop/rdf/mapreduce/characteristics/TripleCharacteristicSetGeneratingReducer.java (added)
+++ jena/Experimental/hadoop-rdf/hadoop-rdf-mapreduce/src/main/java/com/yarcdata/urika/hadoop/rdf/mapreduce/characteristics/TripleCharacteristicSetGeneratingReducer.java Wed Apr 2 09:20:51 2014
@@ -0,0 +1,25 @@
+/*
+ * Copyright 2013 YarcData LLC All Rights Reserved.
+ */
+
+package com.yarcdata.urika.hadoop.rdf.mapreduce.characteristics;
+
+import com.hp.hpl.jena.graph.Triple;
+import com.yarcdata.urika.hadoop.rdf.types.NodeWritable;
+import com.yarcdata.urika.hadoop.rdf.types.TripleWritable;
+
+/**
+ * A reducer which converts triples grouped by some node into characteristic
+ * sets
+ *
+ * @author rvesse
+ *
+ */
+public class TripleCharacteristicSetGeneratingReducer extends AbstractCharacteristicSetGeneratingReducer<Triple, TripleWritable> {
+
+ @Override
+ protected NodeWritable getPredicate(TripleWritable tuple) {
+ return new NodeWritable(tuple.get().getPredicate());
+ }
+
+}
Added: jena/Experimental/hadoop-rdf/hadoop-rdf-mapreduce/src/main/java/com/yarcdata/urika/hadoop/rdf/mapreduce/count/AbstractNodeTupleNodeCountMapper.java
URL: http://svn.apache.org/viewvc/jena/Experimental/hadoop-rdf/hadoop-rdf-mapreduce/src/main/java/com/yarcdata/urika/hadoop/rdf/mapreduce/count/AbstractNodeTupleNodeCountMapper.java?rev=1583942&view=auto
==============================================================================
--- jena/Experimental/hadoop-rdf/hadoop-rdf-mapreduce/src/main/java/com/yarcdata/urika/hadoop/rdf/mapreduce/count/AbstractNodeTupleNodeCountMapper.java (added)
+++ jena/Experimental/hadoop-rdf/hadoop-rdf-mapreduce/src/main/java/com/yarcdata/urika/hadoop/rdf/mapreduce/count/AbstractNodeTupleNodeCountMapper.java Wed Apr 2 09:20:51 2014
@@ -0,0 +1,52 @@
+/*
+ * Copyright 2013 YarcData LLC All Rights Reserved.
+ */
+
+package com.yarcdata.urika.hadoop.rdf.mapreduce.count;
+
+import java.io.IOException;
+
+import org.apache.hadoop.io.LongWritable;
+import org.apache.hadoop.mapreduce.Mapper;
+
+import com.yarcdata.urika.hadoop.rdf.types.AbstractNodeTupleWritable;
+import com.yarcdata.urika.hadoop.rdf.types.NodeWritable;
+
+/**
+ * Abstract mapper class for mappers which split node tuple values into pairs of
+ * node keys with a long value of 1. Can be used in conjunction with a
+ * {@link NodeCountReducer} to count the usages of each unique node.
+ *
+ * @author rvesse
+ *
+ * @param <TKey>
+ * Key type
+ * @param <TValue>
+ * Tuple type
+ * @param <T>
+ * Writable tuple type
+ */
+public abstract class AbstractNodeTupleNodeCountMapper<TKey, TValue, T extends AbstractNodeTupleWritable<TValue>> extends
+ Mapper<TKey, T, NodeWritable, LongWritable> {
+
+ private LongWritable initialCount = new LongWritable(1);
+
+ @Override
+ protected void map(TKey key, T value, Context context) throws IOException,
+ InterruptedException {
+ NodeWritable[] ns = this.getNodes(value);
+ for (NodeWritable n : ns) {
+ context.write(n, this.initialCount);
+ }
+ }
+
+ /**
+ * Gets the nodes of the tuple which are to be counted
+ *
+ * @param tuple
+ * Tuple
+ * @return Nodes
+ */
+ protected abstract NodeWritable[] getNodes(T tuple);
+
+}
Added: jena/Experimental/hadoop-rdf/hadoop-rdf-mapreduce/src/main/java/com/yarcdata/urika/hadoop/rdf/mapreduce/count/NodeCountReducer.java
URL: http://svn.apache.org/viewvc/jena/Experimental/hadoop-rdf/hadoop-rdf-mapreduce/src/main/java/com/yarcdata/urika/hadoop/rdf/mapreduce/count/NodeCountReducer.java?rev=1583942&view=auto
==============================================================================
--- jena/Experimental/hadoop-rdf/hadoop-rdf-mapreduce/src/main/java/com/yarcdata/urika/hadoop/rdf/mapreduce/count/NodeCountReducer.java (added)
+++ jena/Experimental/hadoop-rdf/hadoop-rdf-mapreduce/src/main/java/com/yarcdata/urika/hadoop/rdf/mapreduce/count/NodeCountReducer.java Wed Apr 2 09:20:51 2014
@@ -0,0 +1,36 @@
+/*
+ * Copyright 2013 YarcData LLC All Rights Reserved.
+ */
+
+package com.yarcdata.urika.hadoop.rdf.mapreduce.count;
+
+import java.io.IOException;
+import java.util.Iterator;
+
+import org.apache.hadoop.io.LongWritable;
+import org.apache.hadoop.mapreduce.Reducer;
+
+import com.yarcdata.urika.hadoop.rdf.types.NodeWritable;
+
+/**
+ * A reducer which takes node keys with a sequence of longs representing counts
+ * as the values and sums the counts together into pairs consisting of a node
+ * key and a count value.
+ *
+ * @author rvesse
+ *
+ */
+public class NodeCountReducer extends Reducer<NodeWritable, LongWritable, NodeWritable, LongWritable> {
+
+ @Override
+ protected void reduce(NodeWritable key, Iterable<LongWritable> values, Context context) throws IOException,
+ InterruptedException {
+ long count = 0;
+ Iterator<LongWritable> iter = values.iterator();
+ while (iter.hasNext()) {
+ count += iter.next().get();
+ }
+ context.write(key, new LongWritable(count));
+ }
+
+}
Added: jena/Experimental/hadoop-rdf/hadoop-rdf-mapreduce/src/main/java/com/yarcdata/urika/hadoop/rdf/mapreduce/count/QuadNodeCountMapper.java
URL: http://svn.apache.org/viewvc/jena/Experimental/hadoop-rdf/hadoop-rdf-mapreduce/src/main/java/com/yarcdata/urika/hadoop/rdf/mapreduce/count/QuadNodeCountMapper.java?rev=1583942&view=auto
==============================================================================
--- jena/Experimental/hadoop-rdf/hadoop-rdf-mapreduce/src/main/java/com/yarcdata/urika/hadoop/rdf/mapreduce/count/QuadNodeCountMapper.java (added)
+++ jena/Experimental/hadoop-rdf/hadoop-rdf-mapreduce/src/main/java/com/yarcdata/urika/hadoop/rdf/mapreduce/count/QuadNodeCountMapper.java Wed Apr 2 09:20:51 2014
@@ -0,0 +1,29 @@
+/*
+ * Copyright 2013 YarcData LLC All Rights Reserved.
+ */
+
+package com.yarcdata.urika.hadoop.rdf.mapreduce.count;
+
+import com.hp.hpl.jena.sparql.core.Quad;
+import com.yarcdata.urika.hadoop.rdf.types.NodeWritable;
+import com.yarcdata.urika.hadoop.rdf.types.QuadWritable;
+
+/**
+ * A mapper for counting node usages within quads designed primarily for use in
+ * conjunction with {@link NodeCountReducer}
+ *
+ * @author rvesse
+ *
+ * @param <TKey>
+ * Key type
+ */
+public class QuadNodeCountMapper<TKey> extends AbstractNodeTupleNodeCountMapper<TKey, Quad, QuadWritable> {
+
+ @Override
+ protected NodeWritable[] getNodes(QuadWritable tuple) {
+ Quad q = tuple.get();
+ return new NodeWritable[] { new NodeWritable(q.getGraph()), new NodeWritable(q.getSubject()),
+ new NodeWritable(q.getPredicate()), new NodeWritable(q.getObject()) };
+ }
+
+}
Added: jena/Experimental/hadoop-rdf/hadoop-rdf-mapreduce/src/main/java/com/yarcdata/urika/hadoop/rdf/mapreduce/count/TripleNodeCountMapper.java
URL: http://svn.apache.org/viewvc/jena/Experimental/hadoop-rdf/hadoop-rdf-mapreduce/src/main/java/com/yarcdata/urika/hadoop/rdf/mapreduce/count/TripleNodeCountMapper.java?rev=1583942&view=auto
==============================================================================
--- jena/Experimental/hadoop-rdf/hadoop-rdf-mapreduce/src/main/java/com/yarcdata/urika/hadoop/rdf/mapreduce/count/TripleNodeCountMapper.java (added)
+++ jena/Experimental/hadoop-rdf/hadoop-rdf-mapreduce/src/main/java/com/yarcdata/urika/hadoop/rdf/mapreduce/count/TripleNodeCountMapper.java Wed Apr 2 09:20:51 2014
@@ -0,0 +1,27 @@
+/*
+ * Copyright 2013 YarcData LLC All Rights Reserved.
+ */
+
+package com.yarcdata.urika.hadoop.rdf.mapreduce.count;
+
+import com.hp.hpl.jena.graph.Triple;
+import com.yarcdata.urika.hadoop.rdf.types.NodeWritable;
+import com.yarcdata.urika.hadoop.rdf.types.TripleWritable;
+
+/**
+ * A mapper for counting node usages within triples designed primarily for use
+ * in conjunction with {@link NodeCountReducer}
+ *
+ * @author rvesse
+ *
+ * @param <TKey> Key type
+ */
+public class TripleNodeCountMapper<TKey> extends AbstractNodeTupleNodeCountMapper<TKey, Triple, TripleWritable> {
+
+ @Override
+ protected NodeWritable[] getNodes(TripleWritable tuple) {
+ Triple t = tuple.get();
+ return new NodeWritable[] { new NodeWritable(t.getSubject()), new NodeWritable(t.getPredicate()),
+ new NodeWritable(t.getObject()) };
+ }
+}
Added: jena/Experimental/hadoop-rdf/hadoop-rdf-mapreduce/src/main/java/com/yarcdata/urika/hadoop/rdf/mapreduce/count/datatypes/QuadDataTypeCountMapper.java
URL: http://svn.apache.org/viewvc/jena/Experimental/hadoop-rdf/hadoop-rdf-mapreduce/src/main/java/com/yarcdata/urika/hadoop/rdf/mapreduce/count/datatypes/QuadDataTypeCountMapper.java?rev=1583942&view=auto
==============================================================================
--- jena/Experimental/hadoop-rdf/hadoop-rdf-mapreduce/src/main/java/com/yarcdata/urika/hadoop/rdf/mapreduce/count/datatypes/QuadDataTypeCountMapper.java (added)
+++ jena/Experimental/hadoop-rdf/hadoop-rdf-mapreduce/src/main/java/com/yarcdata/urika/hadoop/rdf/mapreduce/count/datatypes/QuadDataTypeCountMapper.java Wed Apr 2 09:20:51 2014
@@ -0,0 +1,41 @@
+/*
+ * Copyright 2013 YarcData LLC All Rights Reserved.
+ */
+
+package com.yarcdata.urika.hadoop.rdf.mapreduce.count.datatypes;
+
+import com.hp.hpl.jena.graph.Node;
+import com.hp.hpl.jena.graph.NodeFactory;
+import com.yarcdata.urika.hadoop.rdf.mapreduce.count.NodeCountReducer;
+import com.yarcdata.urika.hadoop.rdf.mapreduce.count.QuadNodeCountMapper;
+import com.yarcdata.urika.hadoop.rdf.types.NodeWritable;
+import com.yarcdata.urika.hadoop.rdf.types.QuadWritable;
+
+/**
+ * A mapper for counting data type usages within quads designed primarily for
+ * use in conjunction with {@link NodeCountReducer}
+ * <p>
+ * This mapper extracts the data types for typed literal objects and converts
+ * them into nodes so they can be counted
+ * </p>
+ *
+ * @author rvesse
+ *
+ * @param <TKey>
+ * Key type
+ */
+public class QuadDataTypeCountMapper<TKey> extends QuadNodeCountMapper<TKey> {
+
+ private static final NodeWritable[] EMPTY = new NodeWritable[0];
+
+ @Override
+ protected NodeWritable[] getNodes(QuadWritable tuple) {
+ Node object = tuple.get().getObject();
+ if (!object.isLiteral())
+ return EMPTY;
+ String dtUri = object.getLiteralDatatypeURI();
+ if (dtUri == null)
+ return EMPTY;
+ return new NodeWritable[] { new NodeWritable(NodeFactory.createURI(dtUri)) };
+ }
+}
Added: jena/Experimental/hadoop-rdf/hadoop-rdf-mapreduce/src/main/java/com/yarcdata/urika/hadoop/rdf/mapreduce/count/datatypes/TripleDataTypeCountMapper.java
URL: http://svn.apache.org/viewvc/jena/Experimental/hadoop-rdf/hadoop-rdf-mapreduce/src/main/java/com/yarcdata/urika/hadoop/rdf/mapreduce/count/datatypes/TripleDataTypeCountMapper.java?rev=1583942&view=auto
==============================================================================
--- jena/Experimental/hadoop-rdf/hadoop-rdf-mapreduce/src/main/java/com/yarcdata/urika/hadoop/rdf/mapreduce/count/datatypes/TripleDataTypeCountMapper.java (added)
+++ jena/Experimental/hadoop-rdf/hadoop-rdf-mapreduce/src/main/java/com/yarcdata/urika/hadoop/rdf/mapreduce/count/datatypes/TripleDataTypeCountMapper.java Wed Apr 2 09:20:51 2014
@@ -0,0 +1,41 @@
+/*
+ * Copyright 2013 YarcData LLC All Rights Reserved.
+ */
+
+package com.yarcdata.urika.hadoop.rdf.mapreduce.count.datatypes;
+
+import com.hp.hpl.jena.graph.Node;
+import com.hp.hpl.jena.graph.NodeFactory;
+import com.yarcdata.urika.hadoop.rdf.mapreduce.count.NodeCountReducer;
+import com.yarcdata.urika.hadoop.rdf.mapreduce.count.TripleNodeCountMapper;
+import com.yarcdata.urika.hadoop.rdf.types.NodeWritable;
+import com.yarcdata.urika.hadoop.rdf.types.TripleWritable;
+
+/**
+ * A mapper for counting data type usages within triples designed primarily for
+ * use in conjunction with {@link NodeCountReducer}
+ * <p>
+ * This mapper extracts the data types for typed literal objects and converts
+ * them into nodes so they can be counted
+ * </p>
+ *
+ * @author rvesse
+ *
+ * @param <TKey>
+ * Key type
+ */
+public class TripleDataTypeCountMapper<TKey> extends TripleNodeCountMapper<TKey> {
+
+ private static final NodeWritable[] EMPTY = new NodeWritable[0];
+
+ @Override
+ protected NodeWritable[] getNodes(TripleWritable tuple) {
+ Node object = tuple.get().getObject();
+ if (!object.isLiteral())
+ return EMPTY;
+ String dtUri = object.getLiteralDatatypeURI();
+ if (dtUri == null)
+ return EMPTY;
+ return new NodeWritable[] { new NodeWritable(NodeFactory.createURI(dtUri)) };
+ }
+}
Added: jena/Experimental/hadoop-rdf/hadoop-rdf-mapreduce/src/main/java/com/yarcdata/urika/hadoop/rdf/mapreduce/count/namespaces/AbstractNodeTupleNamespaceCountMapper.java
URL: http://svn.apache.org/viewvc/jena/Experimental/hadoop-rdf/hadoop-rdf-mapreduce/src/main/java/com/yarcdata/urika/hadoop/rdf/mapreduce/count/namespaces/AbstractNodeTupleNamespaceCountMapper.java?rev=1583942&view=auto
==============================================================================
--- jena/Experimental/hadoop-rdf/hadoop-rdf-mapreduce/src/main/java/com/yarcdata/urika/hadoop/rdf/mapreduce/count/namespaces/AbstractNodeTupleNamespaceCountMapper.java (added)
+++ jena/Experimental/hadoop-rdf/hadoop-rdf-mapreduce/src/main/java/com/yarcdata/urika/hadoop/rdf/mapreduce/count/namespaces/AbstractNodeTupleNamespaceCountMapper.java Wed Apr 2 09:20:51 2014
@@ -0,0 +1,120 @@
+/*
+ * Copyright 2013 YarcData LLC All Rights Reserved
+ */
+
+package com.yarcdata.urika.hadoop.rdf.mapreduce.count.namespaces;
+
+import java.io.IOException;
+
+import org.apache.hadoop.io.LongWritable;
+import org.apache.hadoop.io.Text;
+import org.apache.hadoop.mapreduce.Mapper;
+import com.hp.hpl.jena.graph.Node;
+import com.yarcdata.urika.hadoop.rdf.mapreduce.TextCountReducer;
+import com.yarcdata.urika.hadoop.rdf.types.AbstractNodeTupleWritable;
+import com.yarcdata.urika.hadoop.rdf.types.NodeWritable;
+
+/**
+ * Abstract mapper class for mappers which split node tuple values and extract
+ * the namespace URIs they use and outputs pairs of namespaces keys with a long
+ * value of 1. Can be used in conjunction with a {@link TextCountReducer} to
+ * count the usages of each unique namespace.
+ *
+ * @author rvesse
+ *
+ * @param <TKey>
+ * @param <TValue>
+ * @param <T>
+ */
+public abstract class AbstractNodeTupleNamespaceCountMapper<TKey, TValue, T extends AbstractNodeTupleWritable<TValue>> extends
+ Mapper<TKey, T, Text, LongWritable> {
+
+ private LongWritable initialCount = new LongWritable(1);
+ protected static final String NO_NAMESPACE = null;
+
+ @Override
+ protected void map(TKey key, T value, Context context) throws IOException, InterruptedException {
+ NodeWritable[] ns = this.getNodes(value);
+ for (NodeWritable n : ns) {
+ String namespace = this.extractNamespace(n);
+ if (namespace != null) {
+ context.write(new Text(namespace), this.initialCount);
+ }
+ }
+ }
+
+ /**
+ * Extracts the namespace from a node
+ * <p>
+ * Finds the URI for the node (if any) and then invokes
+ * {@link #extractNamespace(String)} to extract the actual namespace URI.
+ * </p>
+ * <p>
+ * Derived classes may override this to change the logic of how namespaces
+ * are extracted.
+ * </p>
+ *
+ * @param nw
+ * Node
+ * @return Namespace
+ */
+ protected String extractNamespace(NodeWritable nw) {
+ Node n = nw.get();
+ if (n.isBlank() || n.isVariable())
+ return NO_NAMESPACE;
+ if (n.isLiteral()) {
+ String dtUri = n.getLiteralDatatypeURI();
+ if (dtUri == null)
+ return NO_NAMESPACE;
+ return extractNamespace(dtUri);
+ }
+ return extractNamespace(n.getURI());
+ }
+
+ /**
+ * Extracts the namespace from a URI
+ * <p>
+ * First tries to extract a hash based namespace. If that is not possible it
+ * tries to extract a slash based namespace, if this is not possible then
+ * the full URI is returned.
+ * </p>
+ * <p>
+ * Derived classes may override this to change the logic of how namespaces
+ * are extracted.
+ * </p>
+ *
+ * @param uri
+ * URI
+ * @return Namespace
+ */
+ protected String extractNamespace(String uri) {
+ if (uri.contains("#")) {
+ // Extract hash namespace
+ return uri.substring(0, uri.lastIndexOf('#') + 1);
+ } else if (uri.contains("/")) {
+ // Ensure that this is not immediately after the scheme component or
+ // at end of URI
+ int index = uri.lastIndexOf('/');
+ int schemeSepIndex = uri.indexOf(':');
+ if (index - schemeSepIndex <= 2 || index == uri.length() - 1) {
+ // Use full URI
+ return uri;
+ }
+
+ // Otherwise safe to extract slash namespace
+ return uri.substring(0, uri.lastIndexOf('/') + 1);
+ } else {
+ // Use full URI
+ return uri;
+ }
+ }
+
+ /**
+ * Gets the nodes of the tuple whose namespaces are to be counted
+ *
+ * @param tuple
+ * Tuple
+ * @return Nodes
+ */
+ protected abstract NodeWritable[] getNodes(T tuple);
+}