You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@jena.apache.org by rv...@apache.org on 2014/09/10 17:20:04 UTC
svn commit: r1624028 - in
/jena/Experimental/hadoop-rdf/hadoop-rdf-common/src:
main/java/org/apache/jena/hadoop/rdf/types/
main/java/org/apache/jena/hadoop/rdf/types/compators/
test/java/org/apache/jena/hadoop/rdf/io/types/
Author: rvesse
Date: Wed Sep 10 15:20:04 2014
New Revision: 1624028
URL: http://svn.apache.org/r1624028
Log:
Switch to using RDF Thrift as the binary encoding for Nodes, Triples, Quads and Tuples
Modified:
jena/Experimental/hadoop-rdf/hadoop-rdf-common/src/main/java/org/apache/jena/hadoop/rdf/types/AbstractNodeTupleWritable.java
jena/Experimental/hadoop-rdf/hadoop-rdf-common/src/main/java/org/apache/jena/hadoop/rdf/types/NodeWritable.java
jena/Experimental/hadoop-rdf/hadoop-rdf-common/src/main/java/org/apache/jena/hadoop/rdf/types/QuadWritable.java
jena/Experimental/hadoop-rdf/hadoop-rdf-common/src/main/java/org/apache/jena/hadoop/rdf/types/TripleWritable.java
jena/Experimental/hadoop-rdf/hadoop-rdf-common/src/main/java/org/apache/jena/hadoop/rdf/types/compators/NodeComparator.java
jena/Experimental/hadoop-rdf/hadoop-rdf-common/src/test/java/org/apache/jena/hadoop/rdf/io/types/RdfTypesTest.java
Modified: jena/Experimental/hadoop-rdf/hadoop-rdf-common/src/main/java/org/apache/jena/hadoop/rdf/types/AbstractNodeTupleWritable.java
URL: http://svn.apache.org/viewvc/jena/Experimental/hadoop-rdf/hadoop-rdf-common/src/main/java/org/apache/jena/hadoop/rdf/types/AbstractNodeTupleWritable.java?rev=1624028&r1=1624027&r2=1624028&view=diff
==============================================================================
--- jena/Experimental/hadoop-rdf/hadoop-rdf-common/src/main/java/org/apache/jena/hadoop/rdf/types/AbstractNodeTupleWritable.java (original)
+++ jena/Experimental/hadoop-rdf/hadoop-rdf-common/src/main/java/org/apache/jena/hadoop/rdf/types/AbstractNodeTupleWritable.java Wed Sep 10 15:20:04 2014
@@ -16,156 +16,178 @@
* limitations under the License.
*/
-package org.apache.jena.hadoop.rdf.types;
-
-import java.io.DataInput;
-import java.io.DataOutput;
-import java.io.IOException;
-import org.apache.hadoop.io.WritableComparable;
-import org.apache.hadoop.io.WritableUtils;
-import com.hp.hpl.jena.graph.Node;
-import com.hp.hpl.jena.sparql.util.NodeUtils;
-
-/**
- * A abstract general purpose writable where the actual class represented is
- * composed of a number of {@link Node} instances
- *
- *
- *
- * @param <T>
- */
-public abstract class AbstractNodeTupleWritable<T> implements WritableComparable<AbstractNodeTupleWritable<T>> {
-
- private T tuple;
-
- /**
- * Creates a new empty instance
- */
- protected AbstractNodeTupleWritable() {
- this(null);
- }
-
- /**
- * Creates a new instance with the given value
- *
- * @param tuple
- * Tuple value
- */
- protected AbstractNodeTupleWritable(T tuple) {
- this.tuple = tuple;
- }
-
- /**
- * Gets the tuple
- *
- * @return Tuple
- */
- public T get() {
- return this.tuple;
- }
-
- /**
- * Sets the tuple
- *
- * @param tuple
- * Tuple
- */
- public void set(T tuple) {
- this.tuple = tuple;
- }
-
- @Override
- public final void readFields(DataInput input) throws IOException {
- // Determine how many nodes
- int size = WritableUtils.readVInt(input);
- Node[] ns = new Node[size];
-
- NodeWritable nw = new NodeWritable();
- for (int i = 0; i < ns.length; i++) {
- nw.readFields(input);
- ns[i] = nw.get();
- }
-
- // Load the tuple
- this.tuple = this.createTuple(ns);
- }
-
- /**
- * Creates the actual tuple type from an array of nodes
- *
- * @param ns
- * Nodes
- * @return Tuple
- */
- protected abstract T createTuple(Node[] ns);
-
- @Override
- public final void write(DataOutput output) throws IOException {
- // Determine how many nodes
- Node[] ns = this.createNodes(this.tuple);
- WritableUtils.writeVInt(output, ns.length);
-
- // Write out nodes
- NodeWritable nw = new NodeWritable();
- for (int i = 0; i < ns.length; i++) {
- nw.set(ns[i]);
- nw.write(output);
- }
- }
-
- /**
- * Converts the actual tuple type into an array of nodes
- *
- * @param tuple
- * Tuples
- * @return Nodes
- */
- protected abstract Node[] createNodes(T tuple);
-
- /**
- * Compares instances node by node
- * <p>
- * Derived implementations may wish to override this and substitute native
- * tuple based comparisons
- * </p>
- *
- * @param other
- * Instance to compare with
- */
- @Override
- public int compareTo(AbstractNodeTupleWritable<T> other) {
- Node[] ns = this.createNodes(this.tuple);
- Node[] otherNs = this.createNodes(other.tuple);
-
- if (ns.length < otherNs.length) {
- return -1;
- } else if (ns.length > otherNs.length) {
- return 1;
- }
- // Compare node by node
- for (int i = 0; i < ns.length; i++) {
- int c = NodeUtils.compareRDFTerms(ns[i], otherNs[i]);
- if (c != 0)
- return c;
- }
- return 0;
- }
-
- @Override
- public String toString() {
- return this.get().toString();
- }
-
- @Override
- public int hashCode() {
- return this.get().hashCode();
- }
-
- @SuppressWarnings("unchecked")
- @Override
- public boolean equals(Object other) {
- if (!(other instanceof AbstractNodeTupleWritable))
- return false;
- return this.compareTo((AbstractNodeTupleWritable<T>) other) == 0;
- }
-}
+package org.apache.jena.hadoop.rdf.types;
+
+import java.io.DataInput;
+import java.io.DataOutput;
+import java.io.IOException;
+import org.apache.hadoop.io.WritableComparable;
+import org.apache.hadoop.io.WritableUtils;
+import com.hp.hpl.jena.graph.Node;
+import com.hp.hpl.jena.sparql.util.NodeUtils;
+
+/**
+ * A abstract general purpose writable where the actual class represented is
+ * composed of a number of {@link Node} instances
+ * <p>
+ * The binary encoding of this base implementation is just a variable integer
+ * indicating the number of nodes present followed by the binary encodings of
+ * the {@link NodeWritable} instances. Derived implementations may wish to
+ * override the {@link #readFields(DataInput)} and {@link #write(DataOutput)}
+ * methods in order to use more specialised encodings.
+ * </p>
+ *
+ * @param <T>
+ * Tuple type
+ */
+public abstract class AbstractNodeTupleWritable<T> implements WritableComparable<AbstractNodeTupleWritable<T>> {
+
+ private T tuple;
+
+ /**
+ * Creates a new empty instance
+ */
+ protected AbstractNodeTupleWritable() {
+ this(null);
+ }
+
+ /**
+ * Creates a new instance with the given value
+ *
+ * @param tuple
+ * Tuple value
+ */
+ protected AbstractNodeTupleWritable(T tuple) {
+ this.tuple = tuple;
+ }
+
+ /**
+ * Gets the tuple
+ *
+ * @return Tuple
+ */
+ public T get() {
+ return this.tuple;
+ }
+
+ /**
+ * Sets the tuple
+ *
+ * @param tuple
+ * Tuple
+ */
+ public void set(T tuple) {
+ this.tuple = tuple;
+ }
+
+ @Override
+ public void readFields(DataInput input) throws IOException {
+ // Determine how many nodes
+ int size = WritableUtils.readVInt(input);
+ Node[] ns = new Node[size];
+
+ NodeWritable nw = new NodeWritable();
+ for (int i = 0; i < ns.length; i++) {
+ nw.readFields(input);
+ ns[i] = nw.get();
+ }
+
+ // Load the tuple
+ this.tuple = this.createTuple(ns);
+ }
+
+ /**
+ * Creates the actual tuple type from an array of nodes
+ *
+ * @param ns
+ * Nodes
+ * @return Tuple
+ */
+ protected abstract T createTuple(Node[] ns);
+
+ @Override
+ public void write(DataOutput output) throws IOException {
+ // Determine how many nodes
+ Node[] ns = this.createNodes(this.tuple);
+ WritableUtils.writeVInt(output, ns.length);
+
+ // Write out nodes
+ NodeWritable nw = new NodeWritable();
+ for (int i = 0; i < ns.length; i++) {
+ nw.set(ns[i]);
+ nw.write(output);
+ }
+ }
+
+ /**
+ * Sets the tuple value
+ * <p>
+ * Intended only for internal use i.e. when a derived implementation
+ * overrides {@link #readFields(DataInput)} and needs to set the tuple value
+ * directly i.e. when a derived implementation is using a custom encoding
+ * scheme
+ * </p>
+ *
+ * @param tuple
+ * Tuple
+ */
+ protected final void setInternal(T tuple) {
+ this.tuple = tuple;
+ }
+
+ /**
+ * Converts the actual tuple type into an array of nodes
+ *
+ * @param tuple
+ * Tuples
+ * @return Nodes
+ */
+ protected abstract Node[] createNodes(T tuple);
+
+ /**
+ * Compares instances node by node
+ * <p>
+ * Derived implementations may wish to override this and substitute native
+ * tuple based comparisons
+ * </p>
+ *
+ * @param other
+ * Instance to compare with
+ */
+ @Override
+ public int compareTo(AbstractNodeTupleWritable<T> other) {
+ Node[] ns = this.createNodes(this.tuple);
+ Node[] otherNs = this.createNodes(other.tuple);
+
+ if (ns.length < otherNs.length) {
+ return -1;
+ } else if (ns.length > otherNs.length) {
+ return 1;
+ }
+ // Compare node by node
+ for (int i = 0; i < ns.length; i++) {
+ int c = NodeUtils.compareRDFTerms(ns[i], otherNs[i]);
+ if (c != 0)
+ return c;
+ }
+ return 0;
+ }
+
+ @Override
+ public String toString() {
+ return this.get().toString();
+ }
+
+ @Override
+ public int hashCode() {
+ return this.get().hashCode();
+ }
+
+ @SuppressWarnings("unchecked")
+ @Override
+ public boolean equals(Object other) {
+ if (!(other instanceof AbstractNodeTupleWritable))
+ return false;
+ return this.compareTo((AbstractNodeTupleWritable<T>) other) == 0;
+ }
+}
Modified: jena/Experimental/hadoop-rdf/hadoop-rdf-common/src/main/java/org/apache/jena/hadoop/rdf/types/NodeWritable.java
URL: http://svn.apache.org/viewvc/jena/Experimental/hadoop-rdf/hadoop-rdf-common/src/main/java/org/apache/jena/hadoop/rdf/types/NodeWritable.java?rev=1624028&r1=1624027&r2=1624028&view=diff
==============================================================================
--- jena/Experimental/hadoop-rdf/hadoop-rdf-common/src/main/java/org/apache/jena/hadoop/rdf/types/NodeWritable.java (original)
+++ jena/Experimental/hadoop-rdf/hadoop-rdf-common/src/main/java/org/apache/jena/hadoop/rdf/types/NodeWritable.java Wed Sep 10 15:20:04 2014
@@ -16,200 +16,144 @@
* limitations under the License.
*/
-package org.apache.jena.hadoop.rdf.types;
-
-import java.io.DataInput;
-import java.io.DataOutput;
-import java.io.IOException;
-import java.io.StringWriter;
-import java.nio.charset.Charset;
-
-import org.apache.hadoop.io.WritableComparable;
-import org.apache.hadoop.io.WritableComparator;
-import org.apache.hadoop.io.WritableUtils;
-import org.apache.jena.atlas.io.AWriter;
-import org.apache.jena.atlas.io.Writer2;
+package org.apache.jena.hadoop.rdf.types;
+
+import java.io.DataInput;
+import java.io.DataOutput;
+import java.io.IOException;
+import org.apache.hadoop.io.WritableComparable;
+import org.apache.hadoop.io.WritableComparator;
import org.apache.jena.hadoop.rdf.types.compators.NodeComparator;
-import org.apache.jena.riot.RiotException;
-import org.apache.jena.riot.lang.LabelToNode;
-import org.apache.jena.riot.out.NodeFmtLib;
-import org.apache.jena.riot.out.NodeFormatter;
-import org.apache.jena.riot.out.NodeFormatterNT;
-import org.apache.jena.riot.out.NodeToLabel;
-import org.apache.jena.riot.system.SyntaxLabels;
-import org.apache.jena.riot.tokens.Token;
-import org.apache.jena.riot.tokens.Tokenizer;
-import org.apache.jena.riot.tokens.TokenizerFactory;
-
-import com.hp.hpl.jena.graph.Node;
-import com.hp.hpl.jena.sparql.util.NodeUtils;
-
-/**
- * A writable for {@link Node} instances
- *
- *
- *
- */
-public class NodeWritable implements WritableComparable<NodeWritable> {
-
- protected static final Charset utf8 = Charset.forName("utf-8");
- protected static final NodeFormatter formatter = new NodeFormatterNT();
- protected static final NodeToLabel nodeToLabel = SyntaxLabels.createNodeToLabelRT();
- protected static final LabelToNode labelToNode = SyntaxLabels.createLabelToNodeRT();
-
- static {
- WritableComparator.define(NodeWritable.class, new NodeComparator());
- }
-
- private Node node;
-
- /**
- * Creates an empty writable
- */
- public NodeWritable() {
- this(null);
- }
-
- /**
- * Creates a new instance from the given input
- *
- * @param input
- * Input
- * @return New instance
- * @throws IOException
- */
- public static NodeWritable read(DataInput input) throws IOException {
- NodeWritable nw = new NodeWritable();
- nw.readFields(input);
- return nw;
- }
-
- /**
- * Creates a new writable with the given value
- *
- * @param n
- * Node
- */
- public NodeWritable(Node n) {
- // Finally set the node
- this.set(n);
- }
-
- /**
- * Gets the node
- *
- * @return Node
- */
- public Node get() {
- return this.node;
- }
-
- /**
- * Sets the node
- *
- * @param n
- * Node
- */
- public void set(Node n) {
- if (n == null || !n.isBlank()) {
- this.node = n;
- } else {
- // Special handling for blank nodes
- this.node = labelToNode.get(null, nodeToLabel.get(null, n));
- }
- }
-
- @Override
- public void readFields(DataInput input) throws IOException {
- // Read type and ignore, used elsewhere for fast binary comparisons
- WritableUtils.readVInt(input);
-
- // Read length and use this to read in the bytes that represent the
- // serialized node
- int length = WritableUtils.readVInt(input);
- byte[] bytes = new byte[length];
- input.readFully(bytes, 0, length);
-
- // Parse in the node
- this.node = this.parseNode(new String(bytes, utf8));
- }
-
- protected Node parseNode(String nodeString) {
- Tokenizer tokenizer = TokenizerFactory.makeTokenizerString(nodeString);
- if (!tokenizer.hasNext())
- throw new RiotException("Empty RDF term");
- Token token = tokenizer.next();
- Node node;
- if (token.isBNode()) {
- String label = NodeFmtLib.decodeBNodeLabel(token.getImage());
- node = labelToNode.get(null, label);
- } else {
- node = token.asNode(null);
- }
- if (node == null)
- throw new RiotException("Bad RDF Term: " + nodeString);
-
- if (tokenizer.hasNext())
- throw new RiotException("Trailing characters in string: " + nodeString);
- if (node.isURI()) {
- // Lightly test for bad URIs.
- String x = node.getURI();
- if (x.indexOf(' ') >= 0)
- throw new RiotException("Space(s) in IRI: " + nodeString);
- }
- return node;
- }
-
- @Override
- public void write(DataOutput output) throws IOException {
- StringWriter strWriter = new StringWriter();
- AWriter writer = Writer2.wrap(strWriter);
- formatter.format(writer, this.node);
- writer.close();
- String nodeStr = strWriter.toString();
- byte[] bytes = nodeStr.getBytes(utf8);
- // Write out type, length and bytes
- WritableUtils.writeVInt(output, this.getTypeId());
- WritableUtils.writeVInt(output, bytes.length);
- output.write(bytes);
- }
-
- /**
- * Gets the node type identifier
- *
- * @return Node type identifier
- */
- protected final int getTypeId() {
- if (node.isVariable())
- return 0;
- if (node.isBlank())
- return 1;
- if (node.isURI())
- return 2;
- if (node.isLiteral())
- return 3;
- throw new RuntimeException("Unknown node type");
- }
-
- @Override
- public int compareTo(NodeWritable other) {
- return NodeUtils.compareRDFTerms(this.node, other.node);
- }
-
- @Override
- public String toString() {
- return this.node.toString();
- }
-
- @Override
- public int hashCode() {
- return this.node.hashCode();
- }
-
- @Override
- public boolean equals(Object other) {
- if (!(other instanceof NodeWritable))
- return false;
- return this.compareTo((NodeWritable) other) == 0;
- }
-}
+import org.apache.jena.hadoop.rdf.types.converters.ThriftConverter;
+import org.apache.jena.riot.thrift.TRDF;
+import org.apache.jena.riot.thrift.ThriftConvert;
+import org.apache.jena.riot.thrift.wire.RDF_Term;
+import org.apache.thrift.TException;
+
+import com.hp.hpl.jena.graph.Node;
+import com.hp.hpl.jena.sparql.util.NodeUtils;
+
+/**
+ * A writable for {@link Node} instances
+ */
+public class NodeWritable implements WritableComparable<NodeWritable> {
+
+ static {
+ WritableComparator.define(NodeWritable.class, new NodeComparator());
+ }
+
+ private Node node;
+ private RDF_Term term = new RDF_Term();
+
+ /**
+ * Creates an empty writable
+ */
+ public NodeWritable() {
+ this(null);
+ }
+
+ /**
+ * Creates a new instance from the given input
+ *
+ * @param input
+ * Input
+ * @return New instance
+ * @throws IOException
+ */
+ public static NodeWritable read(DataInput input) throws IOException {
+ NodeWritable nw = new NodeWritable();
+ nw.readFields(input);
+ return nw;
+ }
+
+ /**
+ * Creates a new writable with the given value
+ *
+ * @param n
+ * Node
+ */
+ public NodeWritable(Node n) {
+ this.set(n);
+ }
+
+ /**
+ * Gets the node
+ *
+ * @return Node
+ */
+ public Node get() {
+ return this.node;
+ }
+
+ /**
+ * Sets the node
+ *
+ * @param n
+ * Node
+ */
+ public void set(Node n) {
+ this.node = n;
+ // Clear the term for now
+ // Only convert the Node to it as and when we want to write it out
+ this.term.clear();
+ }
+
+ @Override
+ public void readFields(DataInput input) throws IOException {
+ this.term.clear();
+ int termLength = input.readInt();
+ byte[] buffer = new byte[termLength];
+ input.readFully(buffer);
+ try {
+ ThriftConverter.fromBytes(buffer, this.term);
+ } catch (TException e) {
+ throw new IOException(e);
+ }
+ this.node = ThriftConvert.convert(this.term);
+ }
+
+ @Override
+ public void write(DataOutput output) throws IOException {
+ // May not yet have prepared the Thrift term
+ if (!this.term.isSet()) {
+ if (this.node == null) {
+ this.term.setUndefined(TRDF.UNDEF);
+ } else {
+ ThriftConvert.toThrift(this.node, null, this.term, false);
+ }
+ }
+
+ byte[] buffer;
+ try {
+ buffer = ThriftConverter.toBytes(this.term);
+ } catch (TException e) {
+ throw new IOException(e);
+ }
+ output.writeInt(buffer.length);
+ output.write(buffer);
+ }
+
+ @Override
+ public int compareTo(NodeWritable other) {
+ return NodeUtils.compareRDFTerms(this.node, other.node);
+ }
+
+ @Override
+ public String toString() {
+ if (this.node == null)
+ return "";
+ return this.node.toString();
+ }
+
+ @Override
+ public int hashCode() {
+ return this.node.hashCode();
+ }
+
+ @Override
+ public boolean equals(Object other) {
+ if (!(other instanceof NodeWritable))
+ return false;
+ return this.compareTo((NodeWritable) other) == 0;
+ }
+}
Modified: jena/Experimental/hadoop-rdf/hadoop-rdf-common/src/main/java/org/apache/jena/hadoop/rdf/types/QuadWritable.java
URL: http://svn.apache.org/viewvc/jena/Experimental/hadoop-rdf/hadoop-rdf-common/src/main/java/org/apache/jena/hadoop/rdf/types/QuadWritable.java?rev=1624028&r1=1624027&r2=1624028&view=diff
==============================================================================
--- jena/Experimental/hadoop-rdf/hadoop-rdf-common/src/main/java/org/apache/jena/hadoop/rdf/types/QuadWritable.java (original)
+++ jena/Experimental/hadoop-rdf/hadoop-rdf-common/src/main/java/org/apache/jena/hadoop/rdf/types/QuadWritable.java Wed Sep 10 15:20:04 2014
@@ -16,64 +16,118 @@
* limitations under the License.
*/
-package org.apache.jena.hadoop.rdf.types;
-
-import java.io.DataInput;
-import java.io.IOException;
-
-import com.hp.hpl.jena.graph.Node;
-import com.hp.hpl.jena.sparql.core.Quad;
-
-/**
- * A writable quad
- *
- *
- *
- */
-public class QuadWritable extends AbstractNodeTupleWritable<Quad> {
-
- /**
- * Creates a new empty instance
- */
- public QuadWritable() {
- this(null);
- }
-
- /**
- * Creates a new instance with the given value
- *
- * @param q
- * Quad
- */
- public QuadWritable(Quad q) {
- super(q);
- }
-
- /**
- * Creates a new instance from the given input
- *
- * @param input
- * Input
- * @return New instance
- * @throws IOException
- */
- public static QuadWritable read(DataInput input) throws IOException {
- QuadWritable q = new QuadWritable();
- q.readFields(input);
- return q;
- }
-
- @Override
- protected Quad createTuple(Node[] ns) {
- if (ns.length != 4)
- throw new IllegalArgumentException(String.format("Incorrect number of nodes to form a quad - got %d but expected 4",
- ns.length));
- return new Quad(ns[0], ns[1], ns[2], ns[3]);
- }
-
- @Override
- protected Node[] createNodes(Quad tuple) {
- return new Node[] { tuple.getGraph(), tuple.getSubject(), tuple.getPredicate(), tuple.getObject() };
- }
-
-}
+package org.apache.jena.hadoop.rdf.types;
+
+import java.io.DataInput;
+import java.io.DataOutput;
+import java.io.IOException;
+
+import org.apache.jena.hadoop.rdf.types.converters.ThriftConverter;
+import org.apache.jena.riot.thrift.ThriftConvert;
+import org.apache.jena.riot.thrift.wire.RDF_Quad;
+import org.apache.thrift.TException;
+
+import com.hp.hpl.jena.graph.Node;
+import com.hp.hpl.jena.sparql.core.Quad;
+
+/**
+ * A writable quad
+ *
+ *
+ *
+ */
+public class QuadWritable extends AbstractNodeTupleWritable<Quad> {
+
+ private RDF_Quad quad = new RDF_Quad();
+
+ /**
+ * Creates a new empty instance
+ */
+ public QuadWritable() {
+ this(null);
+ }
+
+ /**
+ * Creates a new instance with the given value
+ *
+ * @param q
+ * Quad
+ */
+ public QuadWritable(Quad q) {
+ super(q);
+ }
+
+ /**
+ * Creates a new instance from the given input
+ *
+ * @param input
+ * Input
+ * @return New instance
+ * @throws IOException
+ */
+ public static QuadWritable read(DataInput input) throws IOException {
+ QuadWritable q = new QuadWritable();
+ q.readFields(input);
+ return q;
+ }
+
+ @Override
+ public void set(Quad tuple) {
+ super.set(tuple);
+ this.quad.clear();
+ }
+
+ @Override
+ public void readFields(DataInput input) throws IOException {
+ this.quad.clear();
+ int tripleLength = input.readInt();
+ byte[] buffer = new byte[tripleLength];
+ input.readFully(buffer);
+ try {
+ ThriftConverter.fromBytes(buffer, this.quad);
+ } catch (TException e) {
+ throw new IOException(e);
+ }
+ this.setInternal(new Quad(ThriftConvert.convert(this.quad.getG()), ThriftConvert.convert(this.quad.getS()),
+ ThriftConvert.convert(this.quad.getP()), ThriftConvert.convert(this.quad.getO())));
+ }
+
+ @Override
+ public void write(DataOutput output) throws IOException {
+ if (this.get() == null)
+ throw new IOException(
+ "Null quads cannot be written using this class, consider using NodeTupleWritable instead");
+
+ // May not have yet prepared the Thrift triple
+ if (!this.quad.isSetS()) {
+ Quad tuple = this.get();
+ this.quad.setG(ThriftConvert.convert(tuple.getGraph(), false));
+ this.quad.setS(ThriftConvert.convert(tuple.getSubject(), false));
+ this.quad.setP(ThriftConvert.convert(tuple.getPredicate(), false));
+ this.quad.setO(ThriftConvert.convert(tuple.getObject(), false));
+ }
+
+ byte[] buffer;
+ try {
+ buffer = ThriftConverter.toBytes(this.quad);
+ } catch (TException e) {
+ throw new IOException(e);
+ }
+ output.writeInt(buffer.length);
+ output.write(buffer);
+ }
+
+ @Override
+ protected Quad createTuple(Node[] ns) {
+ if (ns.length != 4)
+ throw new IllegalArgumentException(String.format(
+ "Incorrect number of nodes to form a quad - got %d but expected 4", ns.length));
+ return new Quad(ns[0], ns[1], ns[2], ns[3]);
+ }
+
+ @Override
+ protected Node[] createNodes(Quad tuple) {
+ return new Node[] { tuple.getGraph(), tuple.getSubject(), tuple.getPredicate(), tuple.getObject() };
+ }
+
+}
Modified: jena/Experimental/hadoop-rdf/hadoop-rdf-common/src/main/java/org/apache/jena/hadoop/rdf/types/TripleWritable.java
URL: http://svn.apache.org/viewvc/jena/Experimental/hadoop-rdf/hadoop-rdf-common/src/main/java/org/apache/jena/hadoop/rdf/types/TripleWritable.java?rev=1624028&r1=1624027&r2=1624028&view=diff
==============================================================================
--- jena/Experimental/hadoop-rdf/hadoop-rdf-common/src/main/java/org/apache/jena/hadoop/rdf/types/TripleWritable.java (original)
+++ jena/Experimental/hadoop-rdf/hadoop-rdf-common/src/main/java/org/apache/jena/hadoop/rdf/types/TripleWritable.java Wed Sep 10 15:20:04 2014
@@ -16,64 +16,117 @@
* limitations under the License.
*/
-package org.apache.jena.hadoop.rdf.types;
-
-import java.io.DataInput;
-import java.io.IOException;
-
-import com.hp.hpl.jena.graph.Node;
-import com.hp.hpl.jena.graph.Triple;
-
-/**
- * A writable triple
- *
- *
- *
- */
-public class TripleWritable extends AbstractNodeTupleWritable<Triple> {
-
- /**
- * Creates a new instance using the default NTriples node formatter
- */
- public TripleWritable() {
- this(null);
- }
-
- /**
- * Creates a new instance with a given value that uses a specific node
- * formatter
- *
- * @param t
- * Triple
- */
- public TripleWritable(Triple t) {
- super(t);
- }
-
- /**
- * Creates a new instance from the given input
- *
- * @param input
- * Input
- * @return New instance
- * @throws IOException
- */
- public static TripleWritable read(DataInput input) throws IOException {
- TripleWritable t = new TripleWritable();
- t.readFields(input);
- return t;
- }
-
- @Override
- protected Triple createTuple(Node[] ns) {
- if (ns.length != 3)
- throw new IllegalArgumentException(String.format(
- "Incorrect number of nodes to form a triple - got %d but expected 3", ns.length));
- return new Triple(ns[0], ns[1], ns[2]);
- }
-
- @Override
- protected Node[] createNodes(Triple tuple) {
- return new Node[] { tuple.getSubject(), tuple.getPredicate(), tuple.getObject() };
- }
-}
+package org.apache.jena.hadoop.rdf.types;
+
+import java.io.DataInput;
+import java.io.DataOutput;
+import java.io.IOException;
+
+import org.apache.jena.hadoop.rdf.types.converters.ThriftConverter;
+import org.apache.jena.riot.thrift.ThriftConvert;
+import org.apache.jena.riot.thrift.wire.RDF_Triple;
+import org.apache.thrift.TException;
+
+import com.hp.hpl.jena.graph.Node;
+import com.hp.hpl.jena.graph.Triple;
+
+/**
+ * A writable triple
+ *
+ *
+ *
+ */
+public class TripleWritable extends AbstractNodeTupleWritable<Triple> {
+
+ private RDF_Triple triple = new RDF_Triple();
+
+ /**
+ * Creates a new instance using the default NTriples node formatter
+ */
+ public TripleWritable() {
+ this(null);
+ }
+
+ /**
+ * Creates a new instance with a given value that uses a specific node
+ * formatter
+ *
+ * @param t
+ * Triple
+ */
+ public TripleWritable(Triple t) {
+ super(t);
+ }
+
+ /**
+ * Creates a new instance from the given input
+ *
+ * @param input
+ * Input
+ * @return New instance
+ * @throws IOException
+ */
+ public static TripleWritable read(DataInput input) throws IOException {
+ TripleWritable t = new TripleWritable();
+ t.readFields(input);
+ return t;
+ }
+
+ @Override
+ public void set(Triple tuple) {
+ super.set(tuple);
+ this.triple.clear();
+ }
+
+ @Override
+ public void readFields(DataInput input) throws IOException {
+ this.triple.clear();
+ int tripleLength = input.readInt();
+ byte[] buffer = new byte[tripleLength];
+ input.readFully(buffer);
+ try {
+ ThriftConverter.fromBytes(buffer, this.triple);
+ } catch (TException e) {
+ throw new IOException(e);
+ }
+ this.setInternal(new Triple(ThriftConvert.convert(this.triple.getS()),
+ ThriftConvert.convert(this.triple.getP()), ThriftConvert.convert(this.triple.getO())));
+ }
+
+ @Override
+ public void write(DataOutput output) throws IOException {
+ if (this.get() == null)
+ throw new IOException(
+ "Null triples cannot be written using this class, consider using NodeTupleWritable instead");
+
+ // May not have yet prepared the Thrift triple
+ if (!this.triple.isSetS()) {
+ Triple tuple = this.get();
+ this.triple.setS(ThriftConvert.convert(tuple.getSubject(), false));
+ this.triple.setP(ThriftConvert.convert(tuple.getPredicate(), false));
+ this.triple.setO(ThriftConvert.convert(tuple.getObject(), false));
+ }
+
+ byte[] buffer;
+ try {
+ buffer = ThriftConverter.toBytes(this.triple);
+ } catch (TException e) {
+ throw new IOException(e);
+ }
+ output.writeInt(buffer.length);
+ output.write(buffer);
+ }
+
+ @Override
+ protected Triple createTuple(Node[] ns) {
+ if (ns.length != 3)
+ throw new IllegalArgumentException(String.format(
+ "Incorrect number of nodes to form a triple - got %d but expected 3", ns.length));
+ return new Triple(ns[0], ns[1], ns[2]);
+ }
+
+ @Override
+ protected Node[] createNodes(Triple tuple) {
+ return new Node[] { tuple.getSubject(), tuple.getPredicate(), tuple.getObject() };
+ }
+}
Modified: jena/Experimental/hadoop-rdf/hadoop-rdf-common/src/main/java/org/apache/jena/hadoop/rdf/types/compators/NodeComparator.java
URL: http://svn.apache.org/viewvc/jena/Experimental/hadoop-rdf/hadoop-rdf-common/src/main/java/org/apache/jena/hadoop/rdf/types/compators/NodeComparator.java?rev=1624028&r1=1624027&r2=1624028&view=diff
==============================================================================
--- jena/Experimental/hadoop-rdf/hadoop-rdf-common/src/main/java/org/apache/jena/hadoop/rdf/types/compators/NodeComparator.java (original)
+++ jena/Experimental/hadoop-rdf/hadoop-rdf-common/src/main/java/org/apache/jena/hadoop/rdf/types/compators/NodeComparator.java Wed Sep 10 15:20:04 2014
@@ -16,51 +16,21 @@
* limitations under the License.
*/
-package org.apache.jena.hadoop.rdf.types.compators;
-
-import java.io.IOException;
-
-import org.apache.hadoop.io.DataInputBuffer;
-import org.apache.hadoop.io.WritableComparator;
-import org.apache.hadoop.io.WritableUtils;
-
-/**
- * A comparator for nodes that provides an efficient binary comparison
- *
- *
- *
- */
-public class NodeComparator extends WritableComparator {
-
- private DataInputBuffer buffer = new DataInputBuffer();
-
- @Override
- public int compare(byte[] b1, int s1, int l1, byte[] b2, int s2, int l2) {
- try {
- // Read info for first node
- buffer.reset(b1, s1, l1);
- int type1 = WritableUtils.readVInt(buffer);
- int length1 = WritableUtils.readVInt(buffer);
-
- // Read info for second node
- buffer.reset(b2, s2, l2);
- int type2 = WritableUtils.readVInt(buffer);
- int length2 = WritableUtils.readVInt(buffer);
-
- // Are types different?
- if (type1 < type2) {
- return -1;
- } else if (type1 > type2) {
- return 1;
- }
-
- // Otherwise compare lexicographically
- int adj1 = WritableUtils.getVIntSize(type1) + WritableUtils.getVIntSize(length1);
- int adj2 = WritableUtils.getVIntSize(type2) + WritableUtils.getVIntSize(length2);
- return WritableComparator.compareBytes(b1, s1 + adj1, l1 - adj1, b2, s2 + adj2, l2 - adj2);
- } catch (IOException e) {
- throw new RuntimeException(e);
- }
- }
-
-}
+package org.apache.jena.hadoop.rdf.types.compators;
+
+import org.apache.hadoop.io.WritableComparator;
+
+/**
+ * A comparator for nodes that provides an efficient binary comparison
+ *
+ *
+ *
+ */
+public class NodeComparator extends WritableComparator {
+
+ @Override
+ public int compare(byte[] b1, int s1, int l1, byte[] b2, int s2, int l2) {
+ return WritableComparator.compareBytes(b1, s1, l1, b2, s2, l2);
+ }
+
+}
Modified: jena/Experimental/hadoop-rdf/hadoop-rdf-common/src/test/java/org/apache/jena/hadoop/rdf/io/types/RdfTypesTest.java
URL: http://svn.apache.org/viewvc/jena/Experimental/hadoop-rdf/hadoop-rdf-common/src/test/java/org/apache/jena/hadoop/rdf/io/types/RdfTypesTest.java?rev=1624028&r1=1624027&r2=1624028&view=diff
==============================================================================
--- jena/Experimental/hadoop-rdf/hadoop-rdf-common/src/test/java/org/apache/jena/hadoop/rdf/io/types/RdfTypesTest.java (original)
+++ jena/Experimental/hadoop-rdf/hadoop-rdf-common/src/test/java/org/apache/jena/hadoop/rdf/io/types/RdfTypesTest.java Wed Sep 10 15:20:04 2014
@@ -292,6 +292,21 @@ public class RdfTypesTest {
TripleWritable tw = new TripleWritable(t);
testWriteRead(tw, tw);
}
+
+ /**
+ * Basic triple writable round tripping test
+ *
+ * @throws IOException
+ * @throws InstantiationException
+ * @throws IllegalAccessException
+ * @throws ClassNotFoundException
+ */
+ @Test
+ public void triple_writable_02() throws IOException, InstantiationException, IllegalAccessException, ClassNotFoundException {
+ Triple t = new Triple(NodeFactory.createAnon(), NodeFactory.createURI("http://predicate"), NodeFactory.createLiteral("value"));
+ TripleWritable tw = new TripleWritable(t);
+ testWriteRead(tw, tw);
+ }
/**
* Basic quad writable round tripping test
@@ -308,6 +323,22 @@ public class RdfTypesTest {
QuadWritable qw = new QuadWritable(q);
testWriteRead(qw, qw);
}
+
+ /**
+ * Basic quad writable round tripping test
+ *
+ * @throws IOException
+ * @throws InstantiationException
+ * @throws IllegalAccessException
+ * @throws ClassNotFoundException
+ */
+ @Test
+ public void quad_writable_02() throws IOException, InstantiationException, IllegalAccessException, ClassNotFoundException {
+ Quad q = new Quad(Quad.defaultGraphNodeGenerated, NodeFactory.createAnon(), NodeFactory.createURI("http://predicate"),
+ NodeFactory.createLiteral("value"));
+ QuadWritable qw = new QuadWritable(q);
+ testWriteRead(qw, qw);
+ }
/**
* Basic tuple writable round tripping test