You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@jena.apache.org by an...@apache.org on 2018/05/22 16:03:24 UTC
[2/5] jena git commit: JENA-1550: TDB2 Bulk Loaders
http://git-wip-us.apache.org/repos/asf/jena/blob/f5d9e6fa/jena-db/jena-tdb2/src/main/java/org/apache/jena/tdb2/loader/parallel/DataBlock.java
----------------------------------------------------------------------
diff --git a/jena-db/jena-tdb2/src/main/java/org/apache/jena/tdb2/loader/parallel/DataBlock.java b/jena-db/jena-tdb2/src/main/java/org/apache/jena/tdb2/loader/parallel/DataBlock.java
new file mode 100644
index 0000000..2fb540c
--- /dev/null
+++ b/jena-db/jena-tdb2/src/main/java/org/apache/jena/tdb2/loader/parallel/DataBlock.java
@@ -0,0 +1,37 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.jena.tdb2.loader.parallel;
+
+import java.util.List;
+
+import org.apache.jena.graph.Triple;
+import org.apache.jena.sparql.core.Quad;
+
+/** Unit of output from the data batcher */
+public class DataBlock {
+ /** Unique end marker object */
+ static DataBlock END = new DataBlock(null, null);
+
+ List<Triple> triples = null;
+ List<Quad> quads = null;
+ DataBlock( List<Triple> triples, List<Quad> quads) {
+ this.triples = triples;
+ this.quads = quads;
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/jena/blob/f5d9e6fa/jena-db/jena-tdb2/src/main/java/org/apache/jena/tdb2/loader/parallel/DataToTuples.java
----------------------------------------------------------------------
diff --git a/jena-db/jena-tdb2/src/main/java/org/apache/jena/tdb2/loader/parallel/DataToTuples.java b/jena-db/jena-tdb2/src/main/java/org/apache/jena/tdb2/loader/parallel/DataToTuples.java
new file mode 100644
index 0000000..33f107d
--- /dev/null
+++ b/jena-db/jena-tdb2/src/main/java/org/apache/jena/tdb2/loader/parallel/DataToTuples.java
@@ -0,0 +1,205 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.jena.tdb2.loader.parallel;
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.concurrent.ArrayBlockingQueue;
+import java.util.concurrent.BlockingQueue;
+import java.util.function.Consumer;
+import java.util.function.Function;
+
+import org.apache.jena.atlas.lib.tuple.Tuple;
+import org.apache.jena.atlas.lib.tuple.TupleFactory;
+import org.apache.jena.dboe.transaction.txn.Transaction;
+import org.apache.jena.dboe.transaction.txn.TransactionCoordinator;
+import org.apache.jena.graph.Node;
+import org.apache.jena.graph.Triple;
+import org.apache.jena.query.TxnType;
+import org.apache.jena.sparql.core.DatasetPrefixStorage;
+import org.apache.jena.sparql.core.Quad;
+import org.apache.jena.tdb2.loader.BulkLoaderException;
+import org.apache.jena.tdb2.loader.base.MonitorOutput;
+import org.apache.jena.tdb2.store.DatasetGraphTDB;
+import org.apache.jena.tdb2.store.NodeId;
+import org.apache.jena.tdb2.store.nodetable.NodeTable;
+
+/** Batch processing of {@link DataBlock}s (triples or Quads) converting them to two output of
+ * to blocks of {@code Tuple<NodeId>}.
+ * <p>
+ * This class runs one task thread.
+ * <p>
+ * Data is deliver into the process by calling the provided functions for {@code Destination<Tuple<NodeId>}.
+ * <p>
+ * Assumes triples and quads share a node table.
+ */
+
+public class DataToTuples implements BulkStartFinish {
+ private long countTriples;
+ private long countQuads;
+
+ private final Destination<Tuple<NodeId>> dest3;
+ private final Destination<Tuple<NodeId>> dest4;
+ private final DatasetGraphTDB dsgtdb;
+ private final NodeTable nodeTable;
+ private final DatasetPrefixStorage prefixes;
+
+ // Chunk accumulators.
+ private List<Tuple<NodeId>> quads = null;
+ private List<Tuple<NodeId>> triples = null;
+ private final MonitorOutput output;
+ private BlockingQueue<DataBlock> input;
+
+ public DataToTuples(DatasetGraphTDB dsgtdb, Destination<Tuple<NodeId>> tuples3, Destination<Tuple<NodeId>> tuples4, MonitorOutput output) {
+ this.dsgtdb = dsgtdb;
+ this.dest3 = tuples3;
+ this.dest4 = tuples4;
+ this.input = new ArrayBlockingQueue<>(LoaderConst.QueueSizeData);
+ this.nodeTable = dsgtdb.getQuadTable().getNodeTupleTable().getNodeTable();
+ this.prefixes = dsgtdb.getPrefixes();
+ this.output = output;
+
+ NodeTable nodeTable2 = dsgtdb.getTripleTable().getNodeTupleTable().getNodeTable();
+ if ( nodeTable != nodeTable2 )
+ throw new BulkLoaderException("Different node tables");
+ }
+
+ private TransactionCoordinator coordinator;
+ private Transaction transaction;
+
+ public Consumer<DataBlock> data() {
+ return this::index;
+ }
+
+ private void index(DataBlock dataBlock) {
+ try { input.put(dataBlock); }
+ catch (InterruptedException e) {
+ throw new BulkLoaderException("InterruptedException", e);
+ }
+ }
+
+ @Override
+ public void startBulk() {
+ new Thread(()->action()).start();
+ }
+
+ @Override
+ public void finishBulk() { }
+
+ // Triples.
+ private void action() {
+ coordinator = CoLib.newCoordinator();
+ CoLib.add(coordinator, nodeTable);
+ CoLib.start(coordinator);
+ transaction = coordinator.begin(TxnType.WRITE);
+
+ try {
+ for (;;) {
+
+ DataBlock data = input.take();
+ if ( data == DataBlock.END )
+ break;
+ if ( data.triples != null ) {
+ List<Tuple<NodeId>> tuples = new ArrayList<>(data.triples.size());
+ for ( Triple t : data.triples ) {
+ countTriples++;
+ accTuples(t, nodeTable, tuples);
+ }
+ dispatchTuples3(tuples);
+ }
+ if ( data.quads != null ) {
+ List<Tuple<NodeId>> tuples = new ArrayList<>(data.quads.size());
+ for ( Quad q : data.quads ) {
+ countQuads++;
+ accTuples(q, nodeTable, tuples);
+ }
+ dispatchTuples4(tuples);
+ }
+ }
+ dispatchTuples3(LoaderConst.END_TUPLES);
+ dispatchTuples4(LoaderConst.END_TUPLES);
+ transaction.commit();
+ } catch (Exception ex) {
+ ex.printStackTrace();
+ transaction.abort();
+ }
+ transaction.end();
+ CoLib.finish(coordinator);
+ }
+
+ //@Override
+ public long getCountTriples() { return countTriples; }
+ //@Override
+ public long getCountQuads() { return countQuads; }
+
+ private void dispatchTuples3(List<Tuple<NodeId>> chunk) {
+ dest3.deliver(chunk);
+ }
+
+ private void dispatchTuples4(List<Tuple<NodeId>> chunk) {
+ dest4.deliver(chunk);
+ }
+
+ private static void accTuples(Triple triple, NodeTable nodeTable, List<Tuple<NodeId>> acc) {
+ acc.add(nodes(nodeTable, triple));
+ }
+
+ private static void accTuples(Quad quad, NodeTable nodeTable, List<Tuple<NodeId>> acc) {
+ acc.add(nodes(nodeTable, quad));
+ }
+
+ // Recycle?
+ private List<Tuple<NodeId>> allocChunkTriples() {
+ return new ArrayList<>(LoaderConst.ChunkSize);
+ }
+
+ private List<Tuple<NodeId>> allocChunkQuads() {
+ return new ArrayList<>(LoaderConst.ChunkSize);
+ }
+
+ private static Tuple<NodeId> nodes(NodeTable nt, Triple triple) {
+ NodeId s = idForNode(nt, triple.getSubject());
+ NodeId p = idForNode(nt, triple.getPredicate());
+ NodeId o = idForNode(nt, triple.getObject());
+ return TupleFactory.tuple(s,p,o);
+ }
+
+ private Function<List<Quad>, List<Tuple<NodeId>>> quadsToNodeIds(NodeTable nodeTable) {
+ return (List<Quad> quads) -> {
+ List<Tuple<NodeId>> x = new ArrayList<>(quads.size());
+ for(Quad quad: quads) {
+ x.add(nodes(nodeTable, quad));
+ }
+ return x;
+ };
+ }
+
+ private static Tuple<NodeId> nodes(NodeTable nt, Quad quad) {
+ NodeId g = idForNode(nt, quad.getGraph());
+ NodeId s = idForNode(nt, quad.getSubject());
+ NodeId p = idForNode(nt, quad.getPredicate());
+ NodeId o = idForNode(nt, quad.getObject());
+ return TupleFactory.tuple(g,s,p,o);
+ }
+
+ private static final NodeId idForNode(NodeTable nodeTable, Node node) {
+ return nodeTable.getAllocateNodeId(node);
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/jena/blob/f5d9e6fa/jena-db/jena-tdb2/src/main/java/org/apache/jena/tdb2/loader/parallel/DataToTuplesInline.java
----------------------------------------------------------------------
diff --git a/jena-db/jena-tdb2/src/main/java/org/apache/jena/tdb2/loader/parallel/DataToTuplesInline.java b/jena-db/jena-tdb2/src/main/java/org/apache/jena/tdb2/loader/parallel/DataToTuplesInline.java
new file mode 100644
index 0000000..bca7158
--- /dev/null
+++ b/jena-db/jena-tdb2/src/main/java/org/apache/jena/tdb2/loader/parallel/DataToTuplesInline.java
@@ -0,0 +1,199 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.jena.tdb2.loader.parallel;
+
+import java.util.ArrayList;
+import java.util.List;
+
+import org.apache.jena.atlas.lib.tuple.Tuple;
+import org.apache.jena.atlas.lib.tuple.TupleFactory;
+import org.apache.jena.dboe.transaction.txn.Transaction;
+import org.apache.jena.dboe.transaction.txn.TransactionCoordinator;
+import org.apache.jena.graph.Node;
+import org.apache.jena.graph.Triple;
+import org.apache.jena.query.TxnType;
+import org.apache.jena.riot.lang.StreamRDFCounting;
+import org.apache.jena.riot.system.StreamRDF;
+import org.apache.jena.sparql.core.DatasetPrefixStorage;
+import org.apache.jena.sparql.core.Quad;
+import org.apache.jena.tdb2.loader.BulkLoaderException;
+import org.apache.jena.tdb2.loader.base.MonitorOutput;
+import org.apache.jena.tdb2.store.DatasetGraphTDB;
+import org.apache.jena.tdb2.store.DatasetPrefixesTDB;
+import org.apache.jena.tdb2.store.NodeId;
+import org.apache.jena.tdb2.store.nodetable.NodeTable;
+import org.apache.jena.tdb2.store.nodetupletable.NodeTupleTable;
+
+/** Triple to chunks of Tuples.
+ * Same thread version.
+ * This is a {@link StreamRDF}.
+ * Also loads prefixes.
+ */
+public class DataToTuplesInline implements StreamRDFCounting, BulkStartFinish {
+ public static final int DataTickPoint = 100_000;
+ public static final int DataSuperTick = 10;
+
+ private final Destination<Tuple<NodeId>> dest3;
+ private final Destination<Tuple<NodeId>> dest4;
+ private final DatasetGraphTDB dsgtdb;
+ private final NodeTable nodeTable;
+ private final DatasetPrefixStorage prefixes;
+
+ private final MonitorOutput output;
+ // Chunk accumulators.
+ private long countTriples = 0;
+ private long countQuads = 0;
+ private List<Tuple<NodeId>> quads = null;
+ private List<Tuple<NodeId>> triples = null;
+ // Prefix handler.
+ public DataToTuplesInline(DatasetGraphTDB dsgtdb,
+ Destination<Tuple<NodeId>> dest3,
+ Destination<Tuple<NodeId>> dest4,
+ MonitorOutput output) {
+ this.dsgtdb = dsgtdb;
+ this.dest3 = dest3;
+ this.dest4 = dest4;
+ this.output = output;
+ this.nodeTable = dsgtdb.getTripleTable().getNodeTupleTable().getNodeTable();
+ this.prefixes = dsgtdb.getPrefixes();
+ NodeTable nodeTable2 = dsgtdb.getQuadTable().getNodeTupleTable().getNodeTable();
+ if ( nodeTable != nodeTable2 )
+ throw new BulkLoaderException("Different node tables");
+ }
+
+ // StreamRDF
+ private TransactionCoordinator coordinator;
+ private Transaction transaction;
+ @Override
+ public void startBulk() {
+ coordinator = CoLib.newCoordinator();
+ CoLib.add(coordinator, nodeTable);
+
+ // Prefixes
+ NodeTupleTable p = ((DatasetPrefixesTDB)prefixes).getNodeTupleTable();
+ CoLib.add(coordinator, p.getNodeTable());
+ CoLib.add(coordinator, p.getTupleTable().getIndexes());
+ CoLib.start(coordinator);
+ transaction = coordinator.begin(TxnType.WRITE);
+ }
+
+ @Override
+ public void finishBulk() {
+ if ( triples != null && ! triples.isEmpty() ) {
+ dispatchTriples(triples);
+ triples = null;
+ }
+ dispatchTriples(LoaderConst.END_TUPLES);
+ transaction.commit();
+ transaction.end();
+ CoLib.finish(coordinator);
+ }
+
+ @Override public void start() {}
+
+ @Override public void finish() {}
+
+ @Override public long count() { return countTriples() + countQuads(); }
+
+ @Override public long countTriples() { return countTriples; }
+
+ @Override public long countQuads() { return countQuads; }
+
+ @Override
+ public void triple(Triple triple) {
+ countTriples++;
+ if ( triples == null )
+ triples = allocChunkTriples();
+ accTuples(triple, nodeTable, triples);
+ if ( triples.size() >= LoaderConst.ChunkSize ) {
+ dispatchTriples(triples);
+ triples = null;
+ }
+ }
+
+ @Override
+ public void quad(Quad quad) {
+ if ( quad.isTriple() || quad.isDefaultGraph() ) {
+ triple(quad.asTriple());
+ return;
+ }
+ countQuads++;
+ if ( quads == null )
+ quads = allocChunkQuads();
+ accTuples(quad, nodeTable, quads);
+ if ( quads.size() >= LoaderConst.ChunkSize ) {
+ dispatchQuads(quads);
+ quads = null;
+ }
+ }
+
+ private void dispatchQuads(List<Tuple<NodeId>> chunk) {
+ dest4.deliver(chunk);
+ }
+
+ private void dispatchTriples(List<Tuple<NodeId>> chunk) {
+ dest3.deliver(chunk);
+ }
+
+ @Override
+ public void base(String base) {}
+
+ @Override
+ public void prefix(String prefix, String iri) {
+ // Clean constant handling.
+ prefixes.insertPrefix("", prefix, iri);
+ }
+
+ private static void accTuples(Triple triple, NodeTable nodeTable, List<Tuple<NodeId>> acc) {
+ acc.add(nodes(nodeTable, triple));
+ }
+
+ private static void accTuples(Quad quad, NodeTable nodeTable, List<Tuple<NodeId>> acc) {
+ acc.add(nodes(nodeTable, quad));
+ }
+
+ // Recycle?
+ private List<Tuple<NodeId>> allocChunkTriples() {
+ return new ArrayList<>(LoaderConst.ChunkSize);
+ }
+
+ private List<Tuple<NodeId>> allocChunkQuads() {
+ return new ArrayList<>(LoaderConst.ChunkSize);
+ }
+
+ private static Tuple<NodeId> nodes(NodeTable nt, Triple triple) {
+ NodeId s = idForNode(nt, triple.getSubject());
+ NodeId p = idForNode(nt, triple.getPredicate());
+ NodeId o = idForNode(nt, triple.getObject());
+ return TupleFactory.tuple(s,p,o);
+ }
+
+ private static Tuple<NodeId> nodes(NodeTable nt, Quad quad) {
+ NodeId g = idForNode(nt, quad.getGraph());
+ NodeId s = idForNode(nt, quad.getSubject());
+ NodeId p = idForNode(nt, quad.getPredicate());
+ NodeId o = idForNode(nt, quad.getObject());
+ return TupleFactory.tuple(g,s,p,o);
+ }
+
+ private static final NodeId idForNode(NodeTable nodeTable, Node node) {
+ return nodeTable.getAllocateNodeId(node);
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/jena/blob/f5d9e6fa/jena-db/jena-tdb2/src/main/java/org/apache/jena/tdb2/loader/parallel/Destination.java
----------------------------------------------------------------------
diff --git a/jena-db/jena-tdb2/src/main/java/org/apache/jena/tdb2/loader/parallel/Destination.java b/jena-db/jena-tdb2/src/main/java/org/apache/jena/tdb2/loader/parallel/Destination.java
new file mode 100644
index 0000000..00c4d76
--- /dev/null
+++ b/jena-db/jena-tdb2/src/main/java/org/apache/jena/tdb2/loader/parallel/Destination.java
@@ -0,0 +1,27 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.jena.tdb2.loader.parallel;
+
+import java.util.List;
+
+/** Unit of delivery to a processing stage. */
+@FunctionalInterface
+public interface Destination<X> {
+ void deliver(List<X> block);
+}
http://git-wip-us.apache.org/repos/asf/jena/blob/f5d9e6fa/jena-db/jena-tdb2/src/main/java/org/apache/jena/tdb2/loader/parallel/Indexer.java
----------------------------------------------------------------------
diff --git a/jena-db/jena-tdb2/src/main/java/org/apache/jena/tdb2/loader/parallel/Indexer.java b/jena-db/jena-tdb2/src/main/java/org/apache/jena/tdb2/loader/parallel/Indexer.java
new file mode 100644
index 0000000..6dcaf22
--- /dev/null
+++ b/jena-db/jena-tdb2/src/main/java/org/apache/jena/tdb2/loader/parallel/Indexer.java
@@ -0,0 +1,138 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.jena.tdb2.loader.parallel;
+
+import java.util.Arrays;
+import java.util.List;
+import java.util.concurrent.ArrayBlockingQueue;
+import java.util.concurrent.BlockingQueue;
+import java.util.concurrent.Semaphore;
+
+import org.apache.jena.atlas.lib.ArrayUtils;
+import org.apache.jena.atlas.lib.Timer;
+import org.apache.jena.atlas.lib.tuple.Tuple;
+import org.apache.jena.dboe.transaction.txn.Transaction;
+import org.apache.jena.dboe.transaction.txn.TransactionCoordinator;
+import org.apache.jena.query.TxnType;
+import org.apache.jena.tdb2.loader.base.MonitorOutput;
+import org.apache.jena.tdb2.store.NodeId;
+import org.apache.jena.tdb2.store.tupletable.TupleIndex;
+
+/**
+ * Build index(es).
+ * Provides a function {@link #index()} that should be called from another thread
+ * to deliver chunks of tuples ({@code List<Tuple<NodeId>>}).
+ * Each chunk should be the same Tuyple length and this must correspond to the length of the {@link TupleIndex}s being loaded.
+ * <p>
+ * This class creates one thread per {@link TupleIndex}.
+ */
+class Indexer implements BulkStartFinish {
+
+ private BlockingQueue<List<Tuple<NodeId>>>[] pipesTripleIndexers;
+ private final int N;
+ private final MonitorOutput output;
+ private TupleIndex[] indexes;
+ private final Semaphore termination = new Semaphore(0);
+
+ @SuppressWarnings("unchecked")
+ public Indexer(MonitorOutput output, TupleIndex... idxTriples) {
+ pipesTripleIndexers = ArrayUtils.alloc(BlockingQueue.class, idxTriples.length);
+ this.N = idxTriples.length;
+ this.indexes = Arrays.copyOf(idxTriples, N);
+ this.output = output;
+
+ for ( int i = 0 ; i < N ; i++ ) {
+ pipesTripleIndexers[i] = new ArrayBlockingQueue<List<Tuple<NodeId>>>(LoaderConst.QueueSizeTuples);
+ }
+ }
+
+ private static long acquire(Semaphore semaphore, int numPermits) {
+ return Timer.time(()->{
+ try { semaphore.acquire(numPermits); }
+ catch (InterruptedException e) { e.printStackTrace(); }
+ });
+ }
+
+ /** Return a function that delivers multiple {@code List<Tuple<NodeId>>>} to this indexer */
+ public Destination<Tuple<NodeId>> index() {
+ return this::index;
+ }
+
+ private void index(List<Tuple<NodeId>> chunk) {
+ for ( int i = 0 ; i < N ; i++ ) {
+ try {
+ pipesTripleIndexers[i].put(chunk);
+ }
+ catch (InterruptedException e) {
+ e.printStackTrace();
+ }
+ }
+ }
+
+ /** Start the threads that will do the indexing */
+ @Override
+ public void startBulk() {
+ for ( int i = 0 ; i < N ; i++ ) {
+ TupleIndex idx = indexes[i];
+ BlockingQueue<List<Tuple<NodeId>>> pipe = pipesTripleIndexers[i];
+ new Thread(()->stageIndex(pipe, idx)).start();
+ }
+ }
+
+ /** Wait for all the indexing threads to complete. */
+ @Override
+ public void finishBulk() {
+ //output.print("Wait for %d indexers", N);
+ acquire(termination, N);
+ }
+
+ private void stageIndex(BlockingQueue<List<Tuple<NodeId>>> pipe, TupleIndex idx) {
+ TransactionCoordinator coordinator = CoLib.newCoordinator();
+ CoLib.add(coordinator, idx);
+ CoLib.start(coordinator);
+ Transaction transaction = coordinator.begin(TxnType.WRITE);
+ boolean workHasBeenDone;
+ try {
+ Destination<Tuple<NodeId>> loader = loadTuples(idx);
+ for (;;) {
+ List<Tuple<NodeId>> tuples = pipe.take();
+ if ( tuples.isEmpty() )
+ break;
+ loader.deliver(tuples);
+ }
+ workHasBeenDone = ! idx.isEmpty();
+ transaction.commit();
+ } catch (Exception ex) {
+ ex.printStackTrace();
+ transaction.abort();
+ workHasBeenDone = false;
+ }
+ CoLib.finish(coordinator);
+ if ( workHasBeenDone )
+ output.print("Finish - index %s", idx.getName());
+ termination.release();
+ }
+
+ private static Destination<Tuple<NodeId>> loadTuples(TupleIndex index) {
+ return (List<Tuple<NodeId>> tuples) -> {
+ for(Tuple<NodeId> tuple : tuples)
+ index.add(tuple);
+ };
+ }
+}
http://git-wip-us.apache.org/repos/asf/jena/blob/f5d9e6fa/jena-db/jena-tdb2/src/main/java/org/apache/jena/tdb2/loader/parallel/LoaderConst.java
----------------------------------------------------------------------
diff --git a/jena-db/jena-tdb2/src/main/java/org/apache/jena/tdb2/loader/parallel/LoaderConst.java b/jena-db/jena-tdb2/src/main/java/org/apache/jena/tdb2/loader/parallel/LoaderConst.java
new file mode 100644
index 0000000..665a329
--- /dev/null
+++ b/jena-db/jena-tdb2/src/main/java/org/apache/jena/tdb2/loader/parallel/LoaderConst.java
@@ -0,0 +1,41 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.jena.tdb2.loader.parallel;
+
+import java.util.Collections;
+import java.util.List;
+
+import org.apache.jena.atlas.lib.tuple.Tuple;
+import org.apache.jena.tdb2.store.NodeId;
+
+public class LoaderConst {
+
+ /** Chunk size for the triple->tuples output pipe */
+ public final static int ChunkSize = 100_000 ;
+
+ /** Queue size for chunks of tuples Tuples */
+ public final static int QueueSizeTuples = 10;
+
+ //public final static int pipeSize = 10;
+
+ /* package */ static final List<Tuple<NodeId>> END_TUPLES = Collections.emptyList();
+
+ /*package*/ static final int QueueSizeData = 10;
+
+}
http://git-wip-us.apache.org/repos/asf/jena/blob/f5d9e6fa/jena-db/jena-tdb2/src/main/java/org/apache/jena/tdb2/loader/parallel/LoaderParallel.java
----------------------------------------------------------------------
diff --git a/jena-db/jena-tdb2/src/main/java/org/apache/jena/tdb2/loader/parallel/LoaderParallel.java b/jena-db/jena-tdb2/src/main/java/org/apache/jena/tdb2/loader/parallel/LoaderParallel.java
new file mode 100644
index 0000000..5cf95ce
--- /dev/null
+++ b/jena-db/jena-tdb2/src/main/java/org/apache/jena/tdb2/loader/parallel/LoaderParallel.java
@@ -0,0 +1,155 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.jena.tdb2.loader.parallel;
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.function.Consumer;
+
+import org.apache.jena.atlas.lib.tuple.Tuple;
+import org.apache.jena.graph.Node;
+import org.apache.jena.riot.system.StreamRDF;
+import org.apache.jena.sparql.core.DatasetGraph;
+import org.apache.jena.tdb2.loader.DataLoader;
+import org.apache.jena.tdb2.loader.base.LoaderBase;
+import org.apache.jena.tdb2.loader.base.LoaderOps;
+import org.apache.jena.tdb2.loader.base.MonitorOutput;
+import org.apache.jena.tdb2.store.DatasetGraphTDB;
+import org.apache.jena.tdb2.store.DatasetPrefixesTDB;
+import org.apache.jena.tdb2.store.NodeId;
+import org.apache.jena.tdb2.sys.TDBInternal;
+
+/**
+ * The parallel Loader.
+ * <p>
+ * The process is:
+ * <blockquote>
+ * {@code DataBatcher -> DataToTuples -> Indexer}
+ * <blockquote>
+ * {@link DataBatcher} produces {@link DataBlock}s - grouping of triples and quads. It uses
+ * <br/>
+ * {@link DataToTuples} processes {@link DataBlock} to create 2 outputs blocks of {@code Tuple<NodeId>}, one output for triples, oue for quads.
+ * <br/>
+ * {@link Indexer} processes blocks of {@code Tuple<NodeId>} (of the same tuple length) and writes them to a number of indexes.
+ */
+public class LoaderParallel extends LoaderBase implements DataLoader {
+ public static final int DataTickPoint = 500_000;
+ public static final int DataSuperTick = 10;
+// public static final int IndexTickPoint = 1_000_000;
+// public static final int IndexSuperTick = 10;
+
+ private final DatasetGraphTDB dsgtdb;
+ private PrefixHandler prefixHandler;
+ private StreamRDF stream;
+ private DataBatcher dataBatcher;
+ private DataToTuples dtt;
+ private Indexer indexer3;
+ private Indexer indexer4;
+ private List<BulkStartFinish> process = new ArrayList<>();
+
+ public LoaderParallel(DatasetGraph dsg, MonitorOutput output) {
+ this(dsg, null, output);
+ }
+
+ public LoaderParallel(DatasetGraph dsg, Node graphName, MonitorOutput output) {
+ super(dsg, graphName, output);
+ dsgtdb = TDBInternal.getDatasetGraphTDB(dsg);
+
+ DatasetPrefixesTDB dps = (DatasetPrefixesTDB)dsgtdb.getPrefixes();
+ prefixHandler = new PrefixHandler(dps, output);
+ process.add(prefixHandler);
+
+ // Contains a splitter of Tuples -> Tuples per index.
+ indexer3 = new Indexer(output, dsgtdb.getTripleTable().getNodeTupleTable().getTupleTable().getIndexes());
+ indexer4 = new Indexer(output, dsgtdb.getQuadTable().getNodeTupleTable().getTupleTable().getIndexes());
+
+ process.add(indexer4);
+ process.add(indexer3);
+
+ Destination<Tuple<NodeId>> functionIndexer3 = indexer3.index();
+ Destination<Tuple<NodeId>> functionIndexer4 = indexer4.index();
+
+ // Multi-thread input stage (2 threads).
+ dtt = new DataToTuples(dsgtdb, functionIndexer3, functionIndexer4, output);
+ Consumer<DataBlock> dest = dtt.data();
+ dataBatcher = new DataBatcher(dest, prefixHandler.handler(), output);
+ StreamRDF baseInput = dataBatcher;
+ process.add(dtt);
+ process.add(dataBatcher);
+
+ // One thread input stage
+// DataToTuplesInline dttInline = new DataToTuplesInline(dsgtdb, functionIndexer3, functionIndexer4, output);
+// StreamRDF baseInput = dttInline;
+// process.add(dttInline);
+
+ stream = LoaderOps.toNamedGraph(baseInput, graphName);
+ }
+
+ @Override
+ public StreamRDF stream() {
+ return stream;
+ }
+
+ @Override
+ public boolean bulkUseTransaction() {
+ // Manipulate the transactions directly by component.
+ return false;
+ }
+
+ @Override
+ public void startBulk() {
+ // Lock everyone else out while we multithread.
+ dsgtdb.getTxnSystem().getTxnMgr().startExclusiveMode();
+ super.startBulk();
+ BulkProcesses.start(process);
+ }
+
+ @Override
+ public void finishBulk() {
+ if ( output != null )
+ output.print("Waiting for background tasks to complete");
+ BulkProcesses.finish(process);
+ super.finishBulk();
+ dsgtdb.getTxnSystem().getTxnMgr().finishExclusiveMode();
+ }
+
+ @Override
+ public void finishException(Exception ex) {
+ try {
+ dsgtdb.getTxnSystem().getTxnMgr().finishExclusiveMode();
+ } catch (Exception ex2) {
+ ex.addSuppressed(ex2);
+ }
+ }
+
+ @Override
+ public long countTriples() {
+ return dataBatcher.countTriples();
+ }
+
+ @Override
+ public long countQuads() {
+ return dataBatcher.countQuads();
+ }
+
+ @Override
+ protected void loadOne(String filename) {
+ LoaderOps.inputFile(stream, filename, output, DataTickPoint, DataSuperTick);
+ }
+}
http://git-wip-us.apache.org/repos/asf/jena/blob/f5d9e6fa/jena-db/jena-tdb2/src/main/java/org/apache/jena/tdb2/loader/parallel/PrefixHandler.java
----------------------------------------------------------------------
diff --git a/jena-db/jena-tdb2/src/main/java/org/apache/jena/tdb2/loader/parallel/PrefixHandler.java b/jena-db/jena-tdb2/src/main/java/org/apache/jena/tdb2/loader/parallel/PrefixHandler.java
new file mode 100644
index 0000000..fec0803
--- /dev/null
+++ b/jena-db/jena-tdb2/src/main/java/org/apache/jena/tdb2/loader/parallel/PrefixHandler.java
@@ -0,0 +1,68 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.jena.tdb2.loader.parallel;
+
+import java.util.function.BiConsumer;
+
+import org.apache.jena.dboe.transaction.txn.Transaction;
+import org.apache.jena.dboe.transaction.txn.TransactionCoordinator;
+import org.apache.jena.query.TxnType;
+import org.apache.jena.tdb2.loader.base.MonitorOutput;
+import org.apache.jena.tdb2.store.DatasetPrefixesTDB;
+import org.apache.jena.tdb2.store.nodetupletable.NodeTupleTable;
+
+/**
+ * Prefix handler.
+ * <p>
+ * This class is not multithreaded - prefixes are usually few enough in number
+ * and so forking a thread so that work can be done in parallel is not beneficial.
+ */
+public class PrefixHandler implements BulkStartFinish {
+
+ private Transaction transaction;
+ private TransactionCoordinator coordinator;
+ private DatasetPrefixesTDB prefixes;
+ private MonitorOutput output;
+
+ PrefixHandler(DatasetPrefixesTDB prefixes, MonitorOutput output) {
+ this.prefixes = prefixes;
+ this.output = output;
+ }
+
+ // Inline, not a separate thread.
+ @Override
+ public void startBulk() {
+ TransactionCoordinator coordinator = CoLib.newCoordinator();
+ NodeTupleTable p = prefixes.getNodeTupleTable();
+ CoLib.add(coordinator, p.getNodeTable());
+ CoLib.add(coordinator, p.getTupleTable().getIndexes());
+ coordinator.start();
+ transaction = coordinator.begin(TxnType.WRITE);
+ }
+
+ @Override
+ public void finishBulk() {
+ transaction.commit();
+ }
+
+ public BiConsumer<String, String> handler() {
+ return (prefix, uriStr) ->
+ prefixes.insertPrefix(DatasetPrefixesTDB.unnamedGraphURI, prefix, uriStr);
+ }
+}
http://git-wip-us.apache.org/repos/asf/jena/blob/f5d9e6fa/jena-db/jena-tdb2/src/main/java/org/apache/jena/tdb2/loader/sequential/BuilderSecondaryIndexes.java
----------------------------------------------------------------------
diff --git a/jena-db/jena-tdb2/src/main/java/org/apache/jena/tdb2/loader/sequential/BuilderSecondaryIndexes.java b/jena-db/jena-tdb2/src/main/java/org/apache/jena/tdb2/loader/sequential/BuilderSecondaryIndexes.java
new file mode 100644
index 0000000..1c87df0
--- /dev/null
+++ b/jena-db/jena-tdb2/src/main/java/org/apache/jena/tdb2/loader/sequential/BuilderSecondaryIndexes.java
@@ -0,0 +1,31 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.jena.tdb2.loader.sequential;
+
+import org.apache.jena.tdb2.loader.base.MonitorOutput;
+import org.apache.jena.tdb2.store.tupletable.TupleIndex;
+
+/**
+ * This interface is the mechanism for building indexes given that at leasts one index
+ * already exists (the "primary", which normally is SPO or GSPO).
+ */
+public interface BuilderSecondaryIndexes
+{
+ public void createSecondaryIndexes(MonitorOutput output, TupleIndex primaryIndex, TupleIndex[] secondaryIndexes) ;
+}
http://git-wip-us.apache.org/repos/asf/jena/blob/f5d9e6fa/jena-db/jena-tdb2/src/main/java/org/apache/jena/tdb2/loader/sequential/BuilderSecondaryIndexesSequential.java
----------------------------------------------------------------------
diff --git a/jena-db/jena-tdb2/src/main/java/org/apache/jena/tdb2/loader/sequential/BuilderSecondaryIndexesSequential.java b/jena-db/jena-tdb2/src/main/java/org/apache/jena/tdb2/loader/sequential/BuilderSecondaryIndexesSequential.java
new file mode 100644
index 0000000..5293fe2
--- /dev/null
+++ b/jena-db/jena-tdb2/src/main/java/org/apache/jena/tdb2/loader/sequential/BuilderSecondaryIndexesSequential.java
@@ -0,0 +1,57 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.jena.tdb2.loader.sequential;
+
+import org.apache.jena.atlas.lib.Timer ;
+import org.apache.jena.tdb2.loader.base.LoaderOps;
+import org.apache.jena.tdb2.loader.base.MonitorOutput;
+import org.apache.jena.tdb2.loader.base.ProgressMonitor;
+import org.apache.jena.tdb2.loader.base.ProgressMonitorOutput;
+import org.apache.jena.tdb2.store.tupletable.TupleIndex ;
+
+public class BuilderSecondaryIndexesSequential implements BuilderSecondaryIndexes
+{
+ public BuilderSecondaryIndexesSequential() {}
+
+ // Create each secondary indexes, doing one at a time.
+ @Override
+ public void createSecondaryIndexes(MonitorOutput output, TupleIndex primaryIndex, TupleIndex[] secondaryIndexes)
+ {
+ Timer timer = new Timer() ;
+ timer.startTimer() ;
+ boolean printTiming = true;
+ for ( TupleIndex index : secondaryIndexes ) {
+ if ( index != null ) {
+ ProgressMonitor monitor = ProgressMonitorOutput.create(output, index.getName(),
+ LoaderSequential.IndexTickPoint,
+ LoaderSequential.IndexSuperTick);
+ monitor.startMessage();
+ monitor.start();
+
+ long time1 = timer.readTimer() ;
+ LoaderOps.copyIndex(primaryIndex.all(), new TupleIndex[]{index}, monitor) ;
+ long time2 = timer.readTimer() ;
+ monitor.finish();
+ monitor.finishMessage(index.getName()+" indexing: ");
+// if ( printTiming )
+// output.print("Time for %s indexing: %.2fs", index.getName(), (time2-time1)/1000.0) ;
+ }
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/jena/blob/f5d9e6fa/jena-db/jena-tdb2/src/main/java/org/apache/jena/tdb2/loader/sequential/LoaderNodeTupleTable.java
----------------------------------------------------------------------
diff --git a/jena-db/jena-tdb2/src/main/java/org/apache/jena/tdb2/loader/sequential/LoaderNodeTupleTable.java b/jena-db/jena-tdb2/src/main/java/org/apache/jena/tdb2/loader/sequential/LoaderNodeTupleTable.java
new file mode 100644
index 0000000..b15d77f
--- /dev/null
+++ b/jena-db/jena-tdb2/src/main/java/org/apache/jena/tdb2/loader/sequential/LoaderNodeTupleTable.java
@@ -0,0 +1,174 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.jena.tdb2.loader.sequential;
+
+import org.apache.jena.atlas.lib.ArrayUtils;
+import org.apache.jena.atlas.lib.Closeable;
+import org.apache.jena.atlas.lib.Sync;
+import org.apache.jena.graph.Node;
+import org.apache.jena.tdb2.loader.base.MonitorOutput;
+import org.apache.jena.tdb2.store.nodetupletable.NodeTupleTable;
+import org.apache.jena.tdb2.store.tupletable.TupleIndex;
+
+/**
+ * Load into one NodeTupleTable (triples, quads, other).
+ *
+ */
+
+public class LoaderNodeTupleTable implements Closeable, Sync
+{
+ private int numIndexes;
+ private TupleIndex primaryIndex;
+ private TupleIndex[] secondaryIndexes;
+
+ private NodeTupleTable nodeTupleTable;
+
+ private boolean dropAndRebuildIndexes;
+ //private Timer timer;
+ private long count = 0;
+ private long countTriples = 0;
+ private long countQuads = 0;
+ private String itemsName;
+ private final MonitorOutput output;
+
+ private static Object lock = new Object();
+
+ public LoaderNodeTupleTable(NodeTupleTable nodeTupleTable, MonitorOutput output, String itemsName)
+ {
+ this.nodeTupleTable = nodeTupleTable;
+ this.itemsName = itemsName;
+ this.output = output;
+ }
+
+ // -- LoaderFramework
+
+ protected void loadPrepare() {
+ dropAndRebuildIndexes = nodeTupleTable.isEmpty();
+
+ if ( dropAndRebuildIndexes ) {
+ //output.print("** Load empty %s table", itemsName);
+ // SPO, GSPO only.
+ dropSecondaryIndexes();
+ } else {
+ output.print("** Load into %s table with existing data", itemsName);
+ }
+ }
+
+ protected void loadSecondaryIndexes() {
+ if ( count > 0 ) {
+ if ( dropAndRebuildIndexes )
+ // Now do secondary indexes.
+ createSecondaryIndexes();
+ }
+ attachSecondaryIndexes();
+ }
+
+ public void loadStart() {
+ }
+
+ public void loadFinish() {
+ }
+
+ /** Notify start of loading process */
+ public void loadDataStart() {
+ loadPrepare();
+ }
+
+ /** Notify End of data to load - this operation may
+ * undertake a significant amount of work.
+ */
+ public void loadDataFinish()
+ {
+ if ( count > 0 ) {
+ // Do index phase only if any items seen.
+ // monitor.startIndexPhase();
+ }
+ // Always do this - it reattaches the secondary indexes.
+ loadSecondaryIndexes();
+ }
+
+ /** Stream in items to load ... */
+ public void load(Node...nodes) {
+ if ( nodes.length == 3 )
+ countTriples++;
+ if ( nodes.length == 4 )
+ countQuads++;
+ count++;
+ nodeTupleTable.addRow(nodes);
+ }
+
+// public void loadIndexStart()
+// {
+// if ( count > 0 )
+// // Do index phase only if any items seen.
+// monitor.startIndexPhase();
+// // Always do this - it reattaches the secondary indexes.
+// loadSecondaryIndexes();
+// }
+//
+// public void loadIndexFinish()
+// {
+// if ( count > 0 )
+// monitor.finishIndexPhase();
+// }
+
+ public void sync(boolean force) {}
+ @Override
+ public void sync() {}
+
+ // --------
+
+ @Override
+ public void close()
+ { sync(); }
+
+ private void dropSecondaryIndexes() {
+ numIndexes = nodeTupleTable.getTupleTable().numIndexes();
+ primaryIndex = nodeTupleTable.getTupleTable().getIndex(0);
+
+ secondaryIndexes = ArrayUtils.alloc(TupleIndex.class, numIndexes-1);
+ System.arraycopy(nodeTupleTable.getTupleTable().getIndexes(), 1,
+ secondaryIndexes, 0,
+ numIndexes-1);
+ // Set non-primary indexes to null.
+ for ( int i = 1; i < numIndexes; i++ )
+ nodeTupleTable.getTupleTable().setTupleIndex(i, null);
+ }
+
+ private void createSecondaryIndexes() {
+ BuilderSecondaryIndexes builder = new BuilderSecondaryIndexesSequential();
+ builder.createSecondaryIndexes(output, primaryIndex, secondaryIndexes);
+ }
+
+ private void attachSecondaryIndexes() {
+ for ( int i = 1; i < numIndexes; i++ )
+ nodeTupleTable.getTupleTable().setTupleIndex(i, secondaryIndexes[i-1]);
+ }
+
+ static private void sync(TupleIndex[] indexes) {
+ for ( TupleIndex idx : indexes ) {
+ if ( idx != null )
+ idx.sync();
+ }
+ }
+
+ private static boolean tickPoint(long counter, long quantum) {
+ return counter % quantum == 0;
+ }
+}
http://git-wip-us.apache.org/repos/asf/jena/blob/f5d9e6fa/jena-db/jena-tdb2/src/main/java/org/apache/jena/tdb2/loader/sequential/LoaderSequential.java
----------------------------------------------------------------------
diff --git a/jena-db/jena-tdb2/src/main/java/org/apache/jena/tdb2/loader/sequential/LoaderSequential.java b/jena-db/jena-tdb2/src/main/java/org/apache/jena/tdb2/loader/sequential/LoaderSequential.java
new file mode 100644
index 0000000..a71c930
--- /dev/null
+++ b/jena-db/jena-tdb2/src/main/java/org/apache/jena/tdb2/loader/sequential/LoaderSequential.java
@@ -0,0 +1,124 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.jena.tdb2.loader.sequential;
+
+import org.apache.jena.graph.Node;
+import org.apache.jena.graph.Triple;
+import org.apache.jena.riot.system.StreamRDF;
+import org.apache.jena.riot.system.StreamRDFLib;
+import org.apache.jena.riot.system.StreamRDFWrapper;
+import org.apache.jena.sparql.core.DatasetGraph;
+import org.apache.jena.sparql.core.Quad;
+import org.apache.jena.tdb2.loader.BulkLoaderException;
+import org.apache.jena.tdb2.loader.base.LoaderBase;
+import org.apache.jena.tdb2.loader.base.LoaderOps;
+import org.apache.jena.tdb2.loader.base.MonitorOutput;
+import org.apache.jena.tdb2.store.DatasetGraphTDB;
+import org.apache.jena.tdb2.sys.TDBInternal;
+
+/** Bulk loader. Algorithm: Parser to primaryindexes, then build secondary indexes. */
+public class LoaderSequential extends LoaderBase {
+
+ public static final int DataTickPoint = 100_000;
+ public static final int DataSuperTick = 10;
+ public static final int IndexTickPoint = 1_000_000;
+ public static final int IndexSuperTick = 10;
+
+ private final LoaderNodeTupleTable triplesLoader;
+ private final LoaderNodeTupleTable quadsLoader;
+ private final DatasetGraphTDB dsgtdb;
+
+ private long countQuads;
+ private long countTriples;
+ private StreamRDF stream;
+
+ public LoaderSequential(DatasetGraph dsg, Node graphName, MonitorOutput output) {
+ super(dsg, graphName, output);
+
+ if ( ! TDBInternal.isBackedByTDB(dsg) )
+ throw new BulkLoaderException("Not a TDB2 database");
+
+ this.dsgtdb = TDBInternal.getDatasetGraphTDB(dsg);
+ this.triplesLoader = new LoaderNodeTupleTable(dsgtdb.getTripleTable().getNodeTupleTable(), output, "Triples");
+ this.quadsLoader = new LoaderNodeTupleTable(dsgtdb.getQuadTable().getNodeTupleTable(), output, "Quads");
+ StreamRDF s = StreamRDFLib.dataset(dsg);
+ s = new StreamRDFWrapper(s) {
+ @Override
+ public void triple(Triple triple) {
+ triplesLoader.load(triple.getSubject(), triple.getPredicate(), triple.getObject());
+ countTriples++;
+ }
+
+ @Override
+ public void quad(Quad quad) {
+ quadsLoader.load(quad.getGraph(), quad.getSubject(), quad.getPredicate(), quad.getObject());
+ countQuads++;
+ }
+ };
+ this.stream = LoaderOps.toNamedGraph(s, graphName);
+ }
+
+ @Override
+ public void startBulk() {
+ //Not in a transaction.
+ //dsgtdb.getTxnSystem().getTxnMgr().startExclusiveMode();
+ super.startBulk();
+ triplesLoader.loadDataStart();
+ quadsLoader.loadDataStart();
+ }
+
+ @Override
+ public void finishBulk() {
+ triplesLoader.loadDataFinish();
+ quadsLoader.loadDataFinish();
+ super.finishBulk();
+ //dsgtdb.getTxnSystem().getTxnMgr().finishExclusiveMode();
+ }
+
+ @Override
+ public void finishException(Exception ex) {
+ super.finishException(ex);
+ //dsgtdb.getTxnSystem().getTxnMgr().finishExclusiveMode();
+ }
+
+ @Override
+ public StreamRDF stream() {
+ return stream;
+ }
+
+ @Override
+ protected void loadOne(String filename) {
+ LoaderOps.inputFile(stream, filename, output, DataTickPoint, DataSuperTick);
+ }
+
+ @Override
+ public boolean bulkUseTransaction() {
+ return true;
+ }
+
+ @Override
+ public long countTriples() {
+ return countTriples;
+ }
+
+ @Override
+ public long countQuads() {
+ return countQuads;
+ }
+}
http://git-wip-us.apache.org/repos/asf/jena/blob/f5d9e6fa/jena-db/jena-tdb2/src/main/java/org/apache/jena/tdb2/store/DatasetPrefixesTDB.java
----------------------------------------------------------------------
diff --git a/jena-db/jena-tdb2/src/main/java/org/apache/jena/tdb2/store/DatasetPrefixesTDB.java b/jena-db/jena-tdb2/src/main/java/org/apache/jena/tdb2/store/DatasetPrefixesTDB.java
index 282eee5..90f0368 100644
--- a/jena-db/jena-tdb2/src/main/java/org/apache/jena/tdb2/store/DatasetPrefixesTDB.java
+++ b/jena-db/jena-tdb2/src/main/java/org/apache/jena/tdb2/store/DatasetPrefixesTDB.java
@@ -41,7 +41,7 @@ public class DatasetPrefixesTDB implements DatasetPrefixStorage
// See getPrefixMapping.
static final RecordFactory factory = new RecordFactory(3*NodeId.SIZE, 0) ;
- static final String unamedGraphURI = "" ;
+ public static final String unnamedGraphURI = "" ;
private final NodeTupleTable nodeTupleTable ;
@@ -153,7 +153,7 @@ public class DatasetPrefixesTDB implements DatasetPrefixStorage
/** Return a PrefixMapping for the unamed graph */
@Override
public PrefixMapping getPrefixMapping()
- { return getPrefixMapping(unamedGraphURI) ; }
+ { return getPrefixMapping(unnamedGraphURI) ; }
/** Return a PrefixMapping for a named graph */
@Override
http://git-wip-us.apache.org/repos/asf/jena/blob/f5d9e6fa/jena-db/jena-tdb2/src/test/java/org/apache/jena/tdb2/TC_TDB2.java
----------------------------------------------------------------------
diff --git a/jena-db/jena-tdb2/src/test/java/org/apache/jena/tdb2/TC_TDB2.java b/jena-db/jena-tdb2/src/test/java/org/apache/jena/tdb2/TC_TDB2.java
index 65601a1..5d6382b 100644
--- a/jena-db/jena-tdb2/src/test/java/org/apache/jena/tdb2/TC_TDB2.java
+++ b/jena-db/jena-tdb2/src/test/java/org/apache/jena/tdb2/TC_TDB2.java
@@ -24,6 +24,7 @@ import org.apache.jena.sparql.engine.optimizer.reorder.ReorderTransformation ;
import org.apache.jena.tdb2.assembler.TS_TDBAssembler;
import org.apache.jena.tdb2.graph.TS_Graph;
import org.apache.jena.tdb2.lib.TS_LibTDB;
+import org.apache.jena.tdb2.loader.TS_Loader;
import org.apache.jena.tdb2.setup.TS_TDBSetup;
import org.apache.jena.tdb2.solver.TS_SolverTDB;
import org.apache.jena.tdb2.store.TS_Store;
@@ -55,6 +56,7 @@ import org.junit.runners.Suite ;
, TS_Factory.class
, TS_TDBAssembler.class
, TS_Sys.class
+ , TS_Loader.class
} )
public class TC_TDB2
http://git-wip-us.apache.org/repos/asf/jena/blob/f5d9e6fa/jena-db/jena-tdb2/src/test/java/org/apache/jena/tdb2/loader/TS_Loader.java
----------------------------------------------------------------------
diff --git a/jena-db/jena-tdb2/src/test/java/org/apache/jena/tdb2/loader/TS_Loader.java b/jena-db/jena-tdb2/src/test/java/org/apache/jena/tdb2/loader/TS_Loader.java
new file mode 100644
index 0000000..8621cd4
--- /dev/null
+++ b/jena-db/jena-tdb2/src/test/java/org/apache/jena/tdb2/loader/TS_Loader.java
@@ -0,0 +1,29 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.jena.tdb2.loader;
+
+import org.junit.runner.RunWith;
+import org.junit.runners.Suite;
+
+@RunWith(Suite.class)
+@Suite.SuiteClasses( {
+ TestLoader.class
+})
+
+public class TS_Loader {}
http://git-wip-us.apache.org/repos/asf/jena/blob/f5d9e6fa/jena-db/jena-tdb2/src/test/java/org/apache/jena/tdb2/loader/TestLoader.java
----------------------------------------------------------------------
diff --git a/jena-db/jena-tdb2/src/test/java/org/apache/jena/tdb2/loader/TestLoader.java b/jena-db/jena-tdb2/src/test/java/org/apache/jena/tdb2/loader/TestLoader.java
new file mode 100644
index 0000000..c9a024a
--- /dev/null
+++ b/jena-db/jena-tdb2/src/test/java/org/apache/jena/tdb2/loader/TestLoader.java
@@ -0,0 +1,245 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.jena.tdb2.loader ;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.function.BiFunction;
+
+import org.apache.jena.atlas.iterator.Iter;
+import org.apache.jena.atlas.logging.LogCtl;
+import org.apache.jena.graph.Graph;
+import org.apache.jena.graph.Node ;
+import org.apache.jena.graph.NodeFactory ;
+import org.apache.jena.graph.Triple;
+import org.apache.jena.query.ARQ;
+import org.apache.jena.riot.RDFDataMgr;
+import org.apache.jena.sparql.core.DatasetGraph;
+import org.apache.jena.sparql.core.Quad;
+import org.apache.jena.sparql.util.IsoMatcher;
+import org.apache.jena.sys.JenaSystem;
+import org.apache.jena.system.Txn;
+import org.apache.jena.tdb2.DatabaseMgr;
+import org.apache.jena.tdb2.TDB2;
+import org.apache.jena.tdb2.loader.DataLoader;
+import org.apache.jena.tdb2.loader.LoaderFactory;
+import org.apache.jena.tdb2.loader.base.LoaderOps;
+import org.apache.jena.tdb2.loader.base.MonitorOutput;
+import org.junit.AfterClass;
+import org.junit.BeforeClass;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.junit.runners.Parameterized;
+import org.junit.runners.Parameterized.Parameters;
+
+@RunWith(Parameterized.class)
+public class TestLoader {
+ static { JenaSystem.init(); }
+
+ @Parameters(name = "{index}: {0}")
+ public static Iterable<Object[]> data() {
+ List<Object[]> x = new ArrayList<>() ;
+ BiFunction<DatasetGraph, Node, DataLoader> basic = (dsg, gn)->LoaderFactory.basicLoader(dsg, gn, output);
+ BiFunction<DatasetGraph, Node, DataLoader> sequential = (dsg, gn)->LoaderFactory.sequentialLoader(dsg, gn, output);
+ BiFunction<DatasetGraph, Node, DataLoader> parallel = (dsg, gn)->LoaderFactory.parallelLoader(dsg, gn, output);
+ x.add(new Object[]{"Basic loader", basic}) ;
+ x.add(new Object[]{"Sequential loader", sequential}) ;
+ x.add(new Object[]{"Parallel loader", parallel}) ;
+ return x ;
+ }
+
+ static MonitorOutput output = LoaderOps.nullOutput();
+ private String name;
+ private BiFunction<DatasetGraph, Node, DataLoader> maker;
+
+ public TestLoader(String name, BiFunction<DatasetGraph, Node, DataLoader> maker) {
+ this.name = name;
+ this.maker = maker;
+ }
+
+ /** Load the contents of files or remote web data into a dataset. */
+ static void load(DatasetGraph dataset, BiFunction<DatasetGraph, Node, DataLoader> maker, String ... dataURLs) {
+ load(dataset, null, maker, dataURLs);
+ }
+
+ static void load(DatasetGraph dataset, Node graphName, BiFunction<DatasetGraph, Node, DataLoader> maker, String ... dataURLs) {
+ DataLoader loader = maker.apply(dataset, graphName);
+ loader.startBulk();
+ try {
+ loader.load(dataURLs);
+ loader.finishBulk();
+ }
+ catch (RuntimeException ex) {
+ loader.finishException(ex);
+ throw ex;
+ }
+ }
+
+ private static String DIR = "testing/Loader/";
+ private static final Node g = NodeFactory.createURI("g") ;
+ private static final Node s = NodeFactory.createURI("s") ;
+ private static final Node p = NodeFactory.createURI("p") ;
+ private static final Node o = NodeFactory.createURI("o") ;
+ private static final Node o1 = NodeFactory.createURI("o1") ;
+ private static final Node o2 = NodeFactory.createURI("o2") ;
+
+ private static final Node gn = NodeFactory.createURI("http://example/g") ;
+
+ @BeforeClass
+ static public void beforeClass() {
+ LogCtl.disable(ARQ.logExecName) ;
+ //LogCtl.disable(TDB2.logLoaderName) ;
+ }
+
+ @AfterClass
+ static public void afterClass() {
+ LogCtl.enable(ARQ.logExecName) ;
+ LogCtl.enable(TDB2.logLoaderName) ;
+ }
+
+ static DatasetGraph fresh() {
+ return DatabaseMgr.createDatasetGraph() ;
+ }
+
+ @Test
+ public void load_dataset_01() {
+ DatasetGraph dsg = fresh() ;
+ load(dsg, maker, DIR + "data-1.nq") ;
+ Txn.executeRead(dsg, ()->{
+ assertTrue(dsg.getDefaultGraph().isEmpty()) ;
+ assertEquals(1, dsg.getGraph(g).size()) ;
+ });
+ }
+
+ @Test
+ public void load_dataset_02() {
+ DatasetGraph dsg = fresh() ;
+ load(dsg, maker , DIR + "data-1.nq", DIR + "data-2.nt") ;
+ Txn.executeRead(dsg, ()->{
+ assertEquals(1, dsg.getGraph(g).size()) ;
+ assertEquals(2, dsg.getDefaultGraph().size());
+ });
+ }
+
+ @Test
+ public void load_dataset_03() {
+ DatasetGraph dsg = fresh();
+ DataLoader loader = maker.apply(dsg, null);
+ loader.startBulk();
+ RDFDataMgr.parse(loader.stream(), DIR + "data-1.nq");
+ loader.finishBulk();
+ Txn.executeRead(dsg, ()->{
+ assertTrue(dsg.getDefaultGraph().isEmpty()) ;
+ assertEquals(1, dsg.getGraph(g).size()) ;
+ });
+ }
+
+ @Test
+ public void load_dataset_04() {
+ DatasetGraph dsg = fresh() ;
+ load(dsg, maker, DIR + "data-3.trig") ;
+ Txn.executeRead(dsg, ()->{
+ String uri = dsg.getDefaultGraph().getPrefixMapping().getNsPrefixURI("") ;
+ assertEquals("http://example/", uri) ;
+ });
+ }
+
+ @Test public void isomorphic_1() {
+ DatasetGraph dsg = fresh() ;
+ load(dsg, maker , DIR + "data-1.nq", DIR + "data-2.nt") ;
+ Txn.executeRead(dsg, ()->{
+ assertEquals(1, dsg.getGraph(g).size()) ;
+ assertEquals(2, dsg.getDefaultGraph().size());
+ });
+
+ DatasetGraph dsg1 = RDFDataMgr.loadDatasetGraph(DIR + "data-1.nq");
+ RDFDataMgr.read(dsg1, DIR + "data-2.nt");
+ Txn.executeRead(dsg, ()->{
+ boolean b = IsoMatcher.isomorphic(dsg1, dsg);
+ assertTrue("Not isomorphic", b);
+ });
+ }
+
+ @Test
+ public void load_dataset_indexes() {
+ DatasetGraph dsg = fresh() ;
+ load(dsg, maker, DIR + "data-1.nq", DIR + "data-2.nt") ;
+ Txn.executeRead(dsg, ()->{
+ assertEquals(2, dsg.getDefaultGraph().size()) ;
+ assertEquals(1, dsg.getGraph(g).size()) ;
+
+ {
+ // Check indexes.
+ Graph g1 = dsg.getDefaultGraph();
+ List<Triple> x = Iter.toList(g1.find(null, null, null)) ;
+ assertEquals(2, x.size()) ;
+ x = Iter.toList(g1.find(s, null, null)) ;
+ assertEquals(2, x.size()) ;
+ x = Iter.toList(g1.find(null, p, null)) ;
+ assertEquals(2, x.size()) ;
+ x = Iter.toList(g1.find(null, null, o1)) ;
+ assertEquals(1, x.size()) ;
+ }
+
+ {
+ Graph g2 = dsg.getGraph(g);
+ List<Triple> x = Iter.toList(g2.find(null, null, null));
+ assertEquals(1, x.size());
+ x = Iter.toList(g2.find(s, null, null));
+ assertEquals(1, x.size());
+ x = Iter.toList(g2.find(null, p, null));
+ assertEquals(1, x.size());
+ x = Iter.toList(g2.find(null, null, o));
+ assertEquals(1, x.size());
+ }
+
+ {
+ List<Quad> z = Iter.toList(dsg.find(null, null, null, null)) ;
+ assertEquals(3, z.size()) ;
+ z = Iter.toList(dsg.find(Quad.defaultGraphIRI, null, null, null)) ;
+ assertEquals(2, z.size()) ;
+ z = Iter.toList(dsg.find(g, null, null, null)) ;
+ assertEquals(1, z.size()) ;
+ z = Iter.toList(dsg.find(null, s, null, null)) ;
+ assertEquals(3, z.size()) ;
+ z = Iter.toList(dsg.find(null, null, p, null)) ;
+ assertEquals(3, z.size()) ;
+ z = Iter.toList(dsg.find(g, null, p, null)) ;
+ assertEquals(1, z.size()) ;
+ z = Iter.toList(dsg.find(null, null, null, o)) ;
+ assertEquals(1, z.size()) ;
+ }
+ });
+ }
+
+ @Test
+ public void load_graph_1() {
+ DatasetGraph dsg = fresh() ;
+ load(dsg, gn, maker, DIR + "data-2.nt") ;
+ Txn.executeRead(dsg, ()->{
+ assertEquals(0, dsg.getDefaultGraph().size());
+ assertEquals(2, dsg.getGraph(gn).size());
+ });
+ }
+
+ // Try to load quads.
+}
http://git-wip-us.apache.org/repos/asf/jena/blob/f5d9e6fa/jena-db/jena-tdb2/testing/Loader/data-2.nt
----------------------------------------------------------------------
diff --git a/jena-db/jena-tdb2/testing/Loader/data-2.nt b/jena-db/jena-tdb2/testing/Loader/data-2.nt
index 81fc25f..11cb793 100644
--- a/jena-db/jena-tdb2/testing/Loader/data-2.nt
+++ b/jena-db/jena-tdb2/testing/Loader/data-2.nt
@@ -1 +1,2 @@
-<s> <p> <o> .
\ No newline at end of file
+<s> <p> <o1> .
+<s> <p> <o2> .
http://git-wip-us.apache.org/repos/asf/jena/blob/f5d9e6fa/jena-db/jena-tdb2/testing/Loader/data-3.trig
----------------------------------------------------------------------
diff --git a/jena-db/jena-tdb2/testing/Loader/data-3.trig b/jena-db/jena-tdb2/testing/Loader/data-3.trig
index 8000cc1..03fd2b3 100644
--- a/jena-db/jena-tdb2/testing/Loader/data-3.trig
+++ b/jena-db/jena-tdb2/testing/Loader/data-3.trig
@@ -1,3 +1,3 @@
prefix : <http://example/>
-:g { :s :p :o }
+:g { :s :p 123 }
http://git-wip-us.apache.org/repos/asf/jena/blob/f5d9e6fa/jena-db/jena-tdb2/testing/Loader/data-4.ttl
----------------------------------------------------------------------
diff --git a/jena-db/jena-tdb2/testing/Loader/data-4.ttl b/jena-db/jena-tdb2/testing/Loader/data-4.ttl
index 49678f1..c4e188a 100644
--- a/jena-db/jena-tdb2/testing/Loader/data-4.ttl
+++ b/jena-db/jena-tdb2/testing/Loader/data-4.ttl
@@ -1,4 +1,4 @@
prefix : <http://example/>
-:s :p :o .
+:s :p _:a .
http://git-wip-us.apache.org/repos/asf/jena/blob/f5d9e6fa/jena-tdb/src/main/java/org/apache/jena/tdb/TDBLoader.java
----------------------------------------------------------------------
diff --git a/jena-tdb/src/main/java/org/apache/jena/tdb/TDBLoader.java b/jena-tdb/src/main/java/org/apache/jena/tdb/TDBLoader.java
index efc2f95..0183f2f 100644
--- a/jena-tdb/src/main/java/org/apache/jena/tdb/TDBLoader.java
+++ b/jena-tdb/src/main/java/org/apache/jena/tdb/TDBLoader.java
@@ -35,37 +35,31 @@ import org.slf4j.Logger ;
*/
public class TDBLoader
{
- /** Load the contents of URL into a dataset. URL must name a quads format file (NQuads or TriG - NTriples is also accepted).
- * To a triples format, use {@link #load(GraphNonTxnTDB, String)}
- * or {@link #load(DatasetGraphTDB, List, boolean, boolean)}
- */
+ /** Load the contents of URL into a dataset. */
public static void load(DatasetGraphTDB dataset, String url)
{
load(dataset, url, false) ;
}
- /** Load the contents of URL into a dataset. URL must name a quads format file (NQuads or TriG - NTriples is also accepted).
- * To a triples format, use {@link #load(GraphNonTxnTDB, String, boolean)}
- * or {@link #load(DatasetGraphTDB, List, boolean, boolean)}
- */
+ /**
+ * Load the contents of URL into a dataset.
+ */
public static void load(DatasetGraphTDB dataset, String url, boolean showProgress)
{
load(dataset, asList(url), showProgress, true) ;
}
- /** Load the contents of URL into a dataset. URL must name a quads format file (NQuads or TriG - NTriples is also accepted).
- * To load a triples format, use {@link #load(GraphNonTxnTDB, List, boolean)}
- * or {@link #load(DatasetGraphTDB, List, boolean, boolean)}
- */
+ /**
+ * Load the contents of URL into a dataset.
+ */
public static void load(DatasetGraphTDB dataset, List<String> urls)
{
load(dataset, urls, false, true) ;
}
- /** Load the contents of URL into a dataset. URL must name a quads format file (NQuads or TriG - NTriples is also accepted).
- * To load a triples format, use {@link #load(GraphNonTxnTDB, List, boolean)}
- * or {@link #load(DatasetGraphTDB, List, boolean, boolean)}
- */
+ /**
+ * Load the contents of URL into a dataset.
+ */
public static void load(DatasetGraphTDB dataset, List<String> urls, boolean showProgress, boolean generateStats)
{
TDBLoader loader = new TDBLoader() ;
@@ -74,9 +68,9 @@ public class TDBLoader
loader.loadDataset(dataset, urls) ;
}
- /** Load the contents of URL into a dataset. Input is NQUADS.
- * To load a triples format, use {@link #loadGraph}
- */
+ /**
+ * Load the contents of URL into a dataset. Input is N-Quads format.
+ */
public static void load(DatasetGraphTDB dataset, InputStream input, boolean showProgress)
{
TDBLoader loader = new TDBLoader() ;