You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@jena.apache.org by an...@apache.org on 2018/10/11 16:41:11 UTC
[3/5] jena git commit: JENA-1616: Better progress output.
JENA-1616: Better progress output.
Project: http://git-wip-us.apache.org/repos/asf/jena/repo
Commit: http://git-wip-us.apache.org/repos/asf/jena/commit/f5de3418
Tree: http://git-wip-us.apache.org/repos/asf/jena/tree/f5de3418
Diff: http://git-wip-us.apache.org/repos/asf/jena/diff/f5de3418
Branch: refs/heads/master
Commit: f5de3418f8b619648e6cc863050db48d704ade91
Parents: 0dbc1c2
Author: Andy Seaborne <an...@apache.org>
Authored: Tue Oct 9 16:53:37 2018 +0100
Committer: Andy Seaborne <an...@apache.org>
Committed: Tue Oct 9 16:53:37 2018 +0100
----------------------------------------------------------------------
.../java/org/apache/jena/atlas/lib/Timer.java | 18 ++--
jena-cmds/src/main/java/tdb2/tdbloader.java | 20 +++++
.../jena/tdb2/loader/base/LoaderBase.java | 54 ++++++++++--
.../apache/jena/tdb2/loader/base/LoaderOps.java | 9 +-
.../jena/tdb2/loader/base/ProgressMonitor.java | 19 ++++-
.../tdb2/loader/base/ProgressMonitorBasic.java | 48 +++++++++--
.../loader/base/ProgressMonitorContext.java | 61 ++++++++++++++
.../tdb2/loader/base/ProgressMonitorOutput.java | 88 +++++++++++++++-----
.../tdb2/loader/base/ProgressStreamRDF.java | 16 ----
.../jena/tdb2/loader/basic/LoaderBasic.java | 6 +-
.../jena/tdb2/loader/main/LoaderMain.java | 12 +--
.../sequential/BuilderSecondaryIndexes.java | 16 ++--
.../loader/sequential/LoaderSequential.java | 8 +-
13 files changed, 290 insertions(+), 85 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/jena/blob/f5de3418/jena-base/src/main/java/org/apache/jena/atlas/lib/Timer.java
----------------------------------------------------------------------
diff --git a/jena-base/src/main/java/org/apache/jena/atlas/lib/Timer.java b/jena-base/src/main/java/org/apache/jena/atlas/lib/Timer.java
index 340ba37..c3e1ee4 100644
--- a/jena-base/src/main/java/org/apache/jena/atlas/lib/Timer.java
+++ b/jena-base/src/main/java/org/apache/jena/atlas/lib/Timer.java
@@ -32,13 +32,12 @@ public class Timer {
public void startTimer() {
if ( inTimer )
throw new AtlasException("Already in timer") ;
-
timeStart = System.currentTimeMillis() ;
timeFinish = -1 ;
inTimer = true ;
}
- /** Return time in milliseconds */
+ /** Stop timing and return the elapsed time in milliseconds */
public long endTimer() {
if ( !inTimer )
throw new AtlasException("Not in timer") ;
@@ -47,21 +46,30 @@ public class Timer {
return getTimeInterval() ;
}
+ /** Read the timer - either the instantanious value (if running) or elapsed time (if finished). */
+ public long read() {
+ return inTimer
+ ? System.currentTimeMillis() - timeStart
+ : timeFinish - timeStart;
+ }
+
+ /** Read a running timer */
public long readTimer() {
if ( !inTimer )
throw new AtlasException("Not in timer") ;
- return System.currentTimeMillis() - timeStart ;
+ return read() ;
}
+ /** Read an elapsed timer */
public long getTimeInterval() {
if ( inTimer )
throw new AtlasException("Still timing") ;
if ( timeFinish == -1 )
throw new AtlasException("No valid interval") ;
-
- return timeFinish - timeStart ;
+ return read() ;
}
+ /** Helper function to format milliseconds as "%.3f" seconds */
static public String timeStr(long timeInterval) {
return String.format("%.3f", timeInterval / 1000.0) ;
}
http://git-wip-us.apache.org/repos/asf/jena/blob/f5de3418/jena-cmds/src/main/java/tdb2/tdbloader.java
----------------------------------------------------------------------
diff --git a/jena-cmds/src/main/java/tdb2/tdbloader.java b/jena-cmds/src/main/java/tdb2/tdbloader.java
index 1ee896e..2473a8f 100644
--- a/jena-cmds/src/main/java/tdb2/tdbloader.java
+++ b/jena-cmds/src/main/java/tdb2/tdbloader.java
@@ -18,12 +18,16 @@
package tdb2;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.Paths;
import java.util.List;
import java.util.Objects;
import jena.cmd.ArgDecl;
import jena.cmd.CmdException;
import org.apache.jena.atlas.lib.InternalErrorException;
+import org.apache.jena.atlas.lib.ListUtils;
import org.apache.jena.atlas.lib.Timer;
import org.apache.jena.graph.Node;
import org.apache.jena.graph.NodeFactory;
@@ -109,6 +113,8 @@ public class tdbloader extends CmdTDBGraph {
List<String> urls = getPositional();
if ( urls.size() == 0 )
urls.add("-");
+ else
+ checkFiles(urls);
if ( graphName == null ) {
loadQuads(urls);
@@ -128,6 +134,20 @@ public class tdbloader extends CmdTDBGraph {
loadTriples(graphName, urls);
}
+ // Check files exists before starting.
+ private void checkFiles(List<String> urls) {
+ List<String> problemFiles =
+ ListUtils.toList(
+ urls.stream()
+ .map(Paths::get)
+ .filter(p-> !Files.exists(p) || !Files.isRegularFile(p /*follow links*/) || !Files.isReadable(p) )
+ .map(Path::toString)
+ );
+ if ( ! problemFiles.isEmpty() ) {
+ throw new CmdException("Can't read files : ["+problemFiles+"]");
+ }
+ }
+
private void loadTriples(String graphName, List<String> urls) {
execBulkLoad(super.getDatasetGraph(), graphName, urls, showProgress);
}
http://git-wip-us.apache.org/repos/asf/jena/blob/f5de3418/jena-db/jena-tdb2/src/main/java/org/apache/jena/tdb2/loader/base/LoaderBase.java
----------------------------------------------------------------------
diff --git a/jena-db/jena-tdb2/src/main/java/org/apache/jena/tdb2/loader/base/LoaderBase.java b/jena-db/jena-tdb2/src/main/java/org/apache/jena/tdb2/loader/base/LoaderBase.java
index ce205e0..544310e 100644
--- a/jena-db/jena-tdb2/src/main/java/org/apache/jena/tdb2/loader/base/LoaderBase.java
+++ b/jena-db/jena-tdb2/src/main/java/org/apache/jena/tdb2/loader/base/LoaderBase.java
@@ -26,13 +26,13 @@ import org.apache.jena.query.TxnType;
import org.apache.jena.sparql.core.DatasetGraph;
import org.apache.jena.tdb2.loader.DataLoader;
-/** Simple bulk loader framework.
+/**
+ * Bulk loader framework.
* <p>
* It puts a write-transaction around the whole process if {@link #bulkUseTransaction}
- * returns true and then calls abstract {@link #loadOne(String)}
- * for each file.
+ * returns true. It calls {@link #loadOne} for each file.
* <p>
- * If a graph name is provided, it converts triples to quads in that named graph.
+ * If a graph name is provided, it converts triples to quads in that named graph.
*/
public abstract class LoaderBase implements DataLoader {
@@ -62,7 +62,7 @@ public abstract class LoaderBase implements DataLoader {
dsg.end();
}
long totalElapsed = timer.endTimer();
- outputTime(totalElapsed);
+ outputSummary(totalElapsed);
}
@Override
@@ -75,21 +75,57 @@ public abstract class LoaderBase implements DataLoader {
@Override
public void load(List<String> filenames) {
- // Default implementation.
+ if ( filenames.isEmpty() ) {
+ output.print("No files to load");
+ return;
+ }
+
+ ProgressMonitor monitor = createProgressMonitor(output);
+
+ boolean multipleFiles = (filenames.size()>1);
+
+ String file1 = null;
+ if ( ! multipleFiles ) {
+ file1 = filenames.get(0);
+ if ( file1.equals("-") )
+ file1 = "stdin";
+ }
try {
- filenames.forEach(fn->loadOne(fn));
+ if ( multipleFiles )
+ monitor.startMessage("Start: "+filenames.size()+" files");
+ else
+ monitor.startMessage("Start: "+file1);
+ monitor.start();
+ filenames.forEach(fn->{
+ if ( multipleFiles )
+ monitor.startSection();
+ loadOne(fn, monitor);
+ if ( multipleFiles )
+ monitor.finishSection();
+ });
+ monitor.finish();
+ if ( multipleFiles )
+ monitor.finishMessage("Finished: "+filenames.size()+" files");
+ else
+ monitor.finishMessage("Finished: "+file1);
} catch (Exception ex) {
finishException(ex);
throw ex;
}
}
+ protected abstract ProgressMonitor createProgressMonitor(MonitorOutput output);
+
/** Subclasses must provide a setting. */
protected abstract boolean bulkUseTransaction();
- protected abstract void loadOne(String filename);
+ protected void loadOne(String filename, ProgressMonitor monitor) {
+ String label = LoaderOps.label(filename);
+ monitor.setLabel(label);
+ LoaderOps.inputFile(stream(), filename, monitor);
+ }
- protected void outputTime(long totalElapsed) {
+ protected void outputSummary(long totalElapsed) {
if ( output != null ) {
long count = countTriples()+countQuads();
String label = "Triples/Quads";
http://git-wip-us.apache.org/repos/asf/jena/blob/f5de3418/jena-db/jena-tdb2/src/main/java/org/apache/jena/tdb2/loader/base/LoaderOps.java
----------------------------------------------------------------------
diff --git a/jena-db/jena-tdb2/src/main/java/org/apache/jena/tdb2/loader/base/LoaderOps.java b/jena-db/jena-tdb2/src/main/java/org/apache/jena/tdb2/loader/base/LoaderOps.java
index 88b95b7..89e8941 100644
--- a/jena-db/jena-tdb2/src/main/java/org/apache/jena/tdb2/loader/base/LoaderOps.java
+++ b/jena-db/jena-tdb2/src/main/java/org/apache/jena/tdb2/loader/base/LoaderOps.java
@@ -79,7 +79,7 @@ public class LoaderOps {
}
/** Calculate a label for a progress monitor. */
- private static String label(String fileName) {
+ public static String label(String fileName) {
String basename = FileOps.splitDirFile(fileName).get(1);
return basename;
}
@@ -100,17 +100,16 @@ public class LoaderOps {
* "no output".
*/
public static void inputFile(StreamRDF sink, String source, ProgressMonitor monitor) {
-
if ( monitor != null ) {
sink = new ProgressStreamRDF(sink, monitor);
- monitor.start();
+ //monitor.start();
}
sink.start();
RDFDataMgr.parse(sink, source);
sink.finish();
if ( monitor != null ) {
- monitor.finish();
- monitor.finishMessage("Data: " + source);
+ //monitor.finish();
+ //monitor.finishMessage("Data");
}
}
http://git-wip-us.apache.org/repos/asf/jena/blob/f5de3418/jena-db/jena-tdb2/src/main/java/org/apache/jena/tdb2/loader/base/ProgressMonitor.java
----------------------------------------------------------------------
diff --git a/jena-db/jena-tdb2/src/main/java/org/apache/jena/tdb2/loader/base/ProgressMonitor.java b/jena-db/jena-tdb2/src/main/java/org/apache/jena/tdb2/loader/base/ProgressMonitor.java
index 7f114b5..9c3934b 100644
--- a/jena-db/jena-tdb2/src/main/java/org/apache/jena/tdb2/loader/base/ProgressMonitor.java
+++ b/jena-db/jena-tdb2/src/main/java/org/apache/jena/tdb2/loader/base/ProgressMonitor.java
@@ -20,25 +20,31 @@ package org.apache.jena.tdb2.loader.base;
/** Interface {@code ProgressMonitor} - monitor progress.*/
public interface ProgressMonitor {
- public void startMessage();
-
/** Output the starting message.
* The format is implementation dependent.
*/
public void startMessage(String message);
-
+
/**
* Output the finishing message.
* The format is implementation dependent.
*/
public void finishMessage(String message);
+ public void setLabel(String label);
+
/** Start and start timing. This should be paired with a call to {@link #finish()}. */
public void start();
+
+ /** Start a section within the overall start-finish. */
+ public void startSection();
+ /** Finish a section within the overall start-finish. */
+ public void finishSection();
+
/**
* Finish and stop timing. The total time is available with {@link #getTime} and the
- * numbe rof items processes with {@link #getTicks()}.
+ * number of items processes with {@link #getTicks()}.
*/
public void finish();
@@ -51,4 +57,9 @@ public interface ProgressMonitor {
/** Return the elapsed time taken - this is only valid after {@link #finish()} has been called. */
public long getTime();
+ /** Return the number of ticks. Valid after {@link #startSection()} has been called. */
+ public long getSectionTicks();
+
+ /** Return the elapsed section time taken. */
+ public long getSectionTime();
}
http://git-wip-us.apache.org/repos/asf/jena/blob/f5de3418/jena-db/jena-tdb2/src/main/java/org/apache/jena/tdb2/loader/base/ProgressMonitorBasic.java
----------------------------------------------------------------------
diff --git a/jena-db/jena-tdb2/src/main/java/org/apache/jena/tdb2/loader/base/ProgressMonitorBasic.java b/jena-db/jena-tdb2/src/main/java/org/apache/jena/tdb2/loader/base/ProgressMonitorBasic.java
index 81ee9a4..19ccc2b 100644
--- a/jena-db/jena-tdb2/src/main/java/org/apache/jena/tdb2/loader/base/ProgressMonitorBasic.java
+++ b/jena-db/jena-tdb2/src/main/java/org/apache/jena/tdb2/loader/base/ProgressMonitorBasic.java
@@ -19,20 +19,24 @@
package org.apache.jena.tdb2.loader.base;
import org.apache.jena.atlas.lib.Timer;
+import org.apache.jena.tdb2.TDBException;
/** Simple {@link ProgressMonitor} that records time and ticks but does not print anything */
public class ProgressMonitorBasic implements ProgressMonitor {
-
+ // Overall
private Timer timer = new Timer();
private long timeInMillis = -1;
private long tickCounter = 0;
+ //Section
+ private boolean inSection = false;
+ private int sectionCounter = 0;
+ private Timer sectionTimer = null;
+ private long sectionTimeInMillis = -1;
+ private long sectionTickCounter = 0;
public ProgressMonitorBasic() {}
@Override
- public void startMessage() {}
-
- @Override
public void startMessage(String message) {}
@Override
@@ -49,7 +53,30 @@ public class ProgressMonitorBasic implements ProgressMonitor {
}
@Override
- public void tick() { tickCounter++; }
+ public void startSection() {
+ if ( inSection )
+ throw new TDBException("startSection: Already in section");
+ inSection = true;
+ sectionCounter++;
+ sectionTimer = new Timer();
+ sectionTimeInMillis = 0;
+ sectionTickCounter = 0;
+ }
+
+ @Override
+ public void finishSection() {
+ if ( ! inSection )
+ throw new TDBException("finishSection: Not in section");
+ inSection = false;
+ sectionTimeInMillis = sectionTimer.endTimer();
+ }
+
+ @Override
+ public void tick() {
+ tickCounter++;
+ if ( inSection )
+ sectionTickCounter++;
+ }
@Override
public long getTicks() {
@@ -61,4 +88,15 @@ public class ProgressMonitorBasic implements ProgressMonitor {
return timeInMillis;
}
+ @Override
+ public long getSectionTicks() {
+ return sectionTickCounter;
+ }
+ @Override
+ public long getSectionTime() {
+ return sectionTimeInMillis;
+ }
+
+ @Override
+ public void setLabel(String label) {}
}
http://git-wip-us.apache.org/repos/asf/jena/blob/f5de3418/jena-db/jena-tdb2/src/main/java/org/apache/jena/tdb2/loader/base/ProgressMonitorContext.java
----------------------------------------------------------------------
diff --git a/jena-db/jena-tdb2/src/main/java/org/apache/jena/tdb2/loader/base/ProgressMonitorContext.java b/jena-db/jena-tdb2/src/main/java/org/apache/jena/tdb2/loader/base/ProgressMonitorContext.java
new file mode 100644
index 0000000..14d9811
--- /dev/null
+++ b/jena-db/jena-tdb2/src/main/java/org/apache/jena/tdb2/loader/base/ProgressMonitorContext.java
@@ -0,0 +1,61 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.jena.tdb2.loader.base;
+
+import org.apache.jena.atlas.lib.Timer;
+import org.apache.jena.atlas.logging.Log;
+
+/**
+ * The counting state of a {@link ProgressMonitor}. This can be used to across different
+ * {@code ProgressMonitor}s (sequentially) to give running totals
+ */
+
+public class ProgressMonitorContext {
+ /*package*/ long ticks;
+ /*package*/ Timer timer;
+ private int depth = 0;
+
+ public ProgressMonitorContext(long ticks, Timer timer) {
+ super();
+ this.ticks = ticks;
+ this.timer = timer;
+ }
+
+ public void tick() { ticks++; }
+
+ public void start() {
+ if ( depth == 0 )
+ timer.startTimer();
+ depth++;
+ }
+ public void finish() {
+ --depth;
+ if ( depth < 0 ) {
+ Log.error(this, "Misaligned start/finish");
+ return;
+ }
+ if ( depth == 0 )
+ timer.endTimer();
+ }
+
+ //public int getDepth() { return depth; }
+
+ public long getElapsed() { return depth < 0 ? -1 : timer.readTimer(); }
+
+}
http://git-wip-us.apache.org/repos/asf/jena/blob/f5de3418/jena-db/jena-tdb2/src/main/java/org/apache/jena/tdb2/loader/base/ProgressMonitorOutput.java
----------------------------------------------------------------------
diff --git a/jena-db/jena-tdb2/src/main/java/org/apache/jena/tdb2/loader/base/ProgressMonitorOutput.java b/jena-db/jena-tdb2/src/main/java/org/apache/jena/tdb2/loader/base/ProgressMonitorOutput.java
index 9720069..552d2de 100644
--- a/jena-db/jena-tdb2/src/main/java/org/apache/jena/tdb2/loader/base/ProgressMonitorOutput.java
+++ b/jena-db/jena-tdb2/src/main/java/org/apache/jena/tdb2/loader/base/ProgressMonitorOutput.java
@@ -23,6 +23,7 @@ import static org.apache.jena.atlas.lib.DateTimeUtils.nowAsString ;
import java.util.Objects;
import org.apache.jena.atlas.lib.Timer;
+import org.apache.jena.tdb2.TDBException;
import org.slf4j.Logger ;
/** Progress monitor - output lines to show the progress of some long running operation.
@@ -34,9 +35,17 @@ public class ProgressMonitorOutput implements ProgressMonitor {
private final long tickPoint;
private final int superTick;
private final Timer timer;
- private Timer getTimer() { return timer; }
+ private Timer getTimer() { return timer; }
+
+ //Section
+ private boolean inSection = false;
+ private int sectionCounter = 0;
+ private Timer sectionTimer = null;
+ private long sectionTimeInMillis = -1;
+ private long sectionTickCounter = 0;
- private final String label;
+ // Current label.
+ private String label;
// Counters - this monitor.
private long counterBatch = 0;
@@ -72,41 +81,38 @@ public class ProgressMonitorOutput implements ProgressMonitor {
*/
public ProgressMonitorOutput(String label, long tickPoint, int superTick, MonitorOutput output) {
this.output = output;
- this.label = label;
+ setLabel(label);
this.tickPoint = tickPoint;
this.superTick = superTick;
this.timer = new Timer();
}
- /** Print a start message using the label */
- @Override
- public void startMessage() {
- startMessage(null) ;
- }
+// /** Print a start message using the label */
+// @Override
+// public void startMessage() {
+// startMessage(null) ;
+// }
/** Print a start message using a different string. */
@Override
public void startMessage(String msg) {
if ( msg != null )
output.print(msg) ;
- else
- output.print("Start: "+label) ;
}
+ //public void startSource(String msg) {
+ //public void finishSource(String msg) {
+
@Override
public void finishMessage(String msg) {
// Elapsed.
- long timePoint = getTimer().getTimeInterval();
- long counterTotalMsg = getRunningTotal();
-
- // /1000L is milli to second conversion
+ long timePoint = getTimer().read();
if ( timePoint != 0 ) {
double time = timePoint / 1000.0;
- long runAvgRate = (counterTotalMsg * 1000L) / timePoint;
-
- print("%s: %,d %s %.2fs (Avg: %,d)", msg, counterTotalMsg, label, time, runAvgRate);
+ long runAvgRate = (getRunningTotal() * 1000L) / timePoint;
+ print("%s: %,d tuples in %.2fs (Avg: %,d)", msg, getTicks(), time, runAvgRate);
} else
- print("%s: %,d %s (Avg: ----)", msg, counterTotalMsg, label);
+ print("%s: %,d (Avg: ----)", msg, getTicks());
}
@Override
@@ -118,14 +124,19 @@ public class ProgressMonitorOutput implements ProgressMonitor {
@Override
public void finish() {
- timeTotalMillis = getTimer().endTimer();
+ // XXX
+ getTimer().endTimer();
+ timeTotalMillis = getTimer().getTimeInterval();
}
@Override
public void tick() {
+ // The ticking
counterBatch++;
counterTotal++;
-
+ if ( inSection )
+ sectionTickCounter++;
+ // Report overall progress
if ( tickPoint(getRunningTotal(), tickPoint) ) {
long timePoint = getTimer().readTimer();
long thisTime = timePoint - lastTime;
@@ -133,6 +144,8 @@ public class ProgressMonitorOutput implements ProgressMonitor {
// *1000L is milli to second conversion
if ( thisTime != 0 && timePoint != 0 ) {
long batchAvgRate = (counterBatch * 1000L) / thisTime;
+ // XXX Too large : first after file switch. ???timePoint is wrong.
+ //System.err.printf("** %d %d\n",getRunningTotal(), timePoint );
long runAvgRate = (getRunningTotal() * 1000L) / timePoint;
print("Add: %,d %s (Batch: %,d / Avg: %,d)", getRunningTotal(), label, batchAvgRate, runAvgRate);
} else {
@@ -161,6 +174,36 @@ public class ProgressMonitorOutput implements ProgressMonitor {
public long getTime() {
return timeTotalMillis;
}
+
+ @Override
+ public void startSection() {
+ if ( inSection )
+ throw new TDBException("startSection: Already in section");
+ inSection = true;
+ sectionCounter++;
+ sectionTimer = new Timer();
+ sectionTimer.startTimer();
+ sectionTimeInMillis = 0;
+ sectionTickCounter = 0;
+ }
+
+ @Override
+ public void finishSection() {
+ if ( ! inSection )
+ throw new TDBException("finishSection: Not in section");
+ print(" End file: %s (triples/quads = %,d)", label, sectionTickCounter);
+ inSection = false;
+ sectionTimeInMillis = sectionTimer.endTimer();
+ }
+
+ @Override
+ public long getSectionTicks() {
+ return sectionTickCounter;
+ }
+ @Override
+ public long getSectionTime() {
+ return sectionTimeInMillis;
+ }
protected void elapsed(long timerReading) {
float elapsedSecs = timerReading / 1000F;
@@ -177,4 +220,9 @@ public class ProgressMonitorOutput implements ProgressMonitor {
return counter % quantum == 0;
}
+ @Override
+ public void setLabel(String label) {
+ this.label = label;
+ }
+
}
http://git-wip-us.apache.org/repos/asf/jena/blob/f5de3418/jena-db/jena-tdb2/src/main/java/org/apache/jena/tdb2/loader/base/ProgressStreamRDF.java
----------------------------------------------------------------------
diff --git a/jena-db/jena-tdb2/src/main/java/org/apache/jena/tdb2/loader/base/ProgressStreamRDF.java b/jena-db/jena-tdb2/src/main/java/org/apache/jena/tdb2/loader/base/ProgressStreamRDF.java
index e6c7b59..39b31af 100644
--- a/jena-db/jena-tdb2/src/main/java/org/apache/jena/tdb2/loader/base/ProgressStreamRDF.java
+++ b/jena-db/jena-tdb2/src/main/java/org/apache/jena/tdb2/loader/base/ProgressStreamRDF.java
@@ -36,22 +36,6 @@ public class ProgressStreamRDF extends StreamRDFWrapper {
this.monitor = monitor ;
}
- // Better that the app call start/finish on the monitor so that a number of
- // inputs on the stream can call start/finish. i.e the monitor can be used
- // for a batch of oeprations.
-
-// @Override
-// public void start() {
-// monitor.start();
-// super.start();
-// }
-//
-// @Override
-// public void finish() {
-// super.finish();
-// monitor.finish();
-// }
-
@Override
public void triple(Triple triple) {
super.triple(triple);
http://git-wip-us.apache.org/repos/asf/jena/blob/f5de3418/jena-db/jena-tdb2/src/main/java/org/apache/jena/tdb2/loader/basic/LoaderBasic.java
----------------------------------------------------------------------
diff --git a/jena-db/jena-tdb2/src/main/java/org/apache/jena/tdb2/loader/basic/LoaderBasic.java b/jena-db/jena-tdb2/src/main/java/org/apache/jena/tdb2/loader/basic/LoaderBasic.java
index e30e0f5..41dbbbf 100644
--- a/jena-db/jena-tdb2/src/main/java/org/apache/jena/tdb2/loader/basic/LoaderBasic.java
+++ b/jena-db/jena-tdb2/src/main/java/org/apache/jena/tdb2/loader/basic/LoaderBasic.java
@@ -27,6 +27,8 @@ import org.apache.jena.sparql.core.DatasetGraph;
import org.apache.jena.tdb2.loader.base.LoaderBase;
import org.apache.jena.tdb2.loader.base.LoaderOps;
import org.apache.jena.tdb2.loader.base.MonitorOutput;
+import org.apache.jena.tdb2.loader.base.ProgressMonitor;
+import org.apache.jena.tdb2.loader.base.ProgressMonitorOutput;
/** Simple bulk loader. Algorithm: Parser to dataset. */
public class LoaderBasic extends LoaderBase {
@@ -53,8 +55,8 @@ public class LoaderBasic extends LoaderBase {
}
@Override
- protected void loadOne(String source) {
- LoaderOps.inputFile(dest, source, output, DataTickPoint, DataSuperTick);
+ protected ProgressMonitor createProgressMonitor(MonitorOutput output) {
+ return ProgressMonitorOutput.create(output, "<unset>", DataTickPoint, DataSuperTick);
}
@Override
http://git-wip-us.apache.org/repos/asf/jena/blob/f5de3418/jena-db/jena-tdb2/src/main/java/org/apache/jena/tdb2/loader/main/LoaderMain.java
----------------------------------------------------------------------
diff --git a/jena-db/jena-tdb2/src/main/java/org/apache/jena/tdb2/loader/main/LoaderMain.java b/jena-db/jena-tdb2/src/main/java/org/apache/jena/tdb2/loader/main/LoaderMain.java
index 8bd158d..8b0fdf9 100644
--- a/jena-db/jena-tdb2/src/main/java/org/apache/jena/tdb2/loader/main/LoaderMain.java
+++ b/jena-db/jena-tdb2/src/main/java/org/apache/jena/tdb2/loader/main/LoaderMain.java
@@ -96,16 +96,16 @@ public class LoaderMain extends LoaderBase implements DataLoader {
public static final int DataSuperTick = 10;
public static final int IndexTickPoint = 1_000_000;
public static final int IndexSuperTick = 10;
-
+
private final LoaderPlan loaderPlan;
-
+
private final DatasetGraphTDB dsgtdb;
private final StreamRDF stream;
private final Map<String, TupleIndex> indexMap;
private final StreamRDFCounting dataInput;
private final List<BulkStartFinish> dataProcess = new ArrayList<>();
-
+
public LoaderMain(LoaderPlan loaderPlan, DatasetGraph dsg, MonitorOutput output) {
this(loaderPlan, dsg, null, output);
}
@@ -284,7 +284,7 @@ public class LoaderMain extends LoaderBase implements DataLoader {
// Add to processes - we can wait later if we do not touched indexes being built.
processes.add(indexer);
PhasedOps.ReplayResult result = PhasedOps.replay(srcIdx, dest, output);
- // End read tranaction on srcIdx
+ // End read transaction on srcIdx
transaction.end();
String timeStr = "---";
@@ -327,7 +327,7 @@ public class LoaderMain extends LoaderBase implements DataLoader {
}
@Override
- protected void loadOne(String filename) {
- LoaderOps.inputFile(stream, filename, output, DataTickPoint, DataSuperTick);
+ protected ProgressMonitor createProgressMonitor(MonitorOutput output) {
+ return ProgressMonitorOutput.create(output, "<unset>", DataTickPoint, DataSuperTick);
}
}
http://git-wip-us.apache.org/repos/asf/jena/blob/f5de3418/jena-db/jena-tdb2/src/main/java/org/apache/jena/tdb2/loader/sequential/BuilderSecondaryIndexes.java
----------------------------------------------------------------------
diff --git a/jena-db/jena-tdb2/src/main/java/org/apache/jena/tdb2/loader/sequential/BuilderSecondaryIndexes.java b/jena-db/jena-tdb2/src/main/java/org/apache/jena/tdb2/loader/sequential/BuilderSecondaryIndexes.java
index b1bec81..8484a39 100644
--- a/jena-db/jena-tdb2/src/main/java/org/apache/jena/tdb2/loader/sequential/BuilderSecondaryIndexes.java
+++ b/jena-db/jena-tdb2/src/main/java/org/apache/jena/tdb2/loader/sequential/BuilderSecondaryIndexes.java
@@ -18,7 +18,6 @@
package org.apache.jena.tdb2.loader.sequential;
-import org.apache.jena.atlas.lib.Timer;
import org.apache.jena.tdb2.loader.base.LoaderOps;
import org.apache.jena.tdb2.loader.base.MonitorOutput;
import org.apache.jena.tdb2.loader.base.ProgressMonitor;
@@ -26,27 +25,24 @@ import org.apache.jena.tdb2.loader.base.ProgressMonitorOutput;
import org.apache.jena.tdb2.store.tupletable.TupleIndex;
/**
- * This interface is the mechanism for building indexes given that at leasts one index
+ * This interface is the mechanism for building indexes given that at least one index
* already exists (the "primary", which normally is SPO or GSPO).
*/
public class BuilderSecondaryIndexes
{
- public static void createSecondaryIndexes(MonitorOutput output, TupleIndex primaryIndex, TupleIndex[] secondaryIndexes)
- {
- Timer timer = new Timer() ;
- timer.startTimer() ;
+ public static void createSecondaryIndexes(MonitorOutput output, TupleIndex primaryIndex, TupleIndex[] secondaryIndexes) {
boolean printTiming = true;
for ( TupleIndex index : secondaryIndexes ) {
+ String msg = primaryIndex.getName()+"->"+index.getName();
if ( index != null ) {
- ProgressMonitor monitor = ProgressMonitorOutput.create(output, index.getName(),
+ ProgressMonitor monitor = ProgressMonitorOutput.create(output, msg,
LoaderSequential.IndexTickPoint,
LoaderSequential.IndexSuperTick);
- monitor.startMessage();
+ monitor.startMessage(msg);
monitor.start();
- long time1 = timer.readTimer() ;
LoaderOps.copyIndex(primaryIndex.all(), new TupleIndex[]{index}, monitor) ;
- long time2 = timer.readTimer() ;
+
monitor.finish();
monitor.finishMessage(index.getName()+" indexing: ");
}
http://git-wip-us.apache.org/repos/asf/jena/blob/f5de3418/jena-db/jena-tdb2/src/main/java/org/apache/jena/tdb2/loader/sequential/LoaderSequential.java
----------------------------------------------------------------------
diff --git a/jena-db/jena-tdb2/src/main/java/org/apache/jena/tdb2/loader/sequential/LoaderSequential.java b/jena-db/jena-tdb2/src/main/java/org/apache/jena/tdb2/loader/sequential/LoaderSequential.java
index 32c2244..98d98dd 100644
--- a/jena-db/jena-tdb2/src/main/java/org/apache/jena/tdb2/loader/sequential/LoaderSequential.java
+++ b/jena-db/jena-tdb2/src/main/java/org/apache/jena/tdb2/loader/sequential/LoaderSequential.java
@@ -29,6 +29,8 @@ import org.apache.jena.tdb2.loader.BulkLoaderException;
import org.apache.jena.tdb2.loader.base.LoaderBase;
import org.apache.jena.tdb2.loader.base.LoaderOps;
import org.apache.jena.tdb2.loader.base.MonitorOutput;
+import org.apache.jena.tdb2.loader.base.ProgressMonitor;
+import org.apache.jena.tdb2.loader.base.ProgressMonitorOutput;
import org.apache.jena.tdb2.store.DatasetGraphTDB;
import org.apache.jena.tdb2.sys.TDBInternal;
@@ -103,10 +105,10 @@ public class LoaderSequential extends LoaderBase {
}
@Override
- protected void loadOne(String filename) {
- LoaderOps.inputFile(stream, filename, output, DataTickPoint, DataSuperTick);
+ protected ProgressMonitor createProgressMonitor(MonitorOutput output) {
+ return ProgressMonitorOutput.create(output, "<unset>", DataTickPoint, DataSuperTick);
}
-
+
@Override
public boolean bulkUseTransaction() {
return true;