You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@marmotta.apache.org by ja...@apache.org on 2014/03/17 12:05:07 UTC
[22/40] git commit: Fixed marmotta-loader to cleanup the statistics
db on shutdown (also moved the file to ${temp.dir}) - updated statistics to
use nio.Path
Fixed marmotta-loader to cleanup the statistics db on shutdown (also moved the file to ${temp.dir})
- updated statistics to use nio.Path
Project: http://git-wip-us.apache.org/repos/asf/marmotta/repo
Commit: http://git-wip-us.apache.org/repos/asf/marmotta/commit/df99bded
Tree: http://git-wip-us.apache.org/repos/asf/marmotta/tree/df99bded
Diff: http://git-wip-us.apache.org/repos/asf/marmotta/diff/df99bded
Branch: refs/heads/ldp
Commit: df99bdedb722cfe15b4afafec906ab29f12bd07d
Parents: 4b04fff
Author: Jakob Frank <ja...@apache.org>
Authored: Tue Feb 25 10:07:28 2014 +0100
Committer: Jakob Frank <ja...@apache.org>
Committed: Tue Feb 25 10:07:28 2014 +0100
----------------------------------------------------------------------
.../marmotta/loader/statistics/Statistics.java | 74 ++++++++++++--------
.../loader/core/test/LoaderTestBase.java | 7 +-
.../core/test/dummy/DummyLoaderBackend.java | 4 +-
.../core/test/dummy/DummyLoaderHandler.java | 17 ++++-
4 files changed, 67 insertions(+), 35 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/marmotta/blob/df99bded/loader/marmotta-loader-core/src/main/java/org/apache/marmotta/loader/statistics/Statistics.java
----------------------------------------------------------------------
diff --git a/loader/marmotta-loader-core/src/main/java/org/apache/marmotta/loader/statistics/Statistics.java b/loader/marmotta-loader-core/src/main/java/org/apache/marmotta/loader/statistics/Statistics.java
index c44fdf1..2535fd5 100644
--- a/loader/marmotta-loader-core/src/main/java/org/apache/marmotta/loader/statistics/Statistics.java
+++ b/loader/marmotta-loader-core/src/main/java/org/apache/marmotta/loader/statistics/Statistics.java
@@ -31,8 +31,9 @@ import org.slf4j.LoggerFactory;
import javax.imageio.ImageIO;
import java.awt.*;
import java.awt.image.BufferedImage;
-import java.io.File;
import java.io.IOException;
+import java.io.OutputStream;
+import java.nio.file.*;
import java.util.concurrent.Executors;
import java.util.concurrent.ScheduledExecutorService;
import java.util.concurrent.TimeUnit;
@@ -48,7 +49,7 @@ public class Statistics {
private StatisticsHandler handler;
-
+ protected Path statFile;
protected RrdDb statDB;
protected Sample statSample;
protected long statLastDump;
@@ -61,6 +62,7 @@ public class Statistics {
private long start, previous;
private Configuration configuration;
+ private DiagramUpdater diagramUpdater;
public Statistics(StatisticsHandler handler, Configuration configuration) {
this.handler = handler;
@@ -74,20 +76,23 @@ public class Statistics {
this.start = System.currentTimeMillis();
this.previous = System.currentTimeMillis();
- File statFile = new File("kiwiloader.rrd");
- if(statFile.exists()) {
- log.info("deleting old statistics database");
- statFile.delete();
- }
+ try {
+ statFile = Files.createTempFile("kiwiloader.", ".rrd");
+ Path gFile;
+ if (configuration.containsKey(LoaderOptions.STATISTICS_GRAPH)) {
+ gFile = Paths.get(configuration.getString(LoaderOptions.STATISTICS_GRAPH));
+ } else {
+ gFile = Files.createTempFile("marmotta-loader.", ".png");
+ }
- RrdDef stCfg = new RrdDef("kiwiloader.rrd");
- stCfg.setStep(SAMPLE_INTERVAL);
- stCfg.addDatasource("triples", DsType.COUNTER, 600, Double.NaN, Double.NaN);
- stCfg.addArchive(ConsolFun.AVERAGE, 0.5, 1, 1440); // every five seconds for 2 hours
- stCfg.addArchive(ConsolFun.AVERAGE, 0.5, 12, 1440); // every minute for 1 day
- stCfg.addArchive(ConsolFun.AVERAGE, 0.5, 60, 1440); // every five minutes for five days
- try {
+ RrdDef stCfg = new RrdDef(statFile.toString());
+ stCfg.setStep(SAMPLE_INTERVAL);
+ stCfg.addDatasource("triples", DsType.COUNTER, 600, Double.NaN, Double.NaN);
+ stCfg.addArchive(ConsolFun.AVERAGE, 0.5, 1, 1440); // every five seconds for 2 hours
+ stCfg.addArchive(ConsolFun.AVERAGE, 0.5, 12, 1440); // every minute for 1 day
+ stCfg.addArchive(ConsolFun.AVERAGE, 0.5, 60, 1440); // every five minutes for five days
+
statDB = new RrdDb(stCfg);
statSample = statDB.createSample();
statLastDump = System.currentTimeMillis();
@@ -97,7 +102,8 @@ public class Statistics {
statSampler.scheduleAtFixedRate(new StatisticsUpdater(),0, SAMPLE_INTERVAL, TimeUnit.SECONDS);
// create a statistics diagram every 5 minutes
- statSampler.scheduleAtFixedRate(new DiagramUpdater(),DIAGRAM_INTERVAL,DIAGRAM_INTERVAL,TimeUnit.SECONDS);
+ diagramUpdater = new DiagramUpdater(gFile);
+ statSampler.scheduleAtFixedRate(diagramUpdater,DIAGRAM_INTERVAL,DIAGRAM_INTERVAL,TimeUnit.SECONDS);
} catch (IOException e) {
log.warn("could not initialize statistics database: {}",e.getMessage());
}
@@ -105,8 +111,13 @@ public class Statistics {
}
public void stopSampling() {
- DiagramUpdater du = new DiagramUpdater();
- du.run();
+ if(statSampler != null) {
+ statSampler.shutdown();
+ }
+
+ if (diagramUpdater != null) {
+ diagramUpdater.run();
+ }
if(statDB != null) {
try {
@@ -115,8 +126,11 @@ public class Statistics {
log.warn("could not close statistics database...");
}
}
- if(statSampler != null) {
- statSampler.shutdown();
+
+ try {
+ Files.deleteIfExists(statFile);
+ } catch (IOException e) {
+ log.warn("could not cleanup statistics database: {}",e.getMessage());
}
}
@@ -169,15 +183,16 @@ public class Statistics {
private class DiagramUpdater implements Runnable {
+
+ private final Path gFile;
+
+ public DiagramUpdater(Path gFile) {
+ this.gFile = gFile;
+ }
+
@Override
public void run() {
try {
- File gFile = new File(configuration.getString(LoaderOptions.STATISTICS_GRAPH, File.createTempFile("marmotta-loader","png").getAbsolutePath()));
-
- if(gFile.exists()) {
- gFile.delete();
- }
-
// generate PNG diagram
RrdGraphDef gDef = new RrdGraphDef();
gDef.setFilename("-");
@@ -190,7 +205,7 @@ public class Statistics {
gDef.setAntiAliasing(true);
- gDef.datasource("triples", "kiwiloader.rrd", "triples", ConsolFun.AVERAGE);
+ gDef.datasource("triples", statFile.toString(), "triples", ConsolFun.AVERAGE);
gDef.line("triples", Color.BLUE, "Triples Written", 3F);
@@ -201,9 +216,12 @@ public class Statistics {
RrdGraph graph = new RrdGraph(gDef);
BufferedImage img = new BufferedImage(900,750, BufferedImage.TYPE_INT_RGB);
graph.render(img.getGraphics());
- ImageIO.write(img, "png", gFile);
- log.info("updated statistics diagram generated in {}", gFile.getAbsolutePath());
+ try (OutputStream stream = Files.newOutputStream(gFile, StandardOpenOption.TRUNCATE_EXISTING)) {
+ ImageIO.write(img, "png", stream);
+ }
+
+ log.info("updated statistics diagram generated in {}", gFile);
statLastDump = System.currentTimeMillis();
} catch (Exception ex) {
http://git-wip-us.apache.org/repos/asf/marmotta/blob/df99bded/loader/marmotta-loader-core/src/test/java/org/apache/marmotta/loader/core/test/LoaderTestBase.java
----------------------------------------------------------------------
diff --git a/loader/marmotta-loader-core/src/test/java/org/apache/marmotta/loader/core/test/LoaderTestBase.java b/loader/marmotta-loader-core/src/test/java/org/apache/marmotta/loader/core/test/LoaderTestBase.java
index c4dfc1b..2205b38 100644
--- a/loader/marmotta-loader-core/src/test/java/org/apache/marmotta/loader/core/test/LoaderTestBase.java
+++ b/loader/marmotta-loader-core/src/test/java/org/apache/marmotta/loader/core/test/LoaderTestBase.java
@@ -23,12 +23,10 @@ import org.apache.commons.io.FileUtils;
import org.apache.marmotta.loader.api.LoaderHandler;
import org.apache.marmotta.loader.api.LoaderOptions;
import org.apache.marmotta.loader.core.MarmottaLoader;
+import org.apache.marmotta.loader.core.test.dummy.DummyLoaderBackend;
import org.apache.marmotta.loader.core.test.dummy.DummyLoaderHandler;
import org.apache.marmotta.loader.wrapper.LoaderHandlerWrapper;
-import org.junit.AfterClass;
-import org.junit.Assert;
-import org.junit.BeforeClass;
-import org.junit.Test;
+import org.junit.*;
import org.openrdf.model.Model;
import org.openrdf.model.URI;
import org.openrdf.model.impl.URIImpl;
@@ -94,6 +92,7 @@ public abstract class LoaderTestBase {
cfg.setProperty(LoaderOptions.STATISTICS_ENABLED, true);
cfg.setProperty(LoaderOptions.STATISTICS_GRAPH, new File(tempDir.toFile(), "stats.png").toString());
+ cfg.setProperty(DummyLoaderBackend.METHOD_SLEEP_MILLIS, 10);
MarmottaLoader loader = new MarmottaLoader(cfg);
DummyLoaderHandler handler = getBase(loader.load());
http://git-wip-us.apache.org/repos/asf/marmotta/blob/df99bded/loader/marmotta-loader-core/src/test/java/org/apache/marmotta/loader/core/test/dummy/DummyLoaderBackend.java
----------------------------------------------------------------------
diff --git a/loader/marmotta-loader-core/src/test/java/org/apache/marmotta/loader/core/test/dummy/DummyLoaderBackend.java b/loader/marmotta-loader-core/src/test/java/org/apache/marmotta/loader/core/test/dummy/DummyLoaderBackend.java
index 6597e68..828ac6b 100644
--- a/loader/marmotta-loader-core/src/test/java/org/apache/marmotta/loader/core/test/dummy/DummyLoaderBackend.java
+++ b/loader/marmotta-loader-core/src/test/java/org/apache/marmotta/loader/core/test/dummy/DummyLoaderBackend.java
@@ -14,6 +14,8 @@ import java.util.Collection;
* @author Sebastian Schaffert (sschaffert@apache.org)
*/
public class DummyLoaderBackend implements LoaderBackend {
+ public static final String METHOD_SLEEP_MILLIS = "loader.dummy.method_sleep";
+
/**
* Return a unique identifier for the loader; used for identifying the loader to choose on the command line
* in case more than one loader implementation is available.
@@ -35,7 +37,7 @@ public class DummyLoaderBackend implements LoaderBackend {
*/
@Override
public LoaderHandler createLoader(Configuration configuration) {
- return new DummyLoaderHandler();
+ return new DummyLoaderHandler(configuration.getLong(METHOD_SLEEP_MILLIS, 0l));
}
/**
http://git-wip-us.apache.org/repos/asf/marmotta/blob/df99bded/loader/marmotta-loader-core/src/test/java/org/apache/marmotta/loader/core/test/dummy/DummyLoaderHandler.java
----------------------------------------------------------------------
diff --git a/loader/marmotta-loader-core/src/test/java/org/apache/marmotta/loader/core/test/dummy/DummyLoaderHandler.java b/loader/marmotta-loader-core/src/test/java/org/apache/marmotta/loader/core/test/dummy/DummyLoaderHandler.java
index 0c92baa..0869dad 100644
--- a/loader/marmotta-loader-core/src/test/java/org/apache/marmotta/loader/core/test/dummy/DummyLoaderHandler.java
+++ b/loader/marmotta-loader-core/src/test/java/org/apache/marmotta/loader/core/test/dummy/DummyLoaderHandler.java
@@ -14,10 +14,16 @@ import org.openrdf.rio.helpers.RDFHandlerBase;
*/
public class DummyLoaderHandler extends RDFHandlerBase implements LoaderHandler {
- private Model model;
+ private final long sleep;
+ private final Model model;
public DummyLoaderHandler() {
+ this(0);
+ }
+
+ public DummyLoaderHandler(long methodSleep) {
model = new TreeModel();
+ sleep = methodSleep;
}
public Model getModel() {
@@ -36,7 +42,7 @@ public class DummyLoaderHandler extends RDFHandlerBase implements LoaderHandler
}
/**
- * Peform cleanup on shutdown, e.g. re-creating indexes after import completed or freeing resources acquired by
+ * Perform cleanup on shutdown, e.g. re-creating indexes after import completed or freeing resources acquired by
* the handler.
*/
@Override
@@ -47,5 +53,12 @@ public class DummyLoaderHandler extends RDFHandlerBase implements LoaderHandler
@Override
public void handleStatement(Statement st) throws RDFHandlerException {
model.add(st);
+ if (sleep > 0) {
+ try {
+ Thread.sleep(sleep);
+ } catch (InterruptedException e) {
+ // ignored
+ }
+ }
}
}