You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@baremaps.apache.org by bc...@apache.org on 2023/06/23 12:21:35 UTC
[incubator-baremaps] branch 688-export updated (1f21d007 -> a4a06203)
This is an automated email from the ASF dual-hosted git repository.
bchapuis pushed a change to branch 688-export
in repository https://gitbox.apache.org/repos/asf/incubator-baremaps.git
omit 1f21d007 Revert TileStore change
omit e14afe61 Fix minor issues
omit 450804d1 Refactor utility classes
omit ed7933a4 Fix CodeQL and Sonar warnings
omit 24985557 Add mbtiles command
omit a6da66f2 Format code
omit c79a3461 Add batch insert in mbtiles
omit dbd39e20 Add MBTiles benchmark and refactor TileStore
omit 0106bb16 Increase sqlite page size
omit 09734a95 Fix null pointer exception related to bounds
add 7fcd47d3 Fix null pointer exception (#707)
add 89c1c6df Add new cli command to edit styles locally (#704)
add 0912b6ce Correct typo in configuration (#708)
add 8c093e2b Fix: Uncomment ocean tileset (#710)
new 00c7b6fa Fix null pointer exception related to bounds
new 135d6d38 Increase sqlite page size
new fa23fc43 Add MBTiles benchmark and refactor TileStore
new 69aff8bb Add batch insert in mbtiles
new 03d84577 Format code
new b297ed1d Add mbtiles command
new 1cb66e4e Refactor utility classes
new bc439709 Change the TileStore interface
new a4a06203 Fix formatting
This update added new revisions after undoing existing revisions.
That is to say, some revisions that were in the old version of the
branch are not in the new version. This situation occurs
when a user --force pushes a change and generates a repository
containing something like this:
* -- * -- B -- O -- O -- O (1f21d007)
\
N -- N -- N refs/heads/688-export (a4a06203)
You should already have received notification emails for all of the O
revisions, and so the following emails describe only the N revisions
from the common base, B.
Any revisions marked "omit" are not gone; other references still
refer to them. Any revisions marked "discard" are gone forever.
The 9 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
baremaps-cli/pom.xml | 4 +-
.../main/java/org/apache/baremaps/cli/map/Map.java | 3 +-
.../cli/map/{Dev.java => StyleCommand.java} | 56 +++++-----------------
.../org/apache/baremaps/tilestore/TileCache.java | 16 +++----
.../org/apache/baremaps/server/ChangeResource.java | 5 +-
basemap/tileset.js | 2 +-
6 files changed, 29 insertions(+), 57 deletions(-)
copy baremaps-cli/src/main/java/org/apache/baremaps/cli/map/{Dev.java => StyleCommand.java} (62%)
[incubator-baremaps] 01/09: Fix null pointer exception related to bounds
Posted by bc...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
bchapuis pushed a commit to branch 688-export
in repository https://gitbox.apache.org/repos/asf/incubator-baremaps.git
commit 00c7b6fa41b232b32972419d561b1876fda50b73
Author: Bertil Chapuis <bc...@gmail.com>
AuthorDate: Fri Jun 9 21:14:58 2023 +0200
Fix null pointer exception related to bounds
---
.../apache/baremaps/tilestore/mbtiles/MBTiles.java | 5 +++--
.../baremaps/workflow/tasks/ExportVectorTiles.java | 25 ++++++++++++++++------
2 files changed, 21 insertions(+), 9 deletions(-)
diff --git a/baremaps-core/src/main/java/org/apache/baremaps/tilestore/mbtiles/MBTiles.java b/baremaps-core/src/main/java/org/apache/baremaps/tilestore/mbtiles/MBTiles.java
index 9cfbdad7..06d0e0cb 100644
--- a/baremaps-core/src/main/java/org/apache/baremaps/tilestore/mbtiles/MBTiles.java
+++ b/baremaps-core/src/main/java/org/apache/baremaps/tilestore/mbtiles/MBTiles.java
@@ -23,6 +23,7 @@ import java.sql.SQLException;
import java.sql.Statement;
import java.util.HashMap;
import java.util.Map;
+import javax.sql.DataSource;
import org.apache.baremaps.tilestore.TileCoord;
import org.apache.baremaps.tilestore.TileStore;
import org.apache.baremaps.tilestore.TileStoreException;
@@ -58,14 +59,14 @@ public class MBTiles implements TileStore {
private static final String DELETE_METADATA = "DELETE FROM metadata";
- private final org.sqlite.SQLiteDataSource dataSource;
+ private final DataSource dataSource;
/**
* Constructs an {@code MBTiles} with the provided SQLite datasource.
*
* @param dataSource the SQLite datasource
*/
- public MBTiles(org.sqlite.SQLiteDataSource dataSource) {
+ public MBTiles(DataSource dataSource) {
this.dataSource = dataSource;
}
diff --git a/baremaps-core/src/main/java/org/apache/baremaps/workflow/tasks/ExportVectorTiles.java b/baremaps-core/src/main/java/org/apache/baremaps/workflow/tasks/ExportVectorTiles.java
index 7828a049..32246c2d 100644
--- a/baremaps-core/src/main/java/org/apache/baremaps/workflow/tasks/ExportVectorTiles.java
+++ b/baremaps-core/src/main/java/org/apache/baremaps/workflow/tasks/ExportVectorTiles.java
@@ -16,7 +16,10 @@ import static org.apache.baremaps.utils.ObjectMapperUtils.objectMapper;
import com.fasterxml.jackson.core.JsonProcessingException;
import com.fasterxml.jackson.databind.ObjectMapper;
+import com.zaxxer.hikari.HikariConfig;
+import com.zaxxer.hikari.HikariDataSource;
import java.io.IOException;
+import java.nio.file.Files;
import java.nio.file.Path;
import java.util.HashMap;
import java.util.Map;
@@ -61,10 +64,11 @@ public record ExportVectorTiles(
var sourceTileStore = sourceTileStore(tileset, datasource);
var targetTileStore = targetTileStore(tileset);
- var envelope =
- new Envelope(
+ var envelope = tileset.getBounds().size() == 4
+ ? new Envelope(
tileset.getBounds().get(0), tileset.getBounds().get(2),
- tileset.getBounds().get(1), tileset.getBounds().get(3));
+ tileset.getBounds().get(1), tileset.getBounds().get(3))
+ : new Envelope(-180, 180, -85.0511, 85.0511);
var count = TileCoord.count(envelope, tileset.getMinzoom(), tileset.getMaxzoom());
@@ -81,6 +85,8 @@ public record ExportVectorTiles(
private TileStore targetTileStore(Tileset source) throws TileStoreException, IOException {
if (mbtiles) {
+ Files.deleteIfExists(repository);
+
var sqliteConfig = new SQLiteConfig();
sqliteConfig.setCacheSize(1000000);
sqliteConfig.setJournalMode(JournalMode.OFF);
@@ -88,11 +94,16 @@ public record ExportVectorTiles(
sqliteConfig.setSynchronous(SynchronousMode.OFF);
sqliteConfig.setTempStore(TempStore.MEMORY);
- var dataSource = new SQLiteDataSource();
- dataSource.setConfig(sqliteConfig);
- dataSource.setUrl("jdbc:sqlite:" + repository);
+ var sqliteDataSource = new SQLiteDataSource();
+ sqliteDataSource.setConfig(sqliteConfig);
+ sqliteDataSource.setUrl("jdbc:sqlite:" + repository);
+
+ var hikariConfig = new HikariConfig();
+ hikariConfig.setDataSource(sqliteDataSource);
+ hikariConfig.setMaximumPoolSize(1);
+ var hikariDataSource = new HikariDataSource(hikariConfig);
- var tilesStore = new MBTiles(dataSource);
+ var tilesStore = new MBTiles(hikariDataSource);
tilesStore.initializeDatabase();
tilesStore.writeMetadata(metadata(source));
[incubator-baremaps] 02/09: Increase sqlite page size
Posted by bc...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
bchapuis pushed a commit to branch 688-export
in repository https://gitbox.apache.org/repos/asf/incubator-baremaps.git
commit 135d6d38306f32fcb3f83cbd8de42f94de984939
Author: Bertil Chapuis <bc...@gmail.com>
AuthorDate: Mon Jun 12 21:46:00 2023 +0200
Increase sqlite page size
---
.run/openstreetmap-export.run.xml | 2 +-
.../main/java/org/apache/baremaps/workflow/tasks/ExportVectorTiles.java | 1 +
2 files changed, 2 insertions(+), 1 deletion(-)
diff --git a/.run/openstreetmap-export.run.xml b/.run/openstreetmap-export.run.xml
index cb53b2d4..b067f8a0 100644
--- a/.run/openstreetmap-export.run.xml
+++ b/.run/openstreetmap-export.run.xml
@@ -2,7 +2,7 @@
<configuration default="false" name="openstreetmap-export" type="Application" factoryName="Application">
<option name="MAIN_CLASS_NAME" value="org.apache.baremaps.cli.Baremaps" />
<module name="baremaps-cli" />
- <option name="PROGRAM_PARAMETERS" value="map export --tileset tileset.json --repository tiles/" />
+ <option name="PROGRAM_PARAMETERS" value="map export --tileset tileset.json --repository tiles.mbtiles --mbtiles" />
<option name="WORKING_DIRECTORY" value="$PROJECT_DIR$/examples/openstreetmap" />
<extension name="coverage">
<pattern>
diff --git a/baremaps-core/src/main/java/org/apache/baremaps/workflow/tasks/ExportVectorTiles.java b/baremaps-core/src/main/java/org/apache/baremaps/workflow/tasks/ExportVectorTiles.java
index 32246c2d..13e49f48 100644
--- a/baremaps-core/src/main/java/org/apache/baremaps/workflow/tasks/ExportVectorTiles.java
+++ b/baremaps-core/src/main/java/org/apache/baremaps/workflow/tasks/ExportVectorTiles.java
@@ -89,6 +89,7 @@ public record ExportVectorTiles(
var sqliteConfig = new SQLiteConfig();
sqliteConfig.setCacheSize(1000000);
+ sqliteConfig.setPageSize(65536);
sqliteConfig.setJournalMode(JournalMode.OFF);
sqliteConfig.setLockingMode(LockingMode.EXCLUSIVE);
sqliteConfig.setSynchronous(SynchronousMode.OFF);
[incubator-baremaps] 07/09: Refactor utility classes
Posted by bc...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
bchapuis pushed a commit to branch 688-export
in repository https://gitbox.apache.org/repos/asf/incubator-baremaps.git
commit 1cb66e4e807ae6186f716960b02b1c6aca2b507c
Author: Bertil Chapuis <bc...@gmail.com>
AuthorDate: Sun Jun 18 23:24:00 2023 +0200
Refactor utility classes
---
.../baremaps/benchmarks/MBTilesBenchmark.java | 16 ++++-----
.../main/java/org/apache/baremaps/cli/map/Dev.java | 5 +--
.../java/org/apache/baremaps/cli/map/MBTiles.java | 17 ++++-----
.../java/org/apache/baremaps/cli/map/Serve.java | 6 ++--
.../org/apache/baremaps/cli/ogcapi/OgcApi.java | 4 +--
.../mbtiles/{MBTiles.java => MBTilesStore.java} | 4 +--
.../{postgres => utils}/PostgresUtils.java | 15 ++++----
.../org/apache/baremaps/utils/SqliteUtils.java | 39 +++++++++++++++++++++
.../apache/baremaps/workflow/WorkflowContext.java | 4 +--
.../baremaps/workflow/tasks/ExportVectorTiles.java | 40 +++-------------------
.../postgres/PostgresRepositoryTest.java | 2 +-
.../repository/BlockImporterTest.java | 4 +--
.../postgres/metadata/PostgresMetadataTest.java | 2 +-
.../{MBTilesTest.java => MBTilesStoreTest.java} | 8 ++---
.../org/apache/baremaps/ogcapi/OgcApiTest.java | 4 +--
15 files changed, 91 insertions(+), 79 deletions(-)
diff --git a/baremaps-benchmark/src/main/java/org/apache/baremaps/benchmarks/MBTilesBenchmark.java b/baremaps-benchmark/src/main/java/org/apache/baremaps/benchmarks/MBTilesBenchmark.java
index dc333999..3754ae6f 100644
--- a/baremaps-benchmark/src/main/java/org/apache/baremaps/benchmarks/MBTilesBenchmark.java
+++ b/baremaps-benchmark/src/main/java/org/apache/baremaps/benchmarks/MBTilesBenchmark.java
@@ -22,8 +22,8 @@ import java.util.ArrayList;
import java.util.concurrent.TimeUnit;
import org.apache.baremaps.tilestore.TileCoord;
import org.apache.baremaps.tilestore.TileStoreException;
-import org.apache.baremaps.tilestore.mbtiles.MBTiles;
-import org.apache.baremaps.workflow.tasks.ExportVectorTiles;
+import org.apache.baremaps.tilestore.mbtiles.MBTilesStore;
+import org.apache.baremaps.utils.SqliteUtils;
import org.openjdk.jmh.annotations.*;
import org.openjdk.jmh.runner.Runner;
import org.openjdk.jmh.runner.RunnerException;
@@ -42,13 +42,13 @@ public class MBTilesBenchmark {
private Path file;
- private MBTiles mbTiles;
+ private MBTilesStore mbTilesStore;
@Setup
public void setup() throws IOException, TileStoreException {
file = Files.createTempFile(Paths.get("."), "baremaps", ".mbtiles");
- mbTiles = new MBTiles(ExportVectorTiles.createDataSource(file));
- mbTiles.initializeDatabase();
+ mbTilesStore = new MBTilesStore(SqliteUtils.createDataSource(file, false));
+ mbTilesStore.initializeDatabase();
}
@TearDown
@@ -62,7 +62,7 @@ public class MBTilesBenchmark {
for (int i = 0; i < benchmark.iterations; i++) {
var bytes = new byte[1 << 16];
random.nextBytes(bytes);
- mbTiles.put(new TileCoord(0, 0, i), ByteBuffer.wrap(bytes));
+ mbTilesStore.put(new TileCoord(0, 0, i), ByteBuffer.wrap(bytes));
}
}
@@ -78,12 +78,12 @@ public class MBTilesBenchmark {
buffers.add(ByteBuffer.wrap(bytes));
if (coords.size() == 100) {
random.nextBytes(bytes);
- mbTiles.put(coords, buffers);
+ mbTilesStore.put(coords, buffers);
coords.clear();
buffers.clear();
}
}
- mbTiles.put(coords, buffers);
+ mbTilesStore.put(coords, buffers);
coords.clear();
buffers.clear();
}
diff --git a/baremaps-cli/src/main/java/org/apache/baremaps/cli/map/Dev.java b/baremaps-cli/src/main/java/org/apache/baremaps/cli/map/Dev.java
index b0daab3c..c77b1444 100644
--- a/baremaps-cli/src/main/java/org/apache/baremaps/cli/map/Dev.java
+++ b/baremaps-cli/src/main/java/org/apache/baremaps/cli/map/Dev.java
@@ -23,12 +23,13 @@ import java.util.concurrent.Callable;
import java.util.function.Supplier;
import org.apache.baremaps.cli.Options;
import org.apache.baremaps.config.ConfigReader;
-import org.apache.baremaps.postgres.PostgresUtils;
import org.apache.baremaps.server.*;
import org.apache.baremaps.tilestore.TileStore;
import org.apache.baremaps.tilestore.postgres.PostgresTileStore;
import org.apache.baremaps.vectortile.style.Style;
import org.apache.baremaps.vectortile.tilejson.TileJSON;
+import org.apache.baremaps.server.CorsFilter;
+import org.apache.baremaps.utils.PostgresUtils;
import org.apache.baremaps.vectortile.tileset.Tileset;
import org.glassfish.hk2.api.TypeLiteral;
import org.glassfish.hk2.utilities.binding.AbstractBinder;
@@ -69,7 +70,7 @@ public class Dev implements Callable<Integer> {
var configReader = new ConfigReader();
var objectMapper = objectMapper();
var tileset = objectMapper.readValue(configReader.read(this.tilesetPath), Tileset.class);
- var datasource = PostgresUtils.dataSource(tileset.getDatabase());
+ var datasource = PostgresUtils.createDataSource(tileset.getDatabase());
var tileStoreType = new TypeLiteral<Supplier<TileStore>>() {};
var tileStoreSupplier = (Supplier<TileStore>) () -> {
diff --git a/baremaps-cli/src/main/java/org/apache/baremaps/cli/map/MBTiles.java b/baremaps-cli/src/main/java/org/apache/baremaps/cli/map/MBTiles.java
index 0f8c7f50..67924764 100644
--- a/baremaps-cli/src/main/java/org/apache/baremaps/cli/map/MBTiles.java
+++ b/baremaps-cli/src/main/java/org/apache/baremaps/cli/map/MBTiles.java
@@ -26,7 +26,8 @@ import org.apache.baremaps.server.CorsFilter;
import org.apache.baremaps.server.ServerResources;
import org.apache.baremaps.tilestore.TileCache;
import org.apache.baremaps.tilestore.TileStore;
-import org.apache.baremaps.workflow.tasks.ExportVectorTiles;
+import org.apache.baremaps.tilestore.mbtiles.MBTilesStore;
+import org.apache.baremaps.utils.SqliteUtils;
import org.glassfish.hk2.utilities.binding.AbstractBinder;
import org.glassfish.jersey.server.ResourceConfig;
import org.slf4j.Logger;
@@ -48,15 +49,15 @@ public class MBTiles implements Callable<Integer> {
@Option(names = {"--mbtiles"}, paramLabel = "MBTILES", description = "The mbtiles file.",
required = true)
- private Path mbtiles;
+ private Path mbtilesPath;
@Option(names = {"--tilejson"}, paramLabel = "TILEJSON", description = "The tileJSON file.",
required = true)
- private Path tileset;
+ private Path tileJSONPath;
@Option(names = {"--style"}, paramLabel = "STYLE", description = "The style file.",
required = true)
- private Path style;
+ private Path stylePath;
@Option(names = {"--port"}, paramLabel = "PORT", description = "The port of the server.")
private int port = 9000;
@@ -65,9 +66,9 @@ public class MBTiles implements Callable<Integer> {
public Integer call() throws Exception {
var objectMapper = objectMapper();
var caffeineSpec = CaffeineSpec.parse(cache);
- var datasource = ExportVectorTiles.createDataSource(mbtiles);
+ var datasource = SqliteUtils.createDataSource(mbtilesPath, true);
- var tileStore = new org.apache.baremaps.tilestore.mbtiles.MBTiles(datasource);
+ var tileStore = new MBTilesStore(datasource);
var tileCache = new TileCache(tileStore, caffeineSpec);
// Configure the application
@@ -76,8 +77,8 @@ public class MBTiles implements Callable<Integer> {
.register(newContextResolver(objectMapper)).register(new AbstractBinder() {
@Override
protected void configure() {
- bind(tileset).to(Path.class).named("tileset");
- bind(style).to(Path.class).named("style");
+ bind(tileJSONPath).to(Path.class).named("tileset");
+ bind(stylePath).to(Path.class).named("style");
bind(tileCache).to(TileStore.class);
bind(objectMapper).to(ObjectMapper.class);
}
diff --git a/baremaps-cli/src/main/java/org/apache/baremaps/cli/map/Serve.java b/baremaps-cli/src/main/java/org/apache/baremaps/cli/map/Serve.java
index e0002b03..5c1fa5ad 100644
--- a/baremaps-cli/src/main/java/org/apache/baremaps/cli/map/Serve.java
+++ b/baremaps-cli/src/main/java/org/apache/baremaps/cli/map/Serve.java
@@ -23,13 +23,15 @@ import java.util.concurrent.Callable;
import java.util.function.Supplier;
import org.apache.baremaps.cli.Options;
import org.apache.baremaps.config.ConfigReader;
-import org.apache.baremaps.postgres.PostgresUtils;
+
import org.apache.baremaps.server.*;
import org.apache.baremaps.tilestore.TileCache;
import org.apache.baremaps.tilestore.TileStore;
import org.apache.baremaps.tilestore.postgres.PostgresTileStore;
import org.apache.baremaps.vectortile.style.Style;
import org.apache.baremaps.vectortile.tilejson.TileJSON;
+import org.apache.baremaps.server.CorsFilter;
+import org.apache.baremaps.utils.PostgresUtils;
import org.apache.baremaps.vectortile.tileset.Tileset;
import org.glassfish.hk2.api.TypeLiteral;
import org.glassfish.hk2.utilities.binding.AbstractBinder;
@@ -71,7 +73,7 @@ public class Serve implements Callable<Integer> {
var configReader = new ConfigReader();
var tileset = objectMapper.readValue(configReader.read(this.tilesetPath), Tileset.class);
var caffeineSpec = CaffeineSpec.parse(cache);
- var datasource = PostgresUtils.dataSource(tileset.getDatabase());
+ var datasource = PostgresUtils.createDataSource(tileset.getDatabase());
var tileStoreSupplierType = new TypeLiteral<Supplier<TileStore>>() {};
var tileStore = new PostgresTileStore(datasource, tileset);
diff --git a/baremaps-cli/src/main/java/org/apache/baremaps/cli/ogcapi/OgcApi.java b/baremaps-cli/src/main/java/org/apache/baremaps/cli/ogcapi/OgcApi.java
index ea1a7e24..e596edc4 100644
--- a/baremaps-cli/src/main/java/org/apache/baremaps/cli/ogcapi/OgcApi.java
+++ b/baremaps-cli/src/main/java/org/apache/baremaps/cli/ogcapi/OgcApi.java
@@ -27,11 +27,11 @@ import javax.sql.DataSource;
import org.apache.baremaps.cli.Options;
import org.apache.baremaps.config.ConfigReader;
import org.apache.baremaps.ogcapi.*;
-import org.apache.baremaps.postgres.PostgresUtils;
import org.apache.baremaps.server.CorsFilter;
import org.apache.baremaps.tilestore.TileCache;
import org.apache.baremaps.tilestore.TileStore;
import org.apache.baremaps.tilestore.postgres.PostgresTileStore;
+import org.apache.baremaps.utils.PostgresUtils;
import org.apache.baremaps.vectortile.tileset.Tileset;
import org.glassfish.hk2.utilities.binding.AbstractBinder;
import org.glassfish.jersey.media.multipart.MultiPartFeature;
@@ -78,7 +78,7 @@ public class OgcApi implements Callable<Integer> {
var configReader = new ConfigReader();
var config = objectMapper.readValue(configReader.read(this.tileset), Tileset.class);
var caffeineSpec = CaffeineSpec.parse(cache);
- var dataSource = PostgresUtils.dataSource(database);
+ var dataSource = PostgresUtils.createDataSource(database);
var tileStore = new PostgresTileStore(dataSource, config);
var tileCache = new TileCache(tileStore, caffeineSpec);
diff --git a/baremaps-core/src/main/java/org/apache/baremaps/tilestore/mbtiles/MBTiles.java b/baremaps-core/src/main/java/org/apache/baremaps/tilestore/mbtiles/MBTilesStore.java
similarity index 98%
rename from baremaps-core/src/main/java/org/apache/baremaps/tilestore/mbtiles/MBTiles.java
rename to baremaps-core/src/main/java/org/apache/baremaps/tilestore/mbtiles/MBTilesStore.java
index 0344aed5..9539d9a5 100644
--- a/baremaps-core/src/main/java/org/apache/baremaps/tilestore/mbtiles/MBTiles.java
+++ b/baremaps-core/src/main/java/org/apache/baremaps/tilestore/mbtiles/MBTilesStore.java
@@ -34,7 +34,7 @@ import org.apache.baremaps.tilestore.TileStoreException;
* <a href="https://docs.mapbox.com/help/glossary/mbtiles/">MBTiles</a> file format for storing
* tiles.
*/
-public class MBTiles implements TileStore {
+public class MBTilesStore implements TileStore {
private static final String CREATE_TABLE_METADATA =
"CREATE TABLE IF NOT EXISTS metadata (name TEXT, value TEXT, PRIMARY KEY (name))";
@@ -67,7 +67,7 @@ public class MBTiles implements TileStore {
*
* @param dataSource the SQLite datasource
*/
- public MBTiles(DataSource dataSource) {
+ public MBTilesStore(DataSource dataSource) {
this.dataSource = dataSource;
}
diff --git a/baremaps-core/src/main/java/org/apache/baremaps/postgres/PostgresUtils.java b/baremaps-core/src/main/java/org/apache/baremaps/utils/PostgresUtils.java
similarity index 81%
rename from baremaps-core/src/main/java/org/apache/baremaps/postgres/PostgresUtils.java
rename to baremaps-core/src/main/java/org/apache/baremaps/utils/PostgresUtils.java
index 70f0667a..ab5cf6e3 100644
--- a/baremaps-core/src/main/java/org/apache/baremaps/postgres/PostgresUtils.java
+++ b/baremaps-core/src/main/java/org/apache/baremaps/utils/PostgresUtils.java
@@ -10,7 +10,7 @@
* the License.
*/
-package org.apache.baremaps.postgres;
+package org.apache.baremaps.utils;
@@ -29,10 +29,11 @@ public final class PostgresUtils {
private PostgresUtils() {}
- public static HikariDataSource dataSource(String host, Integer port, String database,
+ public static HikariDataSource createDataSource(String host, Integer port, String database,
String username, String password) {
- return dataSource(String.format("jdbc:postgresql://%s:%s/%s?&user=%s&password=%s", host, port,
- database, username, password));
+ return createDataSource(
+ String.format("jdbc:postgresql://%s:%s/%s?&user=%s&password=%s", host, port,
+ database, username, password));
}
/**
@@ -42,8 +43,8 @@ public final class PostgresUtils {
* @param url the JDBC url
* @return the data source
*/
- public static HikariDataSource dataSource(String url) {
- return dataSource(url, Runtime.getRuntime().availableProcessors());
+ public static HikariDataSource createDataSource(String url) {
+ return createDataSource(url, Runtime.getRuntime().availableProcessors());
}
/**
@@ -53,7 +54,7 @@ public final class PostgresUtils {
* @param poolSize the pool size
* @return the data source
*/
- public static HikariDataSource dataSource(String url, int poolSize) {
+ public static HikariDataSource createDataSource(String url, int poolSize) {
if (poolSize < 1) {
throw new IllegalArgumentException("PoolSize cannot be inferior to 1");
}
diff --git a/baremaps-core/src/main/java/org/apache/baremaps/utils/SqliteUtils.java b/baremaps-core/src/main/java/org/apache/baremaps/utils/SqliteUtils.java
index 4b937954..1ca6baaa 100644
--- a/baremaps-core/src/main/java/org/apache/baremaps/utils/SqliteUtils.java
+++ b/baremaps-core/src/main/java/org/apache/baremaps/utils/SqliteUtils.java
@@ -15,17 +15,56 @@ package org.apache.baremaps.utils;
import com.google.common.io.Resources;
+import com.zaxxer.hikari.HikariConfig;
+import com.zaxxer.hikari.HikariDataSource;
import java.io.IOException;
import java.net.URL;
import java.nio.charset.StandardCharsets;
+import java.nio.file.Path;
import java.sql.Connection;
import java.sql.DriverManager;
import java.sql.SQLException;
import java.sql.Statement;
+import org.sqlite.SQLiteConfig;
+import org.sqlite.SQLiteConfig.JournalMode;
+import org.sqlite.SQLiteConfig.LockingMode;
+import org.sqlite.SQLiteConfig.SynchronousMode;
+import org.sqlite.SQLiteConfig.TempStore;
+import org.sqlite.SQLiteDataSource;
/** A helper class for creating executing sql scripts onto a SQLite database */
public final class SqliteUtils {
+ private SqliteUtils() {}
+
+ /**
+ * Create a SQLite data source.
+ *
+ * @param path the path to the SQLite database
+ * @param readOnly
+ * @return the SQLite data source
+ */
+ public static HikariDataSource createDataSource(Path path, boolean readOnly) {
+ var sqliteConfig = new SQLiteConfig();
+ sqliteConfig.setCacheSize(1000000);
+ sqliteConfig.setPageSize(65536);
+ sqliteConfig.setJournalMode(JournalMode.OFF);
+ sqliteConfig.setLockingMode(LockingMode.EXCLUSIVE);
+ sqliteConfig.setSynchronous(SynchronousMode.OFF);
+ sqliteConfig.setTempStore(TempStore.MEMORY);
+ sqliteConfig.setReadOnly(readOnly);
+
+ var sqliteDataSource = new SQLiteDataSource();
+ sqliteDataSource.setConfig(sqliteConfig);
+ sqliteDataSource.setUrl("jdbc:sqlite:" + path);
+
+ var hikariConfig = new HikariConfig();
+ hikariConfig.setDataSource(sqliteDataSource);
+ hikariConfig.setMaximumPoolSize(1);
+
+ return new HikariDataSource(hikariConfig);
+ }
+
/**
* Executes the queries contained in a resource file.
*
diff --git a/baremaps-core/src/main/java/org/apache/baremaps/workflow/WorkflowContext.java b/baremaps-core/src/main/java/org/apache/baremaps/workflow/WorkflowContext.java
index 1cc7fab6..eff85f26 100644
--- a/baremaps-core/src/main/java/org/apache/baremaps/workflow/WorkflowContext.java
+++ b/baremaps-core/src/main/java/org/apache/baremaps/workflow/WorkflowContext.java
@@ -17,7 +17,7 @@ package org.apache.baremaps.workflow;
import java.util.Map;
import java.util.concurrent.ConcurrentHashMap;
import javax.sql.DataSource;
-import org.apache.baremaps.postgres.PostgresUtils;
+import org.apache.baremaps.utils.PostgresUtils;
/**
* A context that is passed to the tasks of a workflow and used to share data between tasks.
@@ -33,7 +33,7 @@ public class WorkflowContext {
* @return the data source
*/
public DataSource getDataSource(String database) {
- return dataSources.computeIfAbsent(database, d -> PostgresUtils.dataSource(d));
+ return dataSources.computeIfAbsent(database, d -> PostgresUtils.createDataSource(d));
}
}
diff --git a/baremaps-core/src/main/java/org/apache/baremaps/workflow/tasks/ExportVectorTiles.java b/baremaps-core/src/main/java/org/apache/baremaps/workflow/tasks/ExportVectorTiles.java
index 43040de8..b33377e5 100644
--- a/baremaps-core/src/main/java/org/apache/baremaps/workflow/tasks/ExportVectorTiles.java
+++ b/baremaps-core/src/main/java/org/apache/baremaps/workflow/tasks/ExportVectorTiles.java
@@ -16,8 +16,6 @@ import static org.apache.baremaps.utils.ObjectMapperUtils.objectMapper;
import com.fasterxml.jackson.core.JsonProcessingException;
import com.fasterxml.jackson.databind.ObjectMapper;
-import com.zaxxer.hikari.HikariConfig;
-import com.zaxxer.hikari.HikariDataSource;
import java.io.IOException;
import java.nio.file.Files;
import java.nio.file.Path;
@@ -30,8 +28,9 @@ import org.apache.baremaps.stream.ProgressLogger;
import org.apache.baremaps.stream.StreamUtils;
import org.apache.baremaps.tilestore.*;
import org.apache.baremaps.tilestore.file.FileTileStore;
-import org.apache.baremaps.tilestore.mbtiles.MBTiles;
+import org.apache.baremaps.tilestore.mbtiles.MBTilesStore;
import org.apache.baremaps.tilestore.postgres.PostgresTileStore;
+import org.apache.baremaps.utils.SqliteUtils;
import org.apache.baremaps.vectortile.tileset.Tileset;
import org.apache.baremaps.vectortile.tileset.TilesetQuery;
import org.apache.baremaps.workflow.Task;
@@ -39,12 +38,6 @@ import org.apache.baremaps.workflow.WorkflowContext;
import org.locationtech.jts.geom.Envelope;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import org.sqlite.SQLiteConfig;
-import org.sqlite.SQLiteConfig.JournalMode;
-import org.sqlite.SQLiteConfig.LockingMode;
-import org.sqlite.SQLiteConfig.SynchronousMode;
-import org.sqlite.SQLiteConfig.TempStore;
-import org.sqlite.SQLiteDataSource;
public record ExportVectorTiles(
Path tileset,
@@ -86,8 +79,8 @@ public record ExportVectorTiles(
private TileStore targetTileStore(Tileset source) throws TileStoreException, IOException {
if (mbtiles) {
Files.deleteIfExists(repository);
- var dataSource = createDataSource(repository);
- var tilesStore = new MBTiles(dataSource);
+ var dataSource = SqliteUtils.createDataSource(repository, false);
+ var tilesStore = new MBTilesStore(dataSource);
tilesStore.initializeDatabase();
tilesStore.writeMetadata(metadata(source));
return tilesStore;
@@ -138,29 +131,4 @@ public record ExportVectorTiles(
return metadata;
}
- /**
- * Create a SQLite data source.
- *
- * @param path the path to the SQLite database
- * @return the SQLite data source
- */
- public static DataSource createDataSource(Path path) {
- var sqliteConfig = new SQLiteConfig();
- sqliteConfig.setCacheSize(1000000);
- sqliteConfig.setPageSize(65536);
- sqliteConfig.setJournalMode(JournalMode.OFF);
- sqliteConfig.setLockingMode(LockingMode.EXCLUSIVE);
- sqliteConfig.setSynchronous(SynchronousMode.OFF);
- sqliteConfig.setTempStore(TempStore.MEMORY);
-
- var sqliteDataSource = new SQLiteDataSource();
- sqliteDataSource.setConfig(sqliteConfig);
- sqliteDataSource.setUrl("jdbc:sqlite:" + path);
-
- var hikariConfig = new HikariConfig();
- hikariConfig.setDataSource(sqliteDataSource);
- hikariConfig.setMaximumPoolSize(1);
-
- return new HikariDataSource(hikariConfig);
- }
}
diff --git a/baremaps-core/src/test/java/org/apache/baremaps/openstreetmap/postgres/PostgresRepositoryTest.java b/baremaps-core/src/test/java/org/apache/baremaps/openstreetmap/postgres/PostgresRepositoryTest.java
index 00a1f888..9da06fdf 100644
--- a/baremaps-core/src/test/java/org/apache/baremaps/openstreetmap/postgres/PostgresRepositoryTest.java
+++ b/baremaps-core/src/test/java/org/apache/baremaps/openstreetmap/postgres/PostgresRepositoryTest.java
@@ -18,8 +18,8 @@ import java.io.IOException;
import java.sql.Connection;
import java.sql.SQLException;
import javax.sql.DataSource;
-import org.apache.baremaps.postgres.PostgresUtils;
import org.apache.baremaps.testing.PostgresContainerTest;
+import org.apache.baremaps.utils.PostgresUtils;
import org.junit.jupiter.api.BeforeEach;
public abstract class PostgresRepositoryTest extends PostgresContainerTest {
diff --git a/baremaps-core/src/test/java/org/apache/baremaps/openstreetmap/repository/BlockImporterTest.java b/baremaps-core/src/test/java/org/apache/baremaps/openstreetmap/repository/BlockImporterTest.java
index 3d3d1440..3b645c43 100644
--- a/baremaps-core/src/test/java/org/apache/baremaps/openstreetmap/repository/BlockImporterTest.java
+++ b/baremaps-core/src/test/java/org/apache/baremaps/openstreetmap/repository/BlockImporterTest.java
@@ -27,9 +27,9 @@ import org.apache.baremaps.openstreetmap.postgres.PostgresHeaderRepository;
import org.apache.baremaps.openstreetmap.postgres.PostgresNodeRepository;
import org.apache.baremaps.openstreetmap.postgres.PostgresRelationRepository;
import org.apache.baremaps.openstreetmap.postgres.PostgresWayRepository;
-import org.apache.baremaps.postgres.PostgresUtils;
import org.apache.baremaps.testing.PostgresContainerTest;
import org.apache.baremaps.testing.TestFiles;
+import org.apache.baremaps.utils.PostgresUtils;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Tag;
import org.junit.jupiter.api.Test;
@@ -44,7 +44,7 @@ class BlockImporterTest extends PostgresContainerTest {
@BeforeEach
void init() throws SQLException, IOException {
- dataSource = PostgresUtils.dataSource(jdbcUrl(), 1);
+ dataSource = PostgresUtils.createDataSource(jdbcUrl(), 1);
headerRepository = new PostgresHeaderRepository(dataSource);
nodeRepository = new PostgresNodeRepository(dataSource);
tableRepository = new PostgresWayRepository(dataSource);
diff --git a/baremaps-core/src/test/java/org/apache/baremaps/postgres/metadata/PostgresMetadataTest.java b/baremaps-core/src/test/java/org/apache/baremaps/postgres/metadata/PostgresMetadataTest.java
index cf3e9367..411318ce 100644
--- a/baremaps-core/src/test/java/org/apache/baremaps/postgres/metadata/PostgresMetadataTest.java
+++ b/baremaps-core/src/test/java/org/apache/baremaps/postgres/metadata/PostgresMetadataTest.java
@@ -20,8 +20,8 @@ import java.sql.DatabaseMetaData;
import java.sql.DriverManager;
import java.sql.ResultSet;
import java.sql.SQLException;
-import org.apache.baremaps.postgres.PostgresUtils;
import org.apache.baremaps.testing.PostgresContainerTest;
+import org.apache.baremaps.utils.PostgresUtils;
import org.junit.jupiter.api.Tag;
import org.junit.jupiter.api.Test;
diff --git a/baremaps-core/src/test/java/org/apache/baremaps/tilestore/mbtiles/MBTilesTest.java b/baremaps-core/src/test/java/org/apache/baremaps/tilestore/mbtiles/MBTilesStoreTest.java
similarity index 90%
rename from baremaps-core/src/test/java/org/apache/baremaps/tilestore/mbtiles/MBTilesTest.java
rename to baremaps-core/src/test/java/org/apache/baremaps/tilestore/mbtiles/MBTilesStoreTest.java
index b677e5d5..f28154b4 100644
--- a/baremaps-core/src/test/java/org/apache/baremaps/tilestore/mbtiles/MBTilesTest.java
+++ b/baremaps-core/src/test/java/org/apache/baremaps/tilestore/mbtiles/MBTilesStoreTest.java
@@ -28,7 +28,7 @@ import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
import org.sqlite.SQLiteDataSource;
-class MBTilesTest extends TileStoreTest {
+class MBTilesStoreTest extends TileStoreTest {
Path file;
@@ -43,18 +43,18 @@ class MBTilesTest extends TileStoreTest {
}
@Override
- public MBTiles createTileStore() throws Exception {
+ public MBTilesStore createTileStore() throws Exception {
SQLiteDataSource dataSource = new SQLiteDataSource();
String url = "jdbc:sqlite:" + file.toAbsolutePath();
dataSource.setUrl(url);
- MBTiles tilesStore = new MBTiles(dataSource);
+ MBTilesStore tilesStore = new MBTilesStore(dataSource);
tilesStore.initializeDatabase();
return tilesStore;
}
@Test
void readWriteMetadata() throws Exception {
- MBTiles tileStore = createTileStore();
+ MBTilesStore tileStore = createTileStore();
Map<String, String> metadata = tileStore.readMetadata();
assertTrue(metadata.size() == 0);
diff --git a/baremaps-ogcapi/src/test/java/org/apache/baremaps/ogcapi/OgcApiTest.java b/baremaps-ogcapi/src/test/java/org/apache/baremaps/ogcapi/OgcApiTest.java
index 6cff8fa5..0b8b0dbd 100644
--- a/baremaps-ogcapi/src/test/java/org/apache/baremaps/ogcapi/OgcApiTest.java
+++ b/baremaps-ogcapi/src/test/java/org/apache/baremaps/ogcapi/OgcApiTest.java
@@ -20,9 +20,9 @@ import java.io.IOException;
import java.nio.file.Path;
import java.nio.file.Paths;
import javax.sql.DataSource;
-import org.apache.baremaps.postgres.PostgresUtils;
import org.apache.baremaps.tilestore.TileStore;
import org.apache.baremaps.tilestore.postgres.PostgresTileStore;
+import org.apache.baremaps.utils.PostgresUtils;
import org.apache.baremaps.vectortile.tileset.Tileset;
import org.glassfish.hk2.utilities.binding.AbstractBinder;
import org.glassfish.jersey.client.ClientConfig;
@@ -40,7 +40,7 @@ public abstract class OgcApiTest extends JerseyTest {
@Override
protected ResourceConfig configure() {
// Create a datasource to a throwaway postgis database
- dataSource = PostgresUtils.dataSource("jdbc:tc:postgis:13-3.1:///baremaps");
+ dataSource = PostgresUtils.createDataSource("jdbc:tc:postgis:13-3.1:///baremaps");
// Initialize the database
try (var connection = dataSource.getConnection()) {
[incubator-baremaps] 03/09: Add MBTiles benchmark and refactor TileStore
Posted by bc...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
bchapuis pushed a commit to branch 688-export
in repository https://gitbox.apache.org/repos/asf/incubator-baremaps.git
commit fa23fc43c6570a69fde0ee98b1525ae865aa9711
Author: Bertil Chapuis <bc...@gmail.com>
AuthorDate: Tue Jun 13 09:56:38 2023 +0200
Add MBTiles benchmark and refactor TileStore
---
.../baremaps/benchmarks/MBTilesBenchmark.java | 68 ++++++++++++++++++++++
.../org/apache/baremaps/tilestore/TileCache.java | 10 ++--
.../org/apache/baremaps/tilestore/TileChannel.java | 4 +-
.../org/apache/baremaps/tilestore/TileStore.java | 38 ++++++++++--
.../baremaps/tilestore/file/FileTileStore.java | 4 +-
.../apache/baremaps/tilestore/mbtiles/MBTiles.java | 4 +-
.../tilestore/postgres/PostgresTileStore.java | 4 +-
.../baremaps/workflow/tasks/ExportVectorTiles.java | 28 +++++----
.../apache/baremaps/tilestore/TileStoreTest.java | 6 +-
.../org/apache/baremaps/ogcapi/TilesResource.java | 2 +-
10 files changed, 136 insertions(+), 32 deletions(-)
diff --git a/baremaps-benchmark/src/main/java/org/apache/baremaps/benchmarks/MBTilesBenchmark.java b/baremaps-benchmark/src/main/java/org/apache/baremaps/benchmarks/MBTilesBenchmark.java
new file mode 100644
index 00000000..2a71dbe1
--- /dev/null
+++ b/baremaps-benchmark/src/main/java/org/apache/baremaps/benchmarks/MBTilesBenchmark.java
@@ -0,0 +1,68 @@
+/*
+ * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except
+ * in compliance with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software distributed under the License
+ * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
+ * or implied. See the License for the specific language governing permissions and limitations under
+ * the License.
+ */
+
+package org.apache.baremaps.benchmarks;
+
+import java.io.File;
+import java.io.IOException;
+import java.nio.ByteBuffer;
+import java.util.Random;
+import java.util.concurrent.TimeUnit;
+import org.apache.baremaps.tilestore.TileCoord;
+import org.apache.baremaps.tilestore.TileStoreException;
+import org.apache.baremaps.tilestore.mbtiles.MBTiles;
+import org.apache.baremaps.workflow.tasks.ExportVectorTiles;
+import org.openjdk.jmh.annotations.*;
+import org.openjdk.jmh.runner.Runner;
+import org.openjdk.jmh.runner.RunnerException;
+import org.openjdk.jmh.runner.options.Options;
+import org.openjdk.jmh.runner.options.OptionsBuilder;
+
+@State(Scope.Benchmark)
+@OutputTimeUnit(TimeUnit.MILLISECONDS)
+@Fork(value = 1, warmups = 1)
+public class MBTilesBenchmark {
+
+ public Random random = new Random(0);
+
+ @Param({"10", "100", "1000"})
+ public int iterations;
+
+ private MBTiles mbTiles;
+
+ @Setup
+ public void setup() throws IOException, TileStoreException {
+ var sqliteFile = File.createTempFile("baremaps", ".sqlite");
+ sqliteFile.deleteOnExit();
+ var sqliteDataSource = ExportVectorTiles.createDataSource(sqliteFile.toPath());
+ mbTiles = new MBTiles(sqliteDataSource);
+ mbTiles.initializeDatabase();
+ }
+
+ @Benchmark
+ @BenchmarkMode(Mode.SingleShotTime)
+ public void writeMBTiles(MBTilesBenchmark benchmark) throws TileStoreException {
+ for (int i = 0; i < benchmark.iterations; i++) {
+ var bytes = new byte[1 << 16];
+ random.nextBytes(bytes);
+ mbTiles.put(new TileCoord(0, 0, i), ByteBuffer.wrap(bytes));
+ }
+ }
+
+ public static void main(String[] args) throws RunnerException {
+ Options opt =
+ new OptionsBuilder().include(MBTilesBenchmark.class.getSimpleName()).forks(1).build();
+ new Runner(opt).run();
+ }
+
+
+}
diff --git a/baremaps-core/src/main/java/org/apache/baremaps/tilestore/TileCache.java b/baremaps-core/src/main/java/org/apache/baremaps/tilestore/TileCache.java
index 6e037790..05893add 100644
--- a/baremaps-core/src/main/java/org/apache/baremaps/tilestore/TileCache.java
+++ b/baremaps-core/src/main/java/org/apache/baremaps/tilestore/TileCache.java
@@ -50,10 +50,10 @@ public class TileCache implements TileStore {
/** {@inheritDoc} */
@Override
- public ByteBuffer read(TileCoord tileCoord) throws TileStoreException {
+ public ByteBuffer get(TileCoord tileCoord) throws TileStoreException {
var buffer = cache.get(tileCoord, t -> {
try {
- return tileStore.read(t);
+ return tileStore.get(t);
} catch (TileStoreException e) {
logger.error("Unable to read the tile.", e);
return null;
@@ -68,8 +68,8 @@ public class TileCache implements TileStore {
/** {@inheritDoc} */
@Override
- public void write(TileCoord tileCoord, ByteBuffer bytes) throws TileStoreException {
- tileStore.write(tileCoord, bytes);
+ public void put(TileCoord tileCoord, ByteBuffer bytes) throws TileStoreException {
+ tileStore.put(tileCoord, bytes);
cache.invalidate(tileCoord);
}
@@ -79,4 +79,4 @@ public class TileCache implements TileStore {
tileStore.delete(tileCoord);
cache.invalidate(tileCoord);
}
-}
+}
\ No newline at end of file
diff --git a/baremaps-core/src/main/java/org/apache/baremaps/tilestore/TileChannel.java b/baremaps-core/src/main/java/org/apache/baremaps/tilestore/TileChannel.java
index 0620f2b4..e2e92f68 100644
--- a/baremaps-core/src/main/java/org/apache/baremaps/tilestore/TileChannel.java
+++ b/baremaps-core/src/main/java/org/apache/baremaps/tilestore/TileChannel.java
@@ -53,9 +53,9 @@ public class TileChannel implements Consumer<TileCoord> {
@Override
public void accept(TileCoord tileCoord) {
try {
- ByteBuffer blob = source.read(tileCoord);
+ ByteBuffer blob = source.get(tileCoord);
if (blob != null) {
- target.write(tileCoord, blob);
+ target.put(tileCoord, blob);
} else if (deleteEmptyTiles) {
target.delete(tileCoord);
}
diff --git a/baremaps-core/src/main/java/org/apache/baremaps/tilestore/TileStore.java b/baremaps-core/src/main/java/org/apache/baremaps/tilestore/TileStore.java
index c7288383..dc5c3609 100644
--- a/baremaps-core/src/main/java/org/apache/baremaps/tilestore/TileStore.java
+++ b/baremaps-core/src/main/java/org/apache/baremaps/tilestore/TileStore.java
@@ -15,27 +15,57 @@ package org.apache.baremaps.tilestore;
import java.nio.ByteBuffer;
+import java.util.ArrayList;
+import java.util.List;
/** Represents a store for tiles. */
public interface TileStore {
/**
- * Reads the content of a tile.
+ * Gets the content of a tile.
*
* @param tileCoord the tile coordinate
* @return the content of the tile
* @throws TileStoreException
*/
- ByteBuffer read(TileCoord tileCoord) throws TileStoreException;
+ ByteBuffer get(TileCoord tileCoord) throws TileStoreException;
/**
- * Writes the content of a tile.
+ * Gets the content of several tiles.
+ *
+ * @param tileCoords the tile coordinates
+ * @return the content of the tiles
+ * @throws TileStoreException
+ */
+ default List<ByteBuffer> get(List<TileCoord> tileCoords) throws TileStoreException {
+ var blobs = new ArrayList<ByteBuffer>(tileCoords.size());
+ for (var tileCoord : tileCoords) {
+ blobs.add(get(tileCoord));
+ }
+ return blobs;
+ }
+
+ /**
+ * Puts the content of a tile.
*
* @param tileCoord the tile coordinate
* @param blob the content of the tile
* @throws TileStoreException
*/
- void write(TileCoord tileCoord, ByteBuffer blob) throws TileStoreException;
+ void put(TileCoord tileCoord, ByteBuffer blob) throws TileStoreException;
+
+ /**
+ * Puts the content of several tiles.
+ *
+ * @param tileCoords the tile coordinates
+ * @param blobs the content of the tiles
+ * @throws TileStoreException
+ */
+ default void put(List<TileCoord> tileCoords, List<ByteBuffer> blobs) throws TileStoreException {
+ for (int i = 0; i < tileCoords.size(); i++) {
+ put(tileCoords.get(i), blobs.get(i));
+ }
+ }
/**
* Deletes the content of a tile.
diff --git a/baremaps-core/src/main/java/org/apache/baremaps/tilestore/file/FileTileStore.java b/baremaps-core/src/main/java/org/apache/baremaps/tilestore/file/FileTileStore.java
index 77424004..8c254754 100644
--- a/baremaps-core/src/main/java/org/apache/baremaps/tilestore/file/FileTileStore.java
+++ b/baremaps-core/src/main/java/org/apache/baremaps/tilestore/file/FileTileStore.java
@@ -38,7 +38,7 @@ public class FileTileStore implements TileStore {
/** {@inheritDoc} */
@Override
- public ByteBuffer read(TileCoord tileCoord) throws TileStoreException {
+ public ByteBuffer get(TileCoord tileCoord) throws TileStoreException {
try {
return ByteBuffer.wrap(Files.readAllBytes(resolve(tileCoord)));
} catch (IOException e) {
@@ -48,7 +48,7 @@ public class FileTileStore implements TileStore {
/** {@inheritDoc} */
@Override
- public void write(TileCoord tileCoord, ByteBuffer blob) throws TileStoreException {
+ public void put(TileCoord tileCoord, ByteBuffer blob) throws TileStoreException {
try {
var file = resolve(tileCoord);
Files.createDirectories(file.getParent());
diff --git a/baremaps-core/src/main/java/org/apache/baremaps/tilestore/mbtiles/MBTiles.java b/baremaps-core/src/main/java/org/apache/baremaps/tilestore/mbtiles/MBTiles.java
index 06d0e0cb..1a0a5352 100644
--- a/baremaps-core/src/main/java/org/apache/baremaps/tilestore/mbtiles/MBTiles.java
+++ b/baremaps-core/src/main/java/org/apache/baremaps/tilestore/mbtiles/MBTiles.java
@@ -72,7 +72,7 @@ public class MBTiles implements TileStore {
/** {@inheritDoc} */
@Override
- public ByteBuffer read(TileCoord tileCoord) throws TileStoreException {
+ public ByteBuffer get(TileCoord tileCoord) throws TileStoreException {
try (Connection connection = dataSource.getConnection();
PreparedStatement statement = connection.prepareStatement(SELECT_TILE)) {
statement.setInt(1, tileCoord.z());
@@ -92,7 +92,7 @@ public class MBTiles implements TileStore {
/** {@inheritDoc} */
@Override
- public void write(TileCoord tileCoord, ByteBuffer blob) throws TileStoreException {
+ public void put(TileCoord tileCoord, ByteBuffer blob) throws TileStoreException {
try (Connection connection = dataSource.getConnection();
PreparedStatement statement = connection.prepareStatement(INSERT_TILE)) {
statement.setInt(1, tileCoord.z());
diff --git a/baremaps-core/src/main/java/org/apache/baremaps/tilestore/postgres/PostgresTileStore.java b/baremaps-core/src/main/java/org/apache/baremaps/tilestore/postgres/PostgresTileStore.java
index 1e1eb6c3..e4d4485f 100644
--- a/baremaps-core/src/main/java/org/apache/baremaps/tilestore/postgres/PostgresTileStore.java
+++ b/baremaps-core/src/main/java/org/apache/baremaps/tilestore/postgres/PostgresTileStore.java
@@ -99,7 +99,7 @@ public class PostgresTileStore implements TileStore {
/** {@inheritDoc} */
@Override
- public ByteBuffer read(TileCoord tileCoord) throws TileStoreException {
+ public ByteBuffer get(TileCoord tileCoord) throws TileStoreException {
try (Connection connection = datasource.getConnection();
Statement statement = connection.createStatement();
ByteArrayOutputStream data = new ByteArrayOutputStream()) {
@@ -239,7 +239,7 @@ public class PostgresTileStore implements TileStore {
}
/** This operation is not supported. */
- public void write(TileCoord tileCoord, ByteBuffer blob) {
+ public void put(TileCoord tileCoord, ByteBuffer blob) {
throw new UnsupportedOperationException("The postgis tile store is read only");
}
diff --git a/baremaps-core/src/main/java/org/apache/baremaps/workflow/tasks/ExportVectorTiles.java b/baremaps-core/src/main/java/org/apache/baremaps/workflow/tasks/ExportVectorTiles.java
index 13e49f48..f0b8ac83 100644
--- a/baremaps-core/src/main/java/org/apache/baremaps/workflow/tasks/ExportVectorTiles.java
+++ b/baremaps-core/src/main/java/org/apache/baremaps/workflow/tasks/ExportVectorTiles.java
@@ -87,17 +87,7 @@ public record ExportVectorTiles(
if (mbtiles) {
Files.deleteIfExists(repository);
- var sqliteConfig = new SQLiteConfig();
- sqliteConfig.setCacheSize(1000000);
- sqliteConfig.setPageSize(65536);
- sqliteConfig.setJournalMode(JournalMode.OFF);
- sqliteConfig.setLockingMode(LockingMode.EXCLUSIVE);
- sqliteConfig.setSynchronous(SynchronousMode.OFF);
- sqliteConfig.setTempStore(TempStore.MEMORY);
-
- var sqliteDataSource = new SQLiteDataSource();
- sqliteDataSource.setConfig(sqliteConfig);
- sqliteDataSource.setUrl("jdbc:sqlite:" + repository);
+ var sqliteDataSource = createDataSource(repository);
var hikariConfig = new HikariConfig();
hikariConfig.setDataSource(sqliteDataSource);
@@ -114,6 +104,22 @@ public record ExportVectorTiles(
}
}
+ public static SQLiteDataSource createDataSource(Path path) {
+ var sqliteConfig = new SQLiteConfig();
+ sqliteConfig.setCacheSize(1000000);
+ sqliteConfig.setPageSize(65536);
+ sqliteConfig.setJournalMode(JournalMode.OFF);
+ sqliteConfig.setLockingMode(LockingMode.EXCLUSIVE);
+ sqliteConfig.setSynchronous(SynchronousMode.OFF);
+ sqliteConfig.setTempStore(TempStore.MEMORY);
+
+ var sqliteDataSource = new SQLiteDataSource();
+ sqliteDataSource.setConfig(sqliteConfig);
+ sqliteDataSource.setUrl("jdbc:sqlite:" + path);
+
+ return sqliteDataSource;
+ }
+
private Map<String, String> metadata(Tileset tileset) throws JsonProcessingException {
var metadata = new HashMap<String, String>();
diff --git a/baremaps-core/src/test/java/org/apache/baremaps/tilestore/TileStoreTest.java b/baremaps-core/src/test/java/org/apache/baremaps/tilestore/TileStoreTest.java
index b1be4229..7f3ea965 100644
--- a/baremaps-core/src/test/java/org/apache/baremaps/tilestore/TileStoreTest.java
+++ b/baremaps-core/src/test/java/org/apache/baremaps/tilestore/TileStoreTest.java
@@ -31,15 +31,15 @@ public abstract class TileStoreTest {
ByteBuffer blob = ByteBuffer.wrap("tile_content".getBytes());
// Write data
- tileStore.write(tileCoord, blob);
+ tileStore.put(tileCoord, blob);
// Read the data
- ByteBuffer inputStream = tileStore.read(tileCoord);
+ ByteBuffer inputStream = tileStore.get(tileCoord);
assertArrayEquals(blob.array(), inputStream.array());
// Delete the data
tileStore.delete(tileCoord);
- assertThrows(TileStoreException.class, () -> tileStore.read(tileCoord));
+ assertThrows(TileStoreException.class, () -> tileStore.get(tileCoord));
}
public abstract TileStore createTileStore() throws Exception;
diff --git a/baremaps-ogcapi/src/main/java/org/apache/baremaps/ogcapi/TilesResource.java b/baremaps-ogcapi/src/main/java/org/apache/baremaps/ogcapi/TilesResource.java
index 75d2add6..3200996a 100644
--- a/baremaps-ogcapi/src/main/java/org/apache/baremaps/ogcapi/TilesResource.java
+++ b/baremaps-ogcapi/src/main/java/org/apache/baremaps/ogcapi/TilesResource.java
@@ -100,7 +100,7 @@ public class TilesResource implements TilesApi {
int y = tileCol;
TileCoord tileCoord = new TileCoord(x, y, z);
try {
- ByteBuffer blob = tileStore.read(tileCoord);
+ ByteBuffer blob = tileStore.get(tileCoord);
if (blob != null) {
return Response.status(200) // lgtm [java/xss]
.header(ACCESS_CONTROL_ALLOW_ORIGIN, "*")
[incubator-baremaps] 06/09: Add mbtiles command
Posted by bc...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
bchapuis pushed a commit to branch 688-export
in repository https://gitbox.apache.org/repos/asf/incubator-baremaps.git
commit b297ed1d487f9a7d6c1812259559fa2a74a386ab
Author: Bertil Chapuis <bc...@gmail.com>
AuthorDate: Fri Jun 16 22:51:27 2023 +0200
Add mbtiles command
---
.run/basemap-mbtiles.run.xml | 17 ++++
.../baremaps/benchmarks/MBTilesBenchmark.java | 21 +++--
.../java/org/apache/baremaps/cli/map/MBTiles.java | 94 ++++++++++++++++++++++
.../main/java/org/apache/baremaps/cli/map/Map.java | 2 +-
.../tilestore/postgres/PostgresTileStore.java | 2 +
.../baremaps/workflow/tasks/ExportVectorTiles.java | 5 +-
6 files changed, 129 insertions(+), 12 deletions(-)
diff --git a/.run/basemap-mbtiles.run.xml b/.run/basemap-mbtiles.run.xml
new file mode 100644
index 00000000..3a08cf9a
--- /dev/null
+++ b/.run/basemap-mbtiles.run.xml
@@ -0,0 +1,17 @@
+<component name="ProjectRunConfigurationManager">
+ <configuration default="false" name="basemap-mbtiles" type="Application" factoryName="Application">
+ <option name="MAIN_CLASS_NAME" value="org.apache.baremaps.cli.Baremaps" />
+ <module name="baremaps-cli" />
+ <option name="PROGRAM_PARAMETERS" value="map mbtiles --mbtiles $USER_HOME$/Datasets/Baremaps/tiles.mbtiles --tileset tileset.js --style style.js" />
+ <option name="WORKING_DIRECTORY" value="$PROJECT_DIR$/basemap" />
+ <extension name="coverage">
+ <pattern>
+ <option name="PATTERN" value="org.apache.baremaps.server.ogcapi.*" />
+ <option name="ENABLED" value="true" />
+ </pattern>
+ </extension>
+ <method v="2">
+ <option name="Make" enabled="true" />
+ </method>
+ </configuration>
+</component>
\ No newline at end of file
diff --git a/baremaps-benchmark/src/main/java/org/apache/baremaps/benchmarks/MBTilesBenchmark.java b/baremaps-benchmark/src/main/java/org/apache/baremaps/benchmarks/MBTilesBenchmark.java
index b529eb66..dc333999 100644
--- a/baremaps-benchmark/src/main/java/org/apache/baremaps/benchmarks/MBTilesBenchmark.java
+++ b/baremaps-benchmark/src/main/java/org/apache/baremaps/benchmarks/MBTilesBenchmark.java
@@ -12,11 +12,13 @@
package org.apache.baremaps.benchmarks;
-import java.io.File;
import java.io.IOException;
import java.nio.ByteBuffer;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.Paths;
+import java.security.SecureRandom;
import java.util.ArrayList;
-import java.util.Random;
import java.util.concurrent.TimeUnit;
import org.apache.baremaps.tilestore.TileCoord;
import org.apache.baremaps.tilestore.TileStoreException;
@@ -33,22 +35,27 @@ import org.openjdk.jmh.runner.options.OptionsBuilder;
@Fork(value = 1, warmups = 1)
public class MBTilesBenchmark {
- public Random random = new Random(0);
+ public SecureRandom random = new SecureRandom();
@Param({"10", "100", "1000"})
public int iterations;
+ private Path file;
+
private MBTiles mbTiles;
@Setup
public void setup() throws IOException, TileStoreException {
- var sqliteFile = File.createTempFile("baremaps", ".sqlite");
- sqliteFile.deleteOnExit();
- var sqliteDataSource = ExportVectorTiles.createDataSource(sqliteFile.toPath());
- mbTiles = new MBTiles(sqliteDataSource);
+ file = Files.createTempFile(Paths.get("."), "baremaps", ".mbtiles");
+ mbTiles = new MBTiles(ExportVectorTiles.createDataSource(file));
mbTiles.initializeDatabase();
}
+ @TearDown
+ public void tearDown() throws IOException, TileStoreException {
+ Files.delete(file);
+ }
+
@Benchmark
@BenchmarkMode(Mode.SingleShotTime)
public void writeMBTiles(MBTilesBenchmark benchmark) throws TileStoreException {
diff --git a/baremaps-cli/src/main/java/org/apache/baremaps/cli/map/MBTiles.java b/baremaps-cli/src/main/java/org/apache/baremaps/cli/map/MBTiles.java
new file mode 100644
index 00000000..0f8c7f50
--- /dev/null
+++ b/baremaps-cli/src/main/java/org/apache/baremaps/cli/map/MBTiles.java
@@ -0,0 +1,94 @@
+/*
+ * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except
+ * in compliance with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software distributed under the License
+ * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
+ * or implied. See the License for the specific language governing permissions and limitations under
+ * the License.
+ */
+
+package org.apache.baremaps.cli.map;
+
+import static io.servicetalk.data.jackson.jersey.ServiceTalkJacksonSerializerFeature.newContextResolver;
+import static org.apache.baremaps.utils.ObjectMapperUtils.objectMapper;
+
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.github.benmanes.caffeine.cache.CaffeineSpec;
+import io.servicetalk.http.netty.HttpServers;
+import io.servicetalk.http.router.jersey.HttpJerseyRouterBuilder;
+import java.nio.file.Path;
+import java.util.concurrent.Callable;
+import org.apache.baremaps.cli.Options;
+import org.apache.baremaps.server.CorsFilter;
+import org.apache.baremaps.server.ServerResources;
+import org.apache.baremaps.tilestore.TileCache;
+import org.apache.baremaps.tilestore.TileStore;
+import org.apache.baremaps.workflow.tasks.ExportVectorTiles;
+import org.glassfish.hk2.utilities.binding.AbstractBinder;
+import org.glassfish.jersey.server.ResourceConfig;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import picocli.CommandLine.Command;
+import picocli.CommandLine.Mixin;
+import picocli.CommandLine.Option;
+
+@Command(name = "mbtiles", description = "Start a mbtiles server with caching capabilities.")
+public class MBTiles implements Callable<Integer> {
+
+ private static final Logger logger = LoggerFactory.getLogger(MBTiles.class);
+
+ @Mixin
+ private Options options;
+
+ @Option(names = {"--cache"}, paramLabel = "CACHE", description = "The caffeine cache directive.")
+ private String cache = "";
+
+ @Option(names = {"--mbtiles"}, paramLabel = "MBTILES", description = "The mbtiles file.",
+ required = true)
+ private Path mbtiles;
+
+ @Option(names = {"--tilejson"}, paramLabel = "TILEJSON", description = "The tileJSON file.",
+ required = true)
+ private Path tileset;
+
+ @Option(names = {"--style"}, paramLabel = "STYLE", description = "The style file.",
+ required = true)
+ private Path style;
+
+ @Option(names = {"--port"}, paramLabel = "PORT", description = "The port of the server.")
+ private int port = 9000;
+
+ @Override
+ public Integer call() throws Exception {
+ var objectMapper = objectMapper();
+ var caffeineSpec = CaffeineSpec.parse(cache);
+ var datasource = ExportVectorTiles.createDataSource(mbtiles);
+
+ var tileStore = new org.apache.baremaps.tilestore.mbtiles.MBTiles(datasource);
+ var tileCache = new TileCache(tileStore, caffeineSpec);
+
+ // Configure the application
+ var application =
+ new ResourceConfig().register(CorsFilter.class).register(ServerResources.class)
+ .register(newContextResolver(objectMapper)).register(new AbstractBinder() {
+ @Override
+ protected void configure() {
+ bind(tileset).to(Path.class).named("tileset");
+ bind(style).to(Path.class).named("style");
+ bind(tileCache).to(TileStore.class);
+ bind(objectMapper).to(ObjectMapper.class);
+ }
+ });
+
+ var httpService = new HttpJerseyRouterBuilder().buildBlockingStreaming(application);
+ var serverContext = HttpServers.forPort(port).listenBlockingStreamingAndAwait(httpService);
+
+ logger.info("Listening on {}", serverContext.listenAddress());
+
+ serverContext.awaitShutdown();
+ return 0;
+ }
+}
diff --git a/baremaps-cli/src/main/java/org/apache/baremaps/cli/map/Map.java b/baremaps-cli/src/main/java/org/apache/baremaps/cli/map/Map.java
index 294ba714..c7640b8b 100644
--- a/baremaps-cli/src/main/java/org/apache/baremaps/cli/map/Map.java
+++ b/baremaps-cli/src/main/java/org/apache/baremaps/cli/map/Map.java
@@ -18,7 +18,7 @@ import picocli.CommandLine;
import picocli.CommandLine.Command;
@Command(name = "map", description = "Map commands.",
- subcommands = {Init.class, Export.class, Serve.class, Dev.class, StyleCommand.class},
+ subcommands = {Init.class, Export.class, Serve.class, Dev.class, StyleCommand.class, MBTiles.class},
sortOptions = false)
public class Map implements Runnable {
diff --git a/baremaps-core/src/main/java/org/apache/baremaps/tilestore/postgres/PostgresTileStore.java b/baremaps-core/src/main/java/org/apache/baremaps/tilestore/postgres/PostgresTileStore.java
index e4d4485f..af8796f2 100644
--- a/baremaps-core/src/main/java/org/apache/baremaps/tilestore/postgres/PostgresTileStore.java
+++ b/baremaps-core/src/main/java/org/apache/baremaps/tilestore/postgres/PostgresTileStore.java
@@ -239,11 +239,13 @@ public class PostgresTileStore implements TileStore {
}
/** This operation is not supported. */
+ @Override
public void put(TileCoord tileCoord, ByteBuffer blob) {
throw new UnsupportedOperationException("The postgis tile store is read only");
}
/** This operation is not supported. */
+ @Override
public void delete(TileCoord tileCoord) {
throw new UnsupportedOperationException("The postgis tile store is read only");
}
diff --git a/baremaps-core/src/main/java/org/apache/baremaps/workflow/tasks/ExportVectorTiles.java b/baremaps-core/src/main/java/org/apache/baremaps/workflow/tasks/ExportVectorTiles.java
index 5cca2bec..43040de8 100644
--- a/baremaps-core/src/main/java/org/apache/baremaps/workflow/tasks/ExportVectorTiles.java
+++ b/baremaps-core/src/main/java/org/apache/baremaps/workflow/tasks/ExportVectorTiles.java
@@ -86,12 +86,10 @@ public record ExportVectorTiles(
private TileStore targetTileStore(Tileset source) throws TileStoreException, IOException {
if (mbtiles) {
Files.deleteIfExists(repository);
-
var dataSource = createDataSource(repository);
var tilesStore = new MBTiles(dataSource);
tilesStore.initializeDatabase();
tilesStore.writeMetadata(metadata(source));
-
return tilesStore;
} else {
return new FileTileStore(repository);
@@ -162,8 +160,7 @@ public record ExportVectorTiles(
var hikariConfig = new HikariConfig();
hikariConfig.setDataSource(sqliteDataSource);
hikariConfig.setMaximumPoolSize(1);
- var hikariDataSource = new HikariDataSource(hikariConfig);
- return hikariDataSource;
+ return new HikariDataSource(hikariConfig);
}
}
[incubator-baremaps] 04/09: Add batch insert in mbtiles
Posted by bc...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
bchapuis pushed a commit to branch 688-export
in repository https://gitbox.apache.org/repos/asf/incubator-baremaps.git
commit 69aff8bbc7ebd76c33a0ee1a5b4b58d86a3f2a81
Author: Bertil Chapuis <bc...@gmail.com>
AuthorDate: Tue Jun 13 23:49:52 2023 +0200
Add batch insert in mbtiles
---
.../baremaps/benchmarks/MBTilesBenchmark.java | 74 ++++++++++++++--------
.../apache/baremaps/tilestore/mbtiles/MBTiles.java | 21 ++++++
.../baremaps/workflow/tasks/ExportVectorTiles.java | 52 ++++++++-------
3 files changed, 98 insertions(+), 49 deletions(-)
diff --git a/baremaps-benchmark/src/main/java/org/apache/baremaps/benchmarks/MBTilesBenchmark.java b/baremaps-benchmark/src/main/java/org/apache/baremaps/benchmarks/MBTilesBenchmark.java
index 2a71dbe1..9dd817a9 100644
--- a/baremaps-benchmark/src/main/java/org/apache/baremaps/benchmarks/MBTilesBenchmark.java
+++ b/baremaps-benchmark/src/main/java/org/apache/baremaps/benchmarks/MBTilesBenchmark.java
@@ -15,8 +15,10 @@ package org.apache.baremaps.benchmarks;
import java.io.File;
import java.io.IOException;
import java.nio.ByteBuffer;
+import java.util.ArrayList;
import java.util.Random;
import java.util.concurrent.TimeUnit;
+
import org.apache.baremaps.tilestore.TileCoord;
import org.apache.baremaps.tilestore.TileStoreException;
import org.apache.baremaps.tilestore.mbtiles.MBTiles;
@@ -32,37 +34,59 @@ import org.openjdk.jmh.runner.options.OptionsBuilder;
@Fork(value = 1, warmups = 1)
public class MBTilesBenchmark {
- public Random random = new Random(0);
+ public Random random = new Random(0);
- @Param({"10", "100", "1000"})
- public int iterations;
+ @Param({"10", "100", "1000"})
+ public int iterations;
- private MBTiles mbTiles;
+ private MBTiles mbTiles;
- @Setup
- public void setup() throws IOException, TileStoreException {
- var sqliteFile = File.createTempFile("baremaps", ".sqlite");
- sqliteFile.deleteOnExit();
- var sqliteDataSource = ExportVectorTiles.createDataSource(sqliteFile.toPath());
- mbTiles = new MBTiles(sqliteDataSource);
- mbTiles.initializeDatabase();
- }
+ @Setup
+ public void setup() throws IOException, TileStoreException {
+ var sqliteFile = File.createTempFile("baremaps", ".sqlite");
+ sqliteFile.deleteOnExit();
+ var sqliteDataSource = ExportVectorTiles.createDataSource(sqliteFile.toPath());
+ mbTiles = new MBTiles(sqliteDataSource);
+ mbTiles.initializeDatabase();
+ }
- @Benchmark
- @BenchmarkMode(Mode.SingleShotTime)
- public void writeMBTiles(MBTilesBenchmark benchmark) throws TileStoreException {
- for (int i = 0; i < benchmark.iterations; i++) {
- var bytes = new byte[1 << 16];
- random.nextBytes(bytes);
- mbTiles.put(new TileCoord(0, 0, i), ByteBuffer.wrap(bytes));
+ @Benchmark
+ @BenchmarkMode(Mode.SingleShotTime)
+ public void writeMBTiles(MBTilesBenchmark benchmark) throws TileStoreException {
+ for (int i = 0; i < benchmark.iterations; i++) {
+ var bytes = new byte[1 << 16];
+ random.nextBytes(bytes);
+ mbTiles.put(new TileCoord(0, 0, i), ByteBuffer.wrap(bytes));
+ }
}
- }
- public static void main(String[] args) throws RunnerException {
- Options opt =
- new OptionsBuilder().include(MBTilesBenchmark.class.getSimpleName()).forks(1).build();
- new Runner(opt).run();
- }
+ @Benchmark
+ @BenchmarkMode(Mode.SingleShotTime)
+ public void writeMBTilesBatch(MBTilesBenchmark benchmark) throws TileStoreException {
+ var coords = new ArrayList<TileCoord>();
+ var buffers = new ArrayList<ByteBuffer>();
+ for (int i = 0; i < benchmark.iterations; i++) {
+ var bytes = new byte[1 << 16];
+ random.nextBytes(bytes);
+ coords.add(new TileCoord(0, 0, i));
+ buffers.add(ByteBuffer.wrap(bytes));
+ if (coords.size() == 100) {
+ random.nextBytes(bytes);
+ mbTiles.put(coords, buffers);
+ coords.clear();
+ buffers.clear();
+ }
+ }
+ mbTiles.put(coords, buffers);
+ coords.clear();
+ buffers.clear();
+ }
+
+ public static void main(String[] args) throws RunnerException {
+ Options opt =
+ new OptionsBuilder().include(MBTilesBenchmark.class.getSimpleName()).forks(1).build();
+ new Runner(opt).run();
+ }
}
diff --git a/baremaps-core/src/main/java/org/apache/baremaps/tilestore/mbtiles/MBTiles.java b/baremaps-core/src/main/java/org/apache/baremaps/tilestore/mbtiles/MBTiles.java
index 1a0a5352..fa1fb9d7 100644
--- a/baremaps-core/src/main/java/org/apache/baremaps/tilestore/mbtiles/MBTiles.java
+++ b/baremaps-core/src/main/java/org/apache/baremaps/tilestore/mbtiles/MBTiles.java
@@ -22,6 +22,7 @@ import java.sql.ResultSet;
import java.sql.SQLException;
import java.sql.Statement;
import java.util.HashMap;
+import java.util.List;
import java.util.Map;
import javax.sql.DataSource;
import org.apache.baremaps.tilestore.TileCoord;
@@ -105,6 +106,26 @@ public class MBTiles implements TileStore {
}
}
+ /** {@inheritDoc} */
+ @Override
+ public void put(List<TileCoord> tileCoords, List<ByteBuffer> blobs) throws TileStoreException {
+ try (Connection connection = dataSource.getConnection();
+ PreparedStatement statement = connection.prepareStatement(INSERT_TILE)) {
+ for (int i = 0; i < tileCoords.size(); i++) {
+ TileCoord tileCoord = tileCoords.get(i);
+ ByteBuffer blob = blobs.get(i);
+ statement.setInt(1, tileCoord.z());
+ statement.setInt(2, tileCoord.x());
+ statement.setInt(3, reverseY(tileCoord.y(), tileCoord.z()));
+ statement.setBytes(4, blob.array());
+ statement.addBatch();
+ }
+ statement.executeBatch();
+ } catch (SQLException e) {
+ throw new TileStoreException(e);
+ }
+ }
+
/** {@inheritDoc} */
@Override
public void delete(TileCoord tileCoord) throws TileStoreException {
diff --git a/baremaps-core/src/main/java/org/apache/baremaps/workflow/tasks/ExportVectorTiles.java b/baremaps-core/src/main/java/org/apache/baremaps/workflow/tasks/ExportVectorTiles.java
index f0b8ac83..435556ea 100644
--- a/baremaps-core/src/main/java/org/apache/baremaps/workflow/tasks/ExportVectorTiles.java
+++ b/baremaps-core/src/main/java/org/apache/baremaps/workflow/tasks/ExportVectorTiles.java
@@ -87,14 +87,8 @@ public record ExportVectorTiles(
if (mbtiles) {
Files.deleteIfExists(repository);
- var sqliteDataSource = createDataSource(repository);
-
- var hikariConfig = new HikariConfig();
- hikariConfig.setDataSource(sqliteDataSource);
- hikariConfig.setMaximumPoolSize(1);
- var hikariDataSource = new HikariDataSource(hikariConfig);
-
- var tilesStore = new MBTiles(hikariDataSource);
+ var dataSource = createDataSource(repository);
+ var tilesStore = new MBTiles(dataSource);
tilesStore.initializeDatabase();
tilesStore.writeMetadata(metadata(source));
@@ -104,22 +98,6 @@ public record ExportVectorTiles(
}
}
- public static SQLiteDataSource createDataSource(Path path) {
- var sqliteConfig = new SQLiteConfig();
- sqliteConfig.setCacheSize(1000000);
- sqliteConfig.setPageSize(65536);
- sqliteConfig.setJournalMode(JournalMode.OFF);
- sqliteConfig.setLockingMode(LockingMode.EXCLUSIVE);
- sqliteConfig.setSynchronous(SynchronousMode.OFF);
- sqliteConfig.setTempStore(TempStore.MEMORY);
-
- var sqliteDataSource = new SQLiteDataSource();
- sqliteDataSource.setConfig(sqliteConfig);
- sqliteDataSource.setUrl("jdbc:sqlite:" + path);
-
- return sqliteDataSource;
- }
-
private Map<String, String> metadata(Tileset tileset) throws JsonProcessingException {
var metadata = new HashMap<String, String>();
@@ -161,4 +139,30 @@ public record ExportVectorTiles(
return metadata;
}
+
+ /**
+ * Create a SQLite data source.
+ * @param path the path to the SQLite database
+ * @return the SQLite data source
+ */
+ public static DataSource createDataSource(Path path) {
+ var sqliteConfig = new SQLiteConfig();
+ sqliteConfig.setCacheSize(1000000);
+ sqliteConfig.setPageSize(65536);
+ sqliteConfig.setJournalMode(JournalMode.OFF);
+ sqliteConfig.setLockingMode(LockingMode.EXCLUSIVE);
+ sqliteConfig.setSynchronous(SynchronousMode.OFF);
+ sqliteConfig.setTempStore(TempStore.MEMORY);
+
+ var sqliteDataSource = new SQLiteDataSource();
+ sqliteDataSource.setConfig(sqliteConfig);
+ sqliteDataSource.setUrl("jdbc:sqlite:" + path);
+
+ var hikariConfig = new HikariConfig();
+ hikariConfig.setDataSource(sqliteDataSource);
+ hikariConfig.setMaximumPoolSize(1);
+ var hikariDataSource = new HikariDataSource(hikariConfig);
+
+ return hikariDataSource;
+ }
}
[incubator-baremaps] 09/09: Fix formatting
Posted by bc...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
bchapuis pushed a commit to branch 688-export
in repository https://gitbox.apache.org/repos/asf/incubator-baremaps.git
commit a4a06203e99094eb3d00c9fb7db6a8fc8dc2a6a3
Author: Bertil Chapuis <bc...@gmail.com>
AuthorDate: Fri Jun 23 14:20:42 2023 +0200
Fix formatting
---
.../src/main/java/org/apache/baremaps/cli/map/Map.java | 3 ++-
.../java/org/apache/baremaps/tilestore/TileCache.java | 18 +++---------------
2 files changed, 5 insertions(+), 16 deletions(-)
diff --git a/baremaps-cli/src/main/java/org/apache/baremaps/cli/map/Map.java b/baremaps-cli/src/main/java/org/apache/baremaps/cli/map/Map.java
index c7640b8b..4fed8fe9 100644
--- a/baremaps-cli/src/main/java/org/apache/baremaps/cli/map/Map.java
+++ b/baremaps-cli/src/main/java/org/apache/baremaps/cli/map/Map.java
@@ -18,7 +18,8 @@ import picocli.CommandLine;
import picocli.CommandLine.Command;
@Command(name = "map", description = "Map commands.",
- subcommands = {Init.class, Export.class, Serve.class, Dev.class, StyleCommand.class, MBTiles.class},
+ subcommands = {Init.class, Export.class, Serve.class, Dev.class, StyleCommand.class,
+ MBTiles.class},
sortOptions = false)
public class Map implements Runnable {
diff --git a/baremaps-core/src/main/java/org/apache/baremaps/tilestore/TileCache.java b/baremaps-core/src/main/java/org/apache/baremaps/tilestore/TileCache.java
index d6b2fa33..6e037790 100644
--- a/baremaps-core/src/main/java/org/apache/baremaps/tilestore/TileCache.java
+++ b/baremaps-core/src/main/java/org/apache/baremaps/tilestore/TileCache.java
@@ -50,22 +50,10 @@ public class TileCache implements TileStore {
/** {@inheritDoc} */
@Override
-<<<<<<< HEAD
- public ByteBuffer get(TileCoord tileCoord) throws TileStoreException {
- var buffer = cache.get(tileCoord, t -> {
- try {
- return tileStore.get(t);
-=======
public ByteBuffer read(TileCoord tileCoord) throws TileStoreException {
- return cache.get(tileCoord, t -> {
+ var buffer = cache.get(tileCoord, t -> {
try {
- var buffer = tileStore.read(t);
- if (buffer == null) {
- return null;
- } else {
- return buffer;
- }
->>>>>>> 1d7ffeec (Change the TileStore interface)
+ return tileStore.read(t);
} catch (TileStoreException e) {
logger.error("Unable to read the tile.", e);
return null;
@@ -91,4 +79,4 @@ public class TileCache implements TileStore {
tileStore.delete(tileCoord);
cache.invalidate(tileCoord);
}
-}
\ No newline at end of file
+}
[incubator-baremaps] 08/09: Change the TileStore interface
Posted by bc...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
bchapuis pushed a commit to branch 688-export
in repository https://gitbox.apache.org/repos/asf/incubator-baremaps.git
commit bc4397093bcf257c85fb83aa339cb5018d465e70
Author: Bertil Chapuis <bc...@gmail.com>
AuthorDate: Mon Jun 19 12:03:21 2023 +0200
Change the TileStore interface
---
.run/basemap-mbtiles.run.xml | 2 +-
.../baremaps/benchmarks/MBTilesBenchmark.java | 10 +++---
.../main/java/org/apache/baremaps/cli/map/Dev.java | 4 +--
.../java/org/apache/baremaps/cli/map/MBTiles.java | 41 ++++++++++++++++------
.../java/org/apache/baremaps/cli/map/Serve.java | 9 +++--
.../org/apache/baremaps/tilestore/TileCache.java | 16 +++++++--
.../org/apache/baremaps/tilestore/TileChannel.java | 4 +--
.../org/apache/baremaps/tilestore/TileStore.java | 20 +++++------
.../baremaps/tilestore/file/FileTileStore.java | 4 +--
.../baremaps/tilestore/mbtiles/MBTilesStore.java | 6 ++--
.../tilestore/postgres/PostgresTileStore.java | 4 +--
.../org/apache/baremaps/utils/SqliteUtils.java | 4 +--
.../baremaps/workflow/tasks/ExportVectorTiles.java | 2 +-
.../apache/baremaps/tilestore/TileStoreTest.java | 6 ++--
.../tilestore/mbtiles/MBTilesStoreTest.java | 7 ++--
.../org/apache/baremaps/ogcapi/TilesResource.java | 2 +-
16 files changed, 85 insertions(+), 56 deletions(-)
diff --git a/.run/basemap-mbtiles.run.xml b/.run/basemap-mbtiles.run.xml
index 3a08cf9a..6a57530d 100644
--- a/.run/basemap-mbtiles.run.xml
+++ b/.run/basemap-mbtiles.run.xml
@@ -2,7 +2,7 @@
<configuration default="false" name="basemap-mbtiles" type="Application" factoryName="Application">
<option name="MAIN_CLASS_NAME" value="org.apache.baremaps.cli.Baremaps" />
<module name="baremaps-cli" />
- <option name="PROGRAM_PARAMETERS" value="map mbtiles --mbtiles $USER_HOME$/Datasets/Baremaps/tiles.mbtiles --tileset tileset.js --style style.js" />
+ <option name="PROGRAM_PARAMETERS" value="map mbtiles --mbtiles $USER_HOME$/Datasets/Baremaps/tiles.mbtiles --tilejson tileset.js --style style.js" />
<option name="WORKING_DIRECTORY" value="$PROJECT_DIR$/basemap" />
<extension name="coverage">
<pattern>
diff --git a/baremaps-benchmark/src/main/java/org/apache/baremaps/benchmarks/MBTilesBenchmark.java b/baremaps-benchmark/src/main/java/org/apache/baremaps/benchmarks/MBTilesBenchmark.java
index 3754ae6f..d8021f7f 100644
--- a/baremaps-benchmark/src/main/java/org/apache/baremaps/benchmarks/MBTilesBenchmark.java
+++ b/baremaps-benchmark/src/main/java/org/apache/baremaps/benchmarks/MBTilesBenchmark.java
@@ -35,7 +35,7 @@ import org.openjdk.jmh.runner.options.OptionsBuilder;
@Fork(value = 1, warmups = 1)
public class MBTilesBenchmark {
- public SecureRandom random = new SecureRandom();
+ public static SecureRandom random = new SecureRandom();
@Param({"10", "100", "1000"})
public int iterations;
@@ -47,7 +47,7 @@ public class MBTilesBenchmark {
@Setup
public void setup() throws IOException, TileStoreException {
file = Files.createTempFile(Paths.get("."), "baremaps", ".mbtiles");
- mbTilesStore = new MBTilesStore(SqliteUtils.createDataSource(file, false));
+ mbTilesStore = new MBTilesStore(SqliteUtils.createDataSource(file));
mbTilesStore.initializeDatabase();
}
@@ -62,7 +62,7 @@ public class MBTilesBenchmark {
for (int i = 0; i < benchmark.iterations; i++) {
var bytes = new byte[1 << 16];
random.nextBytes(bytes);
- mbTilesStore.put(new TileCoord(0, 0, i), ByteBuffer.wrap(bytes));
+ mbTilesStore.write(new TileCoord(0, 0, i), ByteBuffer.wrap(bytes));
}
}
@@ -78,12 +78,12 @@ public class MBTilesBenchmark {
buffers.add(ByteBuffer.wrap(bytes));
if (coords.size() == 100) {
random.nextBytes(bytes);
- mbTilesStore.put(coords, buffers);
+ mbTilesStore.write(coords, buffers);
coords.clear();
buffers.clear();
}
}
- mbTilesStore.put(coords, buffers);
+ mbTilesStore.write(coords, buffers);
coords.clear();
buffers.clear();
}
diff --git a/baremaps-cli/src/main/java/org/apache/baremaps/cli/map/Dev.java b/baremaps-cli/src/main/java/org/apache/baremaps/cli/map/Dev.java
index c77b1444..2c54f526 100644
--- a/baremaps-cli/src/main/java/org/apache/baremaps/cli/map/Dev.java
+++ b/baremaps-cli/src/main/java/org/apache/baremaps/cli/map/Dev.java
@@ -24,12 +24,12 @@ import java.util.function.Supplier;
import org.apache.baremaps.cli.Options;
import org.apache.baremaps.config.ConfigReader;
import org.apache.baremaps.server.*;
+import org.apache.baremaps.server.CorsFilter;
import org.apache.baremaps.tilestore.TileStore;
import org.apache.baremaps.tilestore.postgres.PostgresTileStore;
+import org.apache.baremaps.utils.PostgresUtils;
import org.apache.baremaps.vectortile.style.Style;
import org.apache.baremaps.vectortile.tilejson.TileJSON;
-import org.apache.baremaps.server.CorsFilter;
-import org.apache.baremaps.utils.PostgresUtils;
import org.apache.baremaps.vectortile.tileset.Tileset;
import org.glassfish.hk2.api.TypeLiteral;
import org.glassfish.hk2.utilities.binding.AbstractBinder;
diff --git a/baremaps-cli/src/main/java/org/apache/baremaps/cli/map/MBTiles.java b/baremaps-cli/src/main/java/org/apache/baremaps/cli/map/MBTiles.java
index 67924764..04739ed9 100644
--- a/baremaps-cli/src/main/java/org/apache/baremaps/cli/map/MBTiles.java
+++ b/baremaps-cli/src/main/java/org/apache/baremaps/cli/map/MBTiles.java
@@ -15,19 +15,22 @@ package org.apache.baremaps.cli.map;
import static io.servicetalk.data.jackson.jersey.ServiceTalkJacksonSerializerFeature.newContextResolver;
import static org.apache.baremaps.utils.ObjectMapperUtils.objectMapper;
-import com.fasterxml.jackson.databind.ObjectMapper;
import com.github.benmanes.caffeine.cache.CaffeineSpec;
import io.servicetalk.http.netty.HttpServers;
import io.servicetalk.http.router.jersey.HttpJerseyRouterBuilder;
import java.nio.file.Path;
import java.util.concurrent.Callable;
+import java.util.function.Supplier;
import org.apache.baremaps.cli.Options;
-import org.apache.baremaps.server.CorsFilter;
-import org.apache.baremaps.server.ServerResources;
+import org.apache.baremaps.config.ConfigReader;
+import org.apache.baremaps.server.*;
import org.apache.baremaps.tilestore.TileCache;
import org.apache.baremaps.tilestore.TileStore;
import org.apache.baremaps.tilestore.mbtiles.MBTilesStore;
import org.apache.baremaps.utils.SqliteUtils;
+import org.apache.baremaps.vectortile.style.Style;
+import org.apache.baremaps.vectortile.tilejson.TileJSON;
+import org.glassfish.hk2.api.TypeLiteral;
import org.glassfish.hk2.utilities.binding.AbstractBinder;
import org.glassfish.jersey.server.ResourceConfig;
import org.slf4j.Logger;
@@ -65,22 +68,40 @@ public class MBTiles implements Callable<Integer> {
@Override
public Integer call() throws Exception {
var objectMapper = objectMapper();
+ var configReader = new ConfigReader();
var caffeineSpec = CaffeineSpec.parse(cache);
- var datasource = SqliteUtils.createDataSource(mbtilesPath, true);
+ var datasource = SqliteUtils.createDataSource(mbtilesPath);
+ var tileStoreSupplierType = new TypeLiteral<Supplier<TileStore>>() {};
var tileStore = new MBTilesStore(datasource);
var tileCache = new TileCache(tileStore, caffeineSpec);
+ var tileStoreSupplier = (Supplier<TileStore>) () -> tileCache;
+
+ var styleSupplierType = new TypeLiteral<Supplier<Style>>() {};
+ var style = objectMapper.readValue(configReader.read(stylePath), Style.class);
+ var styleSupplier = (Supplier<Style>) () -> style;
+
+ var tileJSONSupplierType = new TypeLiteral<Supplier<TileJSON>>() {};
+ var tileJSON = objectMapper.readValue(configReader.read(tileJSONPath), TileJSON.class);
+ var tileJSONSupplier = (Supplier<TileJSON>) () -> tileJSON;
// Configure the application
var application =
- new ResourceConfig().register(CorsFilter.class).register(ServerResources.class)
- .register(newContextResolver(objectMapper)).register(new AbstractBinder() {
+ new ResourceConfig()
+ .register(CorsFilter.class)
+ .register(TileResource.class)
+ .register(StyleResource.class)
+ .register(TilesetResource.class)
+ .register(ClassPathResource.class)
+ .register(newContextResolver(objectMapper))
+ .register(new AbstractBinder() {
@Override
protected void configure() {
- bind(tileJSONPath).to(Path.class).named("tileset");
- bind(stylePath).to(Path.class).named("style");
- bind(tileCache).to(TileStore.class);
- bind(objectMapper).to(ObjectMapper.class);
+ bind("assets").to(String.class).named("directory");
+ bind("server.html").to(String.class).named("index");
+ bind(tileStoreSupplier).to(tileStoreSupplierType);
+ bind(styleSupplier).to(styleSupplierType);
+ bind(tileJSONSupplier).to(tileJSONSupplierType);
}
});
diff --git a/baremaps-cli/src/main/java/org/apache/baremaps/cli/map/Serve.java b/baremaps-cli/src/main/java/org/apache/baremaps/cli/map/Serve.java
index 5c1fa5ad..62978214 100644
--- a/baremaps-cli/src/main/java/org/apache/baremaps/cli/map/Serve.java
+++ b/baremaps-cli/src/main/java/org/apache/baremaps/cli/map/Serve.java
@@ -23,15 +23,14 @@ import java.util.concurrent.Callable;
import java.util.function.Supplier;
import org.apache.baremaps.cli.Options;
import org.apache.baremaps.config.ConfigReader;
-
import org.apache.baremaps.server.*;
+import org.apache.baremaps.server.CorsFilter;
import org.apache.baremaps.tilestore.TileCache;
import org.apache.baremaps.tilestore.TileStore;
import org.apache.baremaps.tilestore.postgres.PostgresTileStore;
+import org.apache.baremaps.utils.PostgresUtils;
import org.apache.baremaps.vectortile.style.Style;
import org.apache.baremaps.vectortile.tilejson.TileJSON;
-import org.apache.baremaps.server.CorsFilter;
-import org.apache.baremaps.utils.PostgresUtils;
import org.apache.baremaps.vectortile.tileset.Tileset;
import org.glassfish.hk2.api.TypeLiteral;
import org.glassfish.hk2.utilities.binding.AbstractBinder;
@@ -71,10 +70,10 @@ public class Serve implements Callable<Integer> {
public Integer call() throws Exception {
var objectMapper = objectMapper();
var configReader = new ConfigReader();
- var tileset = objectMapper.readValue(configReader.read(this.tilesetPath), Tileset.class);
var caffeineSpec = CaffeineSpec.parse(cache);
- var datasource = PostgresUtils.createDataSource(tileset.getDatabase());
+ var tileset = objectMapper.readValue(configReader.read(tilesetPath), Tileset.class);
+ var datasource = PostgresUtils.createDataSource(tileset.getDatabase());
var tileStoreSupplierType = new TypeLiteral<Supplier<TileStore>>() {};
var tileStore = new PostgresTileStore(datasource, tileset);
var tileCache = new TileCache(tileStore, caffeineSpec);
diff --git a/baremaps-core/src/main/java/org/apache/baremaps/tilestore/TileCache.java b/baremaps-core/src/main/java/org/apache/baremaps/tilestore/TileCache.java
index 05893add..d6b2fa33 100644
--- a/baremaps-core/src/main/java/org/apache/baremaps/tilestore/TileCache.java
+++ b/baremaps-core/src/main/java/org/apache/baremaps/tilestore/TileCache.java
@@ -50,10 +50,22 @@ public class TileCache implements TileStore {
/** {@inheritDoc} */
@Override
+<<<<<<< HEAD
public ByteBuffer get(TileCoord tileCoord) throws TileStoreException {
var buffer = cache.get(tileCoord, t -> {
try {
return tileStore.get(t);
+=======
+ public ByteBuffer read(TileCoord tileCoord) throws TileStoreException {
+ return cache.get(tileCoord, t -> {
+ try {
+ var buffer = tileStore.read(t);
+ if (buffer == null) {
+ return null;
+ } else {
+ return buffer;
+ }
+>>>>>>> 1d7ffeec (Change the TileStore interface)
} catch (TileStoreException e) {
logger.error("Unable to read the tile.", e);
return null;
@@ -68,8 +80,8 @@ public class TileCache implements TileStore {
/** {@inheritDoc} */
@Override
- public void put(TileCoord tileCoord, ByteBuffer bytes) throws TileStoreException {
- tileStore.put(tileCoord, bytes);
+ public void write(TileCoord tileCoord, ByteBuffer bytes) throws TileStoreException {
+ tileStore.write(tileCoord, bytes);
cache.invalidate(tileCoord);
}
diff --git a/baremaps-core/src/main/java/org/apache/baremaps/tilestore/TileChannel.java b/baremaps-core/src/main/java/org/apache/baremaps/tilestore/TileChannel.java
index e2e92f68..0620f2b4 100644
--- a/baremaps-core/src/main/java/org/apache/baremaps/tilestore/TileChannel.java
+++ b/baremaps-core/src/main/java/org/apache/baremaps/tilestore/TileChannel.java
@@ -53,9 +53,9 @@ public class TileChannel implements Consumer<TileCoord> {
@Override
public void accept(TileCoord tileCoord) {
try {
- ByteBuffer blob = source.get(tileCoord);
+ ByteBuffer blob = source.read(tileCoord);
if (blob != null) {
- target.put(tileCoord, blob);
+ target.write(tileCoord, blob);
} else if (deleteEmptyTiles) {
target.delete(tileCoord);
}
diff --git a/baremaps-core/src/main/java/org/apache/baremaps/tilestore/TileStore.java b/baremaps-core/src/main/java/org/apache/baremaps/tilestore/TileStore.java
index dc5c3609..926fe088 100644
--- a/baremaps-core/src/main/java/org/apache/baremaps/tilestore/TileStore.java
+++ b/baremaps-core/src/main/java/org/apache/baremaps/tilestore/TileStore.java
@@ -22,48 +22,48 @@ import java.util.List;
public interface TileStore {
/**
- * Gets the content of a tile.
+ * Reads the content of a tile.
*
* @param tileCoord the tile coordinate
* @return the content of the tile
* @throws TileStoreException
*/
- ByteBuffer get(TileCoord tileCoord) throws TileStoreException;
+ ByteBuffer read(TileCoord tileCoord) throws TileStoreException;
/**
- * Gets the content of several tiles.
+ * Reads the content of several tiles.
*
* @param tileCoords the tile coordinates
* @return the content of the tiles
* @throws TileStoreException
*/
- default List<ByteBuffer> get(List<TileCoord> tileCoords) throws TileStoreException {
+ default List<ByteBuffer> read(List<TileCoord> tileCoords) throws TileStoreException {
var blobs = new ArrayList<ByteBuffer>(tileCoords.size());
for (var tileCoord : tileCoords) {
- blobs.add(get(tileCoord));
+ blobs.add(read(tileCoord));
}
return blobs;
}
/**
- * Puts the content of a tile.
+ * Writes the content of a tile.
*
* @param tileCoord the tile coordinate
* @param blob the content of the tile
* @throws TileStoreException
*/
- void put(TileCoord tileCoord, ByteBuffer blob) throws TileStoreException;
+ void write(TileCoord tileCoord, ByteBuffer blob) throws TileStoreException;
/**
- * Puts the content of several tiles.
+ * Writes the content of several tiles.
*
* @param tileCoords the tile coordinates
* @param blobs the content of the tiles
* @throws TileStoreException
*/
- default void put(List<TileCoord> tileCoords, List<ByteBuffer> blobs) throws TileStoreException {
+ default void write(List<TileCoord> tileCoords, List<ByteBuffer> blobs) throws TileStoreException {
for (int i = 0; i < tileCoords.size(); i++) {
- put(tileCoords.get(i), blobs.get(i));
+ write(tileCoords.get(i), blobs.get(i));
}
}
diff --git a/baremaps-core/src/main/java/org/apache/baremaps/tilestore/file/FileTileStore.java b/baremaps-core/src/main/java/org/apache/baremaps/tilestore/file/FileTileStore.java
index 8c254754..77424004 100644
--- a/baremaps-core/src/main/java/org/apache/baremaps/tilestore/file/FileTileStore.java
+++ b/baremaps-core/src/main/java/org/apache/baremaps/tilestore/file/FileTileStore.java
@@ -38,7 +38,7 @@ public class FileTileStore implements TileStore {
/** {@inheritDoc} */
@Override
- public ByteBuffer get(TileCoord tileCoord) throws TileStoreException {
+ public ByteBuffer read(TileCoord tileCoord) throws TileStoreException {
try {
return ByteBuffer.wrap(Files.readAllBytes(resolve(tileCoord)));
} catch (IOException e) {
@@ -48,7 +48,7 @@ public class FileTileStore implements TileStore {
/** {@inheritDoc} */
@Override
- public void put(TileCoord tileCoord, ByteBuffer blob) throws TileStoreException {
+ public void write(TileCoord tileCoord, ByteBuffer blob) throws TileStoreException {
try {
var file = resolve(tileCoord);
Files.createDirectories(file.getParent());
diff --git a/baremaps-core/src/main/java/org/apache/baremaps/tilestore/mbtiles/MBTilesStore.java b/baremaps-core/src/main/java/org/apache/baremaps/tilestore/mbtiles/MBTilesStore.java
index 9539d9a5..847a2c9a 100644
--- a/baremaps-core/src/main/java/org/apache/baremaps/tilestore/mbtiles/MBTilesStore.java
+++ b/baremaps-core/src/main/java/org/apache/baremaps/tilestore/mbtiles/MBTilesStore.java
@@ -73,7 +73,7 @@ public class MBTilesStore implements TileStore {
/** {@inheritDoc} */
@Override
- public ByteBuffer get(TileCoord tileCoord) throws TileStoreException {
+ public ByteBuffer read(TileCoord tileCoord) throws TileStoreException {
try (Connection connection = dataSource.getConnection();
PreparedStatement statement = connection.prepareStatement(SELECT_TILE)) {
statement.setInt(1, tileCoord.z());
@@ -93,7 +93,7 @@ public class MBTilesStore implements TileStore {
/** {@inheritDoc} */
@Override
- public void put(TileCoord tileCoord, ByteBuffer blob) throws TileStoreException {
+ public void write(TileCoord tileCoord, ByteBuffer blob) throws TileStoreException {
try (Connection connection = dataSource.getConnection();
PreparedStatement statement = connection.prepareStatement(INSERT_TILE)) {
statement.setInt(1, tileCoord.z());
@@ -108,7 +108,7 @@ public class MBTilesStore implements TileStore {
/** {@inheritDoc} */
@Override
- public void put(List<TileCoord> tileCoords, List<ByteBuffer> blobs) throws TileStoreException {
+ public void write(List<TileCoord> tileCoords, List<ByteBuffer> blobs) throws TileStoreException {
try (Connection connection = dataSource.getConnection();
PreparedStatement statement = connection.prepareStatement(INSERT_TILE)) {
for (int i = 0; i < tileCoords.size(); i++) {
diff --git a/baremaps-core/src/main/java/org/apache/baremaps/tilestore/postgres/PostgresTileStore.java b/baremaps-core/src/main/java/org/apache/baremaps/tilestore/postgres/PostgresTileStore.java
index af8796f2..8f531921 100644
--- a/baremaps-core/src/main/java/org/apache/baremaps/tilestore/postgres/PostgresTileStore.java
+++ b/baremaps-core/src/main/java/org/apache/baremaps/tilestore/postgres/PostgresTileStore.java
@@ -99,7 +99,7 @@ public class PostgresTileStore implements TileStore {
/** {@inheritDoc} */
@Override
- public ByteBuffer get(TileCoord tileCoord) throws TileStoreException {
+ public ByteBuffer read(TileCoord tileCoord) throws TileStoreException {
try (Connection connection = datasource.getConnection();
Statement statement = connection.createStatement();
ByteArrayOutputStream data = new ByteArrayOutputStream()) {
@@ -240,7 +240,7 @@ public class PostgresTileStore implements TileStore {
/** This operation is not supported. */
@Override
- public void put(TileCoord tileCoord, ByteBuffer blob) {
+ public void write(TileCoord tileCoord, ByteBuffer blob) {
throw new UnsupportedOperationException("The postgis tile store is read only");
}
diff --git a/baremaps-core/src/main/java/org/apache/baremaps/utils/SqliteUtils.java b/baremaps-core/src/main/java/org/apache/baremaps/utils/SqliteUtils.java
index 1ca6baaa..f6ea9051 100644
--- a/baremaps-core/src/main/java/org/apache/baremaps/utils/SqliteUtils.java
+++ b/baremaps-core/src/main/java/org/apache/baremaps/utils/SqliteUtils.java
@@ -41,10 +41,9 @@ public final class SqliteUtils {
* Create a SQLite data source.
*
* @param path the path to the SQLite database
- * @param readOnly
* @return the SQLite data source
*/
- public static HikariDataSource createDataSource(Path path, boolean readOnly) {
+ public static HikariDataSource createDataSource(Path path) {
var sqliteConfig = new SQLiteConfig();
sqliteConfig.setCacheSize(1000000);
sqliteConfig.setPageSize(65536);
@@ -52,7 +51,6 @@ public final class SqliteUtils {
sqliteConfig.setLockingMode(LockingMode.EXCLUSIVE);
sqliteConfig.setSynchronous(SynchronousMode.OFF);
sqliteConfig.setTempStore(TempStore.MEMORY);
- sqliteConfig.setReadOnly(readOnly);
var sqliteDataSource = new SQLiteDataSource();
sqliteDataSource.setConfig(sqliteConfig);
diff --git a/baremaps-core/src/main/java/org/apache/baremaps/workflow/tasks/ExportVectorTiles.java b/baremaps-core/src/main/java/org/apache/baremaps/workflow/tasks/ExportVectorTiles.java
index b33377e5..5f3f4910 100644
--- a/baremaps-core/src/main/java/org/apache/baremaps/workflow/tasks/ExportVectorTiles.java
+++ b/baremaps-core/src/main/java/org/apache/baremaps/workflow/tasks/ExportVectorTiles.java
@@ -79,7 +79,7 @@ public record ExportVectorTiles(
private TileStore targetTileStore(Tileset source) throws TileStoreException, IOException {
if (mbtiles) {
Files.deleteIfExists(repository);
- var dataSource = SqliteUtils.createDataSource(repository, false);
+ var dataSource = SqliteUtils.createDataSource(repository);
var tilesStore = new MBTilesStore(dataSource);
tilesStore.initializeDatabase();
tilesStore.writeMetadata(metadata(source));
diff --git a/baremaps-core/src/test/java/org/apache/baremaps/tilestore/TileStoreTest.java b/baremaps-core/src/test/java/org/apache/baremaps/tilestore/TileStoreTest.java
index 7f3ea965..b1be4229 100644
--- a/baremaps-core/src/test/java/org/apache/baremaps/tilestore/TileStoreTest.java
+++ b/baremaps-core/src/test/java/org/apache/baremaps/tilestore/TileStoreTest.java
@@ -31,15 +31,15 @@ public abstract class TileStoreTest {
ByteBuffer blob = ByteBuffer.wrap("tile_content".getBytes());
// Write data
- tileStore.put(tileCoord, blob);
+ tileStore.write(tileCoord, blob);
// Read the data
- ByteBuffer inputStream = tileStore.get(tileCoord);
+ ByteBuffer inputStream = tileStore.read(tileCoord);
assertArrayEquals(blob.array(), inputStream.array());
// Delete the data
tileStore.delete(tileCoord);
- assertThrows(TileStoreException.class, () -> tileStore.get(tileCoord));
+ assertThrows(TileStoreException.class, () -> tileStore.read(tileCoord));
}
public abstract TileStore createTileStore() throws Exception;
diff --git a/baremaps-core/src/test/java/org/apache/baremaps/tilestore/mbtiles/MBTilesStoreTest.java b/baremaps-core/src/test/java/org/apache/baremaps/tilestore/mbtiles/MBTilesStoreTest.java
index f28154b4..12abdf82 100644
--- a/baremaps-core/src/test/java/org/apache/baremaps/tilestore/mbtiles/MBTilesStoreTest.java
+++ b/baremaps-core/src/test/java/org/apache/baremaps/tilestore/mbtiles/MBTilesStoreTest.java
@@ -13,7 +13,6 @@
package org.apache.baremaps.tilestore.mbtiles;
import static org.junit.jupiter.api.Assertions.assertEquals;
-import static org.junit.jupiter.api.Assertions.assertTrue;
import java.io.IOException;
import java.nio.file.Files;
@@ -56,14 +55,14 @@ class MBTilesStoreTest extends TileStoreTest {
void readWriteMetadata() throws Exception {
MBTilesStore tileStore = createTileStore();
Map<String, String> metadata = tileStore.readMetadata();
- assertTrue(metadata.size() == 0);
+ assertEquals(0, metadata.size());
Map<String, String> m1 = new HashMap<>();
m1.put("test", "test");
tileStore.writeMetadata(m1);
Map<String, String> m2 = tileStore.readMetadata();
- assertTrue(m2.size() == 1);
- assertEquals(m2.get("test"), "test");
+ assertEquals(1, m2.size());
+ assertEquals("test", m2.get("test"));
}
}
diff --git a/baremaps-ogcapi/src/main/java/org/apache/baremaps/ogcapi/TilesResource.java b/baremaps-ogcapi/src/main/java/org/apache/baremaps/ogcapi/TilesResource.java
index 3200996a..75d2add6 100644
--- a/baremaps-ogcapi/src/main/java/org/apache/baremaps/ogcapi/TilesResource.java
+++ b/baremaps-ogcapi/src/main/java/org/apache/baremaps/ogcapi/TilesResource.java
@@ -100,7 +100,7 @@ public class TilesResource implements TilesApi {
int y = tileCol;
TileCoord tileCoord = new TileCoord(x, y, z);
try {
- ByteBuffer blob = tileStore.get(tileCoord);
+ ByteBuffer blob = tileStore.read(tileCoord);
if (blob != null) {
return Response.status(200) // lgtm [java/xss]
.header(ACCESS_CONTROL_ALLOW_ORIGIN, "*")
[incubator-baremaps] 05/09: Format code
Posted by bc...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
bchapuis pushed a commit to branch 688-export
in repository https://gitbox.apache.org/repos/asf/incubator-baremaps.git
commit 03d84577f4d2f72714a38235076ea2702ae6e352
Author: Bertil Chapuis <bc...@gmail.com>
AuthorDate: Wed Jun 14 00:01:16 2023 +0200
Format code
---
.../baremaps/benchmarks/MBTilesBenchmark.java | 85 +++++++++++-----------
.../apache/baremaps/tilestore/mbtiles/MBTiles.java | 2 +-
.../baremaps/workflow/tasks/ExportVectorTiles.java | 1 +
3 files changed, 44 insertions(+), 44 deletions(-)
diff --git a/baremaps-benchmark/src/main/java/org/apache/baremaps/benchmarks/MBTilesBenchmark.java b/baremaps-benchmark/src/main/java/org/apache/baremaps/benchmarks/MBTilesBenchmark.java
index 9dd817a9..b529eb66 100644
--- a/baremaps-benchmark/src/main/java/org/apache/baremaps/benchmarks/MBTilesBenchmark.java
+++ b/baremaps-benchmark/src/main/java/org/apache/baremaps/benchmarks/MBTilesBenchmark.java
@@ -18,7 +18,6 @@ import java.nio.ByteBuffer;
import java.util.ArrayList;
import java.util.Random;
import java.util.concurrent.TimeUnit;
-
import org.apache.baremaps.tilestore.TileCoord;
import org.apache.baremaps.tilestore.TileStoreException;
import org.apache.baremaps.tilestore.mbtiles.MBTiles;
@@ -34,59 +33,59 @@ import org.openjdk.jmh.runner.options.OptionsBuilder;
@Fork(value = 1, warmups = 1)
public class MBTilesBenchmark {
- public Random random = new Random(0);
+ public Random random = new Random(0);
- @Param({"10", "100", "1000"})
- public int iterations;
+ @Param({"10", "100", "1000"})
+ public int iterations;
- private MBTiles mbTiles;
+ private MBTiles mbTiles;
- @Setup
- public void setup() throws IOException, TileStoreException {
- var sqliteFile = File.createTempFile("baremaps", ".sqlite");
- sqliteFile.deleteOnExit();
- var sqliteDataSource = ExportVectorTiles.createDataSource(sqliteFile.toPath());
- mbTiles = new MBTiles(sqliteDataSource);
- mbTiles.initializeDatabase();
- }
+ @Setup
+ public void setup() throws IOException, TileStoreException {
+ var sqliteFile = File.createTempFile("baremaps", ".sqlite");
+ sqliteFile.deleteOnExit();
+ var sqliteDataSource = ExportVectorTiles.createDataSource(sqliteFile.toPath());
+ mbTiles = new MBTiles(sqliteDataSource);
+ mbTiles.initializeDatabase();
+ }
- @Benchmark
- @BenchmarkMode(Mode.SingleShotTime)
- public void writeMBTiles(MBTilesBenchmark benchmark) throws TileStoreException {
- for (int i = 0; i < benchmark.iterations; i++) {
- var bytes = new byte[1 << 16];
- random.nextBytes(bytes);
- mbTiles.put(new TileCoord(0, 0, i), ByteBuffer.wrap(bytes));
- }
+ @Benchmark
+ @BenchmarkMode(Mode.SingleShotTime)
+ public void writeMBTiles(MBTilesBenchmark benchmark) throws TileStoreException {
+ for (int i = 0; i < benchmark.iterations; i++) {
+ var bytes = new byte[1 << 16];
+ random.nextBytes(bytes);
+ mbTiles.put(new TileCoord(0, 0, i), ByteBuffer.wrap(bytes));
}
+ }
- @Benchmark
- @BenchmarkMode(Mode.SingleShotTime)
- public void writeMBTilesBatch(MBTilesBenchmark benchmark) throws TileStoreException {
- var coords = new ArrayList<TileCoord>();
- var buffers = new ArrayList<ByteBuffer>();
- for (int i = 0; i < benchmark.iterations; i++) {
- var bytes = new byte[1 << 16];
- random.nextBytes(bytes);
- coords.add(new TileCoord(0, 0, i));
- buffers.add(ByteBuffer.wrap(bytes));
- if (coords.size() == 100) {
- random.nextBytes(bytes);
- mbTiles.put(coords, buffers);
- coords.clear();
- buffers.clear();
- }
- }
+ @Benchmark
+ @BenchmarkMode(Mode.SingleShotTime)
+ public void writeMBTilesBatch(MBTilesBenchmark benchmark) throws TileStoreException {
+ var coords = new ArrayList<TileCoord>();
+ var buffers = new ArrayList<ByteBuffer>();
+ for (int i = 0; i < benchmark.iterations; i++) {
+ var bytes = new byte[1 << 16];
+ random.nextBytes(bytes);
+ coords.add(new TileCoord(0, 0, i));
+ buffers.add(ByteBuffer.wrap(bytes));
+ if (coords.size() == 100) {
+ random.nextBytes(bytes);
mbTiles.put(coords, buffers);
coords.clear();
buffers.clear();
+ }
}
+ mbTiles.put(coords, buffers);
+ coords.clear();
+ buffers.clear();
+ }
- public static void main(String[] args) throws RunnerException {
- Options opt =
- new OptionsBuilder().include(MBTilesBenchmark.class.getSimpleName()).forks(1).build();
- new Runner(opt).run();
- }
+ public static void main(String[] args) throws RunnerException {
+ Options opt =
+ new OptionsBuilder().include(MBTilesBenchmark.class.getSimpleName()).forks(1).build();
+ new Runner(opt).run();
+ }
}
diff --git a/baremaps-core/src/main/java/org/apache/baremaps/tilestore/mbtiles/MBTiles.java b/baremaps-core/src/main/java/org/apache/baremaps/tilestore/mbtiles/MBTiles.java
index fa1fb9d7..0344aed5 100644
--- a/baremaps-core/src/main/java/org/apache/baremaps/tilestore/mbtiles/MBTiles.java
+++ b/baremaps-core/src/main/java/org/apache/baremaps/tilestore/mbtiles/MBTiles.java
@@ -110,7 +110,7 @@ public class MBTiles implements TileStore {
@Override
public void put(List<TileCoord> tileCoords, List<ByteBuffer> blobs) throws TileStoreException {
try (Connection connection = dataSource.getConnection();
- PreparedStatement statement = connection.prepareStatement(INSERT_TILE)) {
+ PreparedStatement statement = connection.prepareStatement(INSERT_TILE)) {
for (int i = 0; i < tileCoords.size(); i++) {
TileCoord tileCoord = tileCoords.get(i);
ByteBuffer blob = blobs.get(i);
diff --git a/baremaps-core/src/main/java/org/apache/baremaps/workflow/tasks/ExportVectorTiles.java b/baremaps-core/src/main/java/org/apache/baremaps/workflow/tasks/ExportVectorTiles.java
index 435556ea..5cca2bec 100644
--- a/baremaps-core/src/main/java/org/apache/baremaps/workflow/tasks/ExportVectorTiles.java
+++ b/baremaps-core/src/main/java/org/apache/baremaps/workflow/tasks/ExportVectorTiles.java
@@ -142,6 +142,7 @@ public record ExportVectorTiles(
/**
* Create a SQLite data source.
+ *
* @param path the path to the SQLite database
* @return the SQLite data source
*/