You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@baremaps.apache.org by bc...@apache.org on 2023/06/19 11:14:25 UTC
[incubator-baremaps] branch 688-export updated (370ef2e8 -> 710c39fd)
This is an automated email from the ASF dual-hosted git repository.
bchapuis pushed a change to branch 688-export
in repository https://gitbox.apache.org/repos/asf/incubator-baremaps.git
omit 370ef2e8 Fix minor issues
omit 3253dbc2 Refactor utility classes
omit 1dda130b Fix CodeQL and Sonar warnings
omit f5235a8c Add mbtiles command
omit 06fe4110 Format code
omit fa341328 Add batch insert in mbtiles
omit 7cf110ef Add MBTiles benchmark and refactor TileStore
omit 42e3c83d Increase sqlite page size
omit 93f697b5 Fix null pointer exception related to bounds
add d4887bdb Improve zip expansion (#700)
add c5ee7a2e Add support for themes (#691)
add 501a8e8d Fix: Set default theme to default instead of light
add 812da8e2 Improve leisure layering with natural and landuse
add c0b9e920 Refactor the basemap and introduce new themes (#706)
add 4ad3f585 Server refactoring (#705)
new 09734a95 Fix null pointer exception related to bounds
new 0106bb16 Increase sqlite page size
new dbd39e20 Add MBTiles benchmark and refactor TileStore
new c79a3461 Add batch insert in mbtiles
new a6da66f2 Format code
new 24985557 Add mbtiles command
new ed7933a4 Fix CodeQL and Sonar warnings
new 450804d1 Refactor utility classes
new 710c39fd Fix minor issues
This update added new revisions after undoing existing revisions.
That is to say, some revisions that were in the old version of the
branch are not in the new version. This situation occurs
when a user --force pushes a change and generates a repository
containing something like this:
* -- * -- B -- O -- O -- O (370ef2e8)
\
N -- N -- N refs/heads/688-export (710c39fd)
You should already have received notification emails for all of the O
revisions, and so the following emails describe only the N revisions
from the common base, B.
Any revisions marked "omit" are not gone; other references still
refer to them. Any revisions marked "discard" are gone forever.
The 9 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
.../{basemap-dev.run.xml => basemap-serve.run.xml} | 4 +-
.../baremaps/benchmarks/MBTilesBenchmark.java | 2 +-
.../org/apache/baremaps/cli/geocoder/Serve.java | 14 +-
.../java/org/apache/baremaps/cli/iploc/Serve.java | 12 +-
.../main/java/org/apache/baremaps/cli/map/Dev.java | 101 +++-
.../java/org/apache/baremaps/cli/map/MBTiles.java | 50 +-
.../java/org/apache/baremaps/cli/map/Serve.java | 45 +-
.../apache/baremaps/workflow/tasks/UnzipFile.java | 13 +-
.../tilestore/mbtiles/MBTilesStoreTest.java | 7 +-
.../baremaps/server/BaseUriRequestFilter.java | 3 +
.../org/apache/baremaps/server/ChangeResource.java | 135 +++++
.../apache/baremaps/server/ClassPathResource.java | 69 +++
.../org/apache/baremaps/server/CorsFilter.java | 5 +-
.../org/apache/baremaps/server/DevResources.java | 167 ------
.../apache/baremaps/server/DirectoryWatcher.java | 64 ---
...eocoderResources.java => GeocoderResource.java} | 24 +-
.../{IpLocResources.java => IpLocResource.java} | 27 +-
.../apache/baremaps/server/ServerResources.java | 113 ----
.../org/apache/baremaps/server/StyleResource.java | 43 ++
.../org/apache/baremaps/server/TileResource.java | 70 +++
.../apache/baremaps/server/TilesetResource.java | 44 ++
....java => ClassPathResourceIntegrationTest.java} | 37 +-
basemap/layers/aerialway/style.js | 4 +-
basemap/layers/aeroway/line.js | 7 +-
basemap/layers/aeroway/polygon.js | 3 +-
basemap/layers/amenity/background.js | 13 +-
basemap/layers/amenity/fountain.js | 5 +-
basemap/layers/amenity/overlay.js | 12 +-
basemap/layers/attraction/style.js | 4 +-
basemap/layers/background/style.js | 21 +-
basemap/layers/barrier/style.js | 4 +-
basemap/layers/boundary/line.js | 11 +-
basemap/layers/building/number.js | 6 +-
basemap/layers/building/shape.js | 7 +-
basemap/layers/building/tileset.js | 2 +
basemap/layers/highway/bridge_line.js | 25 +-
basemap/layers/highway/bridge_outline.js | 21 +-
basemap/layers/highway/construction_dash.js | 147 ++---
basemap/layers/highway/construction_line.js | 153 +++---
basemap/layers/highway/highway_dash.js | 13 +-
basemap/layers/highway/highway_label.js | 6 +-
basemap/layers/highway/highway_line.js | 25 +-
basemap/layers/highway/highway_outline.js | 27 +-
basemap/layers/highway/pedestrian_area.js | 4 +-
basemap/layers/highway/tileset.js | 1 +
basemap/layers/highway/tunnel_line.js | 25 +-
basemap/layers/highway/tunnel_outline.js | 21 +-
basemap/layers/labels/style.js | 210 +++----
basemap/layers/landuse/background.js | 53 +-
basemap/layers/landuse/overlay.js | 13 +-
basemap/layers/leisure/background.js | 48 +-
basemap/layers/leisure/nature_reserve.js | 5 +-
basemap/layers/leisure/overlay.js | 42 +-
basemap/layers/man_made/bridge.js | 3 +-
basemap/layers/man_made/pier_label.js | 4 +-
basemap/layers/man_made/pier_line.js | 4 +-
basemap/layers/natural/background.js | 22 +-
basemap/layers/natural/overlay.js | 27 +-
basemap/layers/natural/tree.js | 4 +-
basemap/layers/natural/trunk.js | 4 +-
basemap/layers/natural/water.js | 4 +-
basemap/layers/ocean/overlay.js | 3 +-
basemap/layers/point/country_label.js | 5 +-
basemap/layers/point/icon.js | 565 ++++++++++---------
basemap/layers/point/point_label.js | 15 +-
basemap/layers/power/background.js | 7 +-
basemap/layers/power/cable.js | 5 +-
basemap/layers/power/tower.js | 5 +-
basemap/layers/railway/line.js | 28 +-
basemap/layers/railway/tunnel.js | 3 +-
basemap/layers/route/style.js | 4 +-
basemap/layers/tourism/style_zoo_fill.js | 7 +-
basemap/layers/tourism/style_zoo_line.js | 7 +-
basemap/layers/waterway/label.js | 7 +-
basemap/layers/waterway/line.js | 7 +-
basemap/layers/waterway/tunnel_casing.js | 7 +-
basemap/layers/waterway/tunnel_line.js | 7 +-
basemap/style.js | 4 +-
.../metadata.json | 2 +-
.../metadata.json | 2 +-
.../tests/integration/park3-london/metadata.json | 7 +
basemap/theme.js | 3 +
basemap/themes/achromatomaly.js | 13 +
basemap/themes/achromatopsia.js | 13 +
basemap/themes/contrast.js | 13 +
basemap/themes/dark.js | 13 +
basemap/themes/default.js | 609 +++++++++++++++++++++
basemap/themes/deuteranomaly.js | 13 +
basemap/themes/deuteranopia.js | 13 +
basemap/themes/grayscale.js | 13 +
basemap/themes/light.js | 13 +
basemap/themes/protanomaly.js | 13 +
basemap/themes/protanopia.js | 13 +
basemap/themes/transportation.js | 20 +
basemap/themes/tritanomaly.js | 13 +
basemap/themes/tritanopia.js | 13 +
basemap/tileset.js | 4 +-
basemap/utils/color.js | 336 ++++++++++--
98 files changed, 2560 insertions(+), 1324 deletions(-)
copy .run/{basemap-dev.run.xml => basemap-serve.run.xml} (71%)
create mode 100644 baremaps-server/src/main/java/org/apache/baremaps/server/ChangeResource.java
create mode 100644 baremaps-server/src/main/java/org/apache/baremaps/server/ClassPathResource.java
delete mode 100644 baremaps-server/src/main/java/org/apache/baremaps/server/DevResources.java
delete mode 100644 baremaps-server/src/main/java/org/apache/baremaps/server/DirectoryWatcher.java
rename baremaps-server/src/main/java/org/apache/baremaps/server/{GeocoderResources.java => GeocoderResource.java} (83%)
rename baremaps-server/src/main/java/org/apache/baremaps/server/{IpLocResources.java => IpLocResource.java} (84%)
delete mode 100644 baremaps-server/src/main/java/org/apache/baremaps/server/ServerResources.java
create mode 100644 baremaps-server/src/main/java/org/apache/baremaps/server/StyleResource.java
create mode 100644 baremaps-server/src/main/java/org/apache/baremaps/server/TileResource.java
create mode 100644 baremaps-server/src/main/java/org/apache/baremaps/server/TilesetResource.java
copy baremaps-server/src/test/java/org/apache/baremaps/server/{BaseUriRequestFilterIntegrationTest.java => ClassPathResourceIntegrationTest.java} (51%)
copy basemap/tests/integration/{automotive-testing-papenburg-germany => park1-london}/metadata.json (60%)
copy basemap/tests/integration/{automotive-testing-papenburg-germany => park2-london}/metadata.json (60%)
create mode 100644 basemap/tests/integration/park3-london/metadata.json
create mode 100644 basemap/theme.js
create mode 100644 basemap/themes/achromatomaly.js
create mode 100644 basemap/themes/achromatopsia.js
create mode 100644 basemap/themes/contrast.js
create mode 100644 basemap/themes/dark.js
create mode 100644 basemap/themes/default.js
create mode 100644 basemap/themes/deuteranomaly.js
create mode 100644 basemap/themes/deuteranopia.js
create mode 100644 basemap/themes/grayscale.js
create mode 100644 basemap/themes/light.js
create mode 100644 basemap/themes/protanomaly.js
create mode 100644 basemap/themes/protanopia.js
create mode 100644 basemap/themes/transportation.js
create mode 100644 basemap/themes/tritanomaly.js
create mode 100644 basemap/themes/tritanopia.js
[incubator-baremaps] 03/09: Add MBTiles benchmark and refactor TileStore
Posted by bc...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
bchapuis pushed a commit to branch 688-export
in repository https://gitbox.apache.org/repos/asf/incubator-baremaps.git
commit dbd39e2040d4972c64b9fc071935219570f3f567
Author: Bertil Chapuis <bc...@gmail.com>
AuthorDate: Tue Jun 13 09:56:38 2023 +0200
Add MBTiles benchmark and refactor TileStore
---
.../baremaps/benchmarks/MBTilesBenchmark.java | 68 ++++++++++++++++++++++
.../org/apache/baremaps/tilestore/TileCache.java | 8 +--
.../org/apache/baremaps/tilestore/TileChannel.java | 4 +-
.../org/apache/baremaps/tilestore/TileStore.java | 38 ++++++++++--
.../baremaps/tilestore/file/FileTileStore.java | 4 +-
.../apache/baremaps/tilestore/mbtiles/MBTiles.java | 4 +-
.../tilestore/postgres/PostgresTileStore.java | 4 +-
.../baremaps/workflow/tasks/ExportVectorTiles.java | 28 +++++----
.../apache/baremaps/tilestore/TileStoreTest.java | 6 +-
.../org/apache/baremaps/ogcapi/TilesResource.java | 2 +-
10 files changed, 135 insertions(+), 31 deletions(-)
diff --git a/baremaps-benchmark/src/main/java/org/apache/baremaps/benchmarks/MBTilesBenchmark.java b/baremaps-benchmark/src/main/java/org/apache/baremaps/benchmarks/MBTilesBenchmark.java
new file mode 100644
index 00000000..2a71dbe1
--- /dev/null
+++ b/baremaps-benchmark/src/main/java/org/apache/baremaps/benchmarks/MBTilesBenchmark.java
@@ -0,0 +1,68 @@
+/*
+ * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except
+ * in compliance with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software distributed under the License
+ * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
+ * or implied. See the License for the specific language governing permissions and limitations under
+ * the License.
+ */
+
+package org.apache.baremaps.benchmarks;
+
+import java.io.File;
+import java.io.IOException;
+import java.nio.ByteBuffer;
+import java.util.Random;
+import java.util.concurrent.TimeUnit;
+import org.apache.baremaps.tilestore.TileCoord;
+import org.apache.baremaps.tilestore.TileStoreException;
+import org.apache.baremaps.tilestore.mbtiles.MBTiles;
+import org.apache.baremaps.workflow.tasks.ExportVectorTiles;
+import org.openjdk.jmh.annotations.*;
+import org.openjdk.jmh.runner.Runner;
+import org.openjdk.jmh.runner.RunnerException;
+import org.openjdk.jmh.runner.options.Options;
+import org.openjdk.jmh.runner.options.OptionsBuilder;
+
+@State(Scope.Benchmark)
+@OutputTimeUnit(TimeUnit.MILLISECONDS)
+@Fork(value = 1, warmups = 1)
+public class MBTilesBenchmark {
+
+ public Random random = new Random(0);
+
+ @Param({"10", "100", "1000"})
+ public int iterations;
+
+ private MBTiles mbTiles;
+
+ @Setup
+ public void setup() throws IOException, TileStoreException {
+ var sqliteFile = File.createTempFile("baremaps", ".sqlite");
+ sqliteFile.deleteOnExit();
+ var sqliteDataSource = ExportVectorTiles.createDataSource(sqliteFile.toPath());
+ mbTiles = new MBTiles(sqliteDataSource);
+ mbTiles.initializeDatabase();
+ }
+
+ @Benchmark
+ @BenchmarkMode(Mode.SingleShotTime)
+ public void writeMBTiles(MBTilesBenchmark benchmark) throws TileStoreException {
+ for (int i = 0; i < benchmark.iterations; i++) {
+ var bytes = new byte[1 << 16];
+ random.nextBytes(bytes);
+ mbTiles.put(new TileCoord(0, 0, i), ByteBuffer.wrap(bytes));
+ }
+ }
+
+ public static void main(String[] args) throws RunnerException {
+ Options opt =
+ new OptionsBuilder().include(MBTilesBenchmark.class.getSimpleName()).forks(1).build();
+ new Runner(opt).run();
+ }
+
+
+}
diff --git a/baremaps-core/src/main/java/org/apache/baremaps/tilestore/TileCache.java b/baremaps-core/src/main/java/org/apache/baremaps/tilestore/TileCache.java
index 03610767..3d64fd9a 100644
--- a/baremaps-core/src/main/java/org/apache/baremaps/tilestore/TileCache.java
+++ b/baremaps-core/src/main/java/org/apache/baremaps/tilestore/TileCache.java
@@ -50,10 +50,10 @@ public class TileCache implements TileStore {
/** {@inheritDoc} */
@Override
- public ByteBuffer read(TileCoord tileCoord) throws TileStoreException {
+ public ByteBuffer get(TileCoord tileCoord) throws TileStoreException {
return cache.get(tileCoord, t -> {
try {
- var buffer = tileStore.read(t);
+ var buffer = tileStore.get(t);
if (buffer == null) {
return null;
} else {
@@ -68,8 +68,8 @@ public class TileCache implements TileStore {
/** {@inheritDoc} */
@Override
- public void write(TileCoord tileCoord, ByteBuffer bytes) throws TileStoreException {
- tileStore.write(tileCoord, bytes);
+ public void put(TileCoord tileCoord, ByteBuffer bytes) throws TileStoreException {
+ tileStore.put(tileCoord, bytes);
cache.invalidate(tileCoord);
}
diff --git a/baremaps-core/src/main/java/org/apache/baremaps/tilestore/TileChannel.java b/baremaps-core/src/main/java/org/apache/baremaps/tilestore/TileChannel.java
index 0620f2b4..e2e92f68 100644
--- a/baremaps-core/src/main/java/org/apache/baremaps/tilestore/TileChannel.java
+++ b/baremaps-core/src/main/java/org/apache/baremaps/tilestore/TileChannel.java
@@ -53,9 +53,9 @@ public class TileChannel implements Consumer<TileCoord> {
@Override
public void accept(TileCoord tileCoord) {
try {
- ByteBuffer blob = source.read(tileCoord);
+ ByteBuffer blob = source.get(tileCoord);
if (blob != null) {
- target.write(tileCoord, blob);
+ target.put(tileCoord, blob);
} else if (deleteEmptyTiles) {
target.delete(tileCoord);
}
diff --git a/baremaps-core/src/main/java/org/apache/baremaps/tilestore/TileStore.java b/baremaps-core/src/main/java/org/apache/baremaps/tilestore/TileStore.java
index c7288383..dc5c3609 100644
--- a/baremaps-core/src/main/java/org/apache/baremaps/tilestore/TileStore.java
+++ b/baremaps-core/src/main/java/org/apache/baremaps/tilestore/TileStore.java
@@ -15,27 +15,57 @@ package org.apache.baremaps.tilestore;
import java.nio.ByteBuffer;
+import java.util.ArrayList;
+import java.util.List;
/** Represents a store for tiles. */
public interface TileStore {
/**
- * Reads the content of a tile.
+ * Gets the content of a tile.
*
* @param tileCoord the tile coordinate
* @return the content of the tile
* @throws TileStoreException
*/
- ByteBuffer read(TileCoord tileCoord) throws TileStoreException;
+ ByteBuffer get(TileCoord tileCoord) throws TileStoreException;
/**
- * Writes the content of a tile.
+ * Gets the content of several tiles.
+ *
+ * @param tileCoords the tile coordinates
+ * @return the content of the tiles
+ * @throws TileStoreException
+ */
+ default List<ByteBuffer> get(List<TileCoord> tileCoords) throws TileStoreException {
+ var blobs = new ArrayList<ByteBuffer>(tileCoords.size());
+ for (var tileCoord : tileCoords) {
+ blobs.add(get(tileCoord));
+ }
+ return blobs;
+ }
+
+ /**
+ * Puts the content of a tile.
*
* @param tileCoord the tile coordinate
* @param blob the content of the tile
* @throws TileStoreException
*/
- void write(TileCoord tileCoord, ByteBuffer blob) throws TileStoreException;
+ void put(TileCoord tileCoord, ByteBuffer blob) throws TileStoreException;
+
+ /**
+ * Puts the content of several tiles.
+ *
+ * @param tileCoords the tile coordinates
+ * @param blobs the content of the tiles
+ * @throws TileStoreException
+ */
+ default void put(List<TileCoord> tileCoords, List<ByteBuffer> blobs) throws TileStoreException {
+ for (int i = 0; i < tileCoords.size(); i++) {
+ put(tileCoords.get(i), blobs.get(i));
+ }
+ }
/**
* Deletes the content of a tile.
diff --git a/baremaps-core/src/main/java/org/apache/baremaps/tilestore/file/FileTileStore.java b/baremaps-core/src/main/java/org/apache/baremaps/tilestore/file/FileTileStore.java
index 77424004..8c254754 100644
--- a/baremaps-core/src/main/java/org/apache/baremaps/tilestore/file/FileTileStore.java
+++ b/baremaps-core/src/main/java/org/apache/baremaps/tilestore/file/FileTileStore.java
@@ -38,7 +38,7 @@ public class FileTileStore implements TileStore {
/** {@inheritDoc} */
@Override
- public ByteBuffer read(TileCoord tileCoord) throws TileStoreException {
+ public ByteBuffer get(TileCoord tileCoord) throws TileStoreException {
try {
return ByteBuffer.wrap(Files.readAllBytes(resolve(tileCoord)));
} catch (IOException e) {
@@ -48,7 +48,7 @@ public class FileTileStore implements TileStore {
/** {@inheritDoc} */
@Override
- public void write(TileCoord tileCoord, ByteBuffer blob) throws TileStoreException {
+ public void put(TileCoord tileCoord, ByteBuffer blob) throws TileStoreException {
try {
var file = resolve(tileCoord);
Files.createDirectories(file.getParent());
diff --git a/baremaps-core/src/main/java/org/apache/baremaps/tilestore/mbtiles/MBTiles.java b/baremaps-core/src/main/java/org/apache/baremaps/tilestore/mbtiles/MBTiles.java
index 06d0e0cb..1a0a5352 100644
--- a/baremaps-core/src/main/java/org/apache/baremaps/tilestore/mbtiles/MBTiles.java
+++ b/baremaps-core/src/main/java/org/apache/baremaps/tilestore/mbtiles/MBTiles.java
@@ -72,7 +72,7 @@ public class MBTiles implements TileStore {
/** {@inheritDoc} */
@Override
- public ByteBuffer read(TileCoord tileCoord) throws TileStoreException {
+ public ByteBuffer get(TileCoord tileCoord) throws TileStoreException {
try (Connection connection = dataSource.getConnection();
PreparedStatement statement = connection.prepareStatement(SELECT_TILE)) {
statement.setInt(1, tileCoord.z());
@@ -92,7 +92,7 @@ public class MBTiles implements TileStore {
/** {@inheritDoc} */
@Override
- public void write(TileCoord tileCoord, ByteBuffer blob) throws TileStoreException {
+ public void put(TileCoord tileCoord, ByteBuffer blob) throws TileStoreException {
try (Connection connection = dataSource.getConnection();
PreparedStatement statement = connection.prepareStatement(INSERT_TILE)) {
statement.setInt(1, tileCoord.z());
diff --git a/baremaps-core/src/main/java/org/apache/baremaps/tilestore/postgres/PostgresTileStore.java b/baremaps-core/src/main/java/org/apache/baremaps/tilestore/postgres/PostgresTileStore.java
index 1e1eb6c3..e4d4485f 100644
--- a/baremaps-core/src/main/java/org/apache/baremaps/tilestore/postgres/PostgresTileStore.java
+++ b/baremaps-core/src/main/java/org/apache/baremaps/tilestore/postgres/PostgresTileStore.java
@@ -99,7 +99,7 @@ public class PostgresTileStore implements TileStore {
/** {@inheritDoc} */
@Override
- public ByteBuffer read(TileCoord tileCoord) throws TileStoreException {
+ public ByteBuffer get(TileCoord tileCoord) throws TileStoreException {
try (Connection connection = datasource.getConnection();
Statement statement = connection.createStatement();
ByteArrayOutputStream data = new ByteArrayOutputStream()) {
@@ -239,7 +239,7 @@ public class PostgresTileStore implements TileStore {
}
/** This operation is not supported. */
- public void write(TileCoord tileCoord, ByteBuffer blob) {
+ public void put(TileCoord tileCoord, ByteBuffer blob) {
throw new UnsupportedOperationException("The postgis tile store is read only");
}
diff --git a/baremaps-core/src/main/java/org/apache/baremaps/workflow/tasks/ExportVectorTiles.java b/baremaps-core/src/main/java/org/apache/baremaps/workflow/tasks/ExportVectorTiles.java
index 13e49f48..f0b8ac83 100644
--- a/baremaps-core/src/main/java/org/apache/baremaps/workflow/tasks/ExportVectorTiles.java
+++ b/baremaps-core/src/main/java/org/apache/baremaps/workflow/tasks/ExportVectorTiles.java
@@ -87,17 +87,7 @@ public record ExportVectorTiles(
if (mbtiles) {
Files.deleteIfExists(repository);
- var sqliteConfig = new SQLiteConfig();
- sqliteConfig.setCacheSize(1000000);
- sqliteConfig.setPageSize(65536);
- sqliteConfig.setJournalMode(JournalMode.OFF);
- sqliteConfig.setLockingMode(LockingMode.EXCLUSIVE);
- sqliteConfig.setSynchronous(SynchronousMode.OFF);
- sqliteConfig.setTempStore(TempStore.MEMORY);
-
- var sqliteDataSource = new SQLiteDataSource();
- sqliteDataSource.setConfig(sqliteConfig);
- sqliteDataSource.setUrl("jdbc:sqlite:" + repository);
+ var sqliteDataSource = createDataSource(repository);
var hikariConfig = new HikariConfig();
hikariConfig.setDataSource(sqliteDataSource);
@@ -114,6 +104,22 @@ public record ExportVectorTiles(
}
}
+ public static SQLiteDataSource createDataSource(Path path) {
+ var sqliteConfig = new SQLiteConfig();
+ sqliteConfig.setCacheSize(1000000);
+ sqliteConfig.setPageSize(65536);
+ sqliteConfig.setJournalMode(JournalMode.OFF);
+ sqliteConfig.setLockingMode(LockingMode.EXCLUSIVE);
+ sqliteConfig.setSynchronous(SynchronousMode.OFF);
+ sqliteConfig.setTempStore(TempStore.MEMORY);
+
+ var sqliteDataSource = new SQLiteDataSource();
+ sqliteDataSource.setConfig(sqliteConfig);
+ sqliteDataSource.setUrl("jdbc:sqlite:" + path);
+
+ return sqliteDataSource;
+ }
+
private Map<String, String> metadata(Tileset tileset) throws JsonProcessingException {
var metadata = new HashMap<String, String>();
diff --git a/baremaps-core/src/test/java/org/apache/baremaps/tilestore/TileStoreTest.java b/baremaps-core/src/test/java/org/apache/baremaps/tilestore/TileStoreTest.java
index b1be4229..7f3ea965 100644
--- a/baremaps-core/src/test/java/org/apache/baremaps/tilestore/TileStoreTest.java
+++ b/baremaps-core/src/test/java/org/apache/baremaps/tilestore/TileStoreTest.java
@@ -31,15 +31,15 @@ public abstract class TileStoreTest {
ByteBuffer blob = ByteBuffer.wrap("tile_content".getBytes());
// Write data
- tileStore.write(tileCoord, blob);
+ tileStore.put(tileCoord, blob);
// Read the data
- ByteBuffer inputStream = tileStore.read(tileCoord);
+ ByteBuffer inputStream = tileStore.get(tileCoord);
assertArrayEquals(blob.array(), inputStream.array());
// Delete the data
tileStore.delete(tileCoord);
- assertThrows(TileStoreException.class, () -> tileStore.read(tileCoord));
+ assertThrows(TileStoreException.class, () -> tileStore.get(tileCoord));
}
public abstract TileStore createTileStore() throws Exception;
diff --git a/baremaps-ogcapi/src/main/java/org/apache/baremaps/ogcapi/TilesResource.java b/baremaps-ogcapi/src/main/java/org/apache/baremaps/ogcapi/TilesResource.java
index 75d2add6..3200996a 100644
--- a/baremaps-ogcapi/src/main/java/org/apache/baremaps/ogcapi/TilesResource.java
+++ b/baremaps-ogcapi/src/main/java/org/apache/baremaps/ogcapi/TilesResource.java
@@ -100,7 +100,7 @@ public class TilesResource implements TilesApi {
int y = tileCol;
TileCoord tileCoord = new TileCoord(x, y, z);
try {
- ByteBuffer blob = tileStore.read(tileCoord);
+ ByteBuffer blob = tileStore.get(tileCoord);
if (blob != null) {
return Response.status(200) // lgtm [java/xss]
.header(ACCESS_CONTROL_ALLOW_ORIGIN, "*")
[incubator-baremaps] 02/09: Increase sqlite page size
Posted by bc...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
bchapuis pushed a commit to branch 688-export
in repository https://gitbox.apache.org/repos/asf/incubator-baremaps.git
commit 0106bb1639c5fe55241a75a156a51eb39276c8cb
Author: Bertil Chapuis <bc...@gmail.com>
AuthorDate: Mon Jun 12 21:46:00 2023 +0200
Increase sqlite page size
---
.run/openstreetmap-export.run.xml | 2 +-
.../main/java/org/apache/baremaps/workflow/tasks/ExportVectorTiles.java | 1 +
2 files changed, 2 insertions(+), 1 deletion(-)
diff --git a/.run/openstreetmap-export.run.xml b/.run/openstreetmap-export.run.xml
index cb53b2d4..b067f8a0 100644
--- a/.run/openstreetmap-export.run.xml
+++ b/.run/openstreetmap-export.run.xml
@@ -2,7 +2,7 @@
<configuration default="false" name="openstreetmap-export" type="Application" factoryName="Application">
<option name="MAIN_CLASS_NAME" value="org.apache.baremaps.cli.Baremaps" />
<module name="baremaps-cli" />
- <option name="PROGRAM_PARAMETERS" value="map export --tileset tileset.json --repository tiles/" />
+ <option name="PROGRAM_PARAMETERS" value="map export --tileset tileset.json --repository tiles.mbtiles --mbtiles" />
<option name="WORKING_DIRECTORY" value="$PROJECT_DIR$/examples/openstreetmap" />
<extension name="coverage">
<pattern>
diff --git a/baremaps-core/src/main/java/org/apache/baremaps/workflow/tasks/ExportVectorTiles.java b/baremaps-core/src/main/java/org/apache/baremaps/workflow/tasks/ExportVectorTiles.java
index 32246c2d..13e49f48 100644
--- a/baremaps-core/src/main/java/org/apache/baremaps/workflow/tasks/ExportVectorTiles.java
+++ b/baremaps-core/src/main/java/org/apache/baremaps/workflow/tasks/ExportVectorTiles.java
@@ -89,6 +89,7 @@ public record ExportVectorTiles(
var sqliteConfig = new SQLiteConfig();
sqliteConfig.setCacheSize(1000000);
+ sqliteConfig.setPageSize(65536);
sqliteConfig.setJournalMode(JournalMode.OFF);
sqliteConfig.setLockingMode(LockingMode.EXCLUSIVE);
sqliteConfig.setSynchronous(SynchronousMode.OFF);
[incubator-baremaps] 04/09: Add batch insert in mbtiles
Posted by bc...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
bchapuis pushed a commit to branch 688-export
in repository https://gitbox.apache.org/repos/asf/incubator-baremaps.git
commit c79a346103062dd99282c625bfc79d6f5d884ae1
Author: Bertil Chapuis <bc...@gmail.com>
AuthorDate: Tue Jun 13 23:49:52 2023 +0200
Add batch insert in mbtiles
---
.../baremaps/benchmarks/MBTilesBenchmark.java | 74 ++++++++++++++--------
.../apache/baremaps/tilestore/mbtiles/MBTiles.java | 21 ++++++
.../baremaps/workflow/tasks/ExportVectorTiles.java | 52 ++++++++-------
3 files changed, 98 insertions(+), 49 deletions(-)
diff --git a/baremaps-benchmark/src/main/java/org/apache/baremaps/benchmarks/MBTilesBenchmark.java b/baremaps-benchmark/src/main/java/org/apache/baremaps/benchmarks/MBTilesBenchmark.java
index 2a71dbe1..9dd817a9 100644
--- a/baremaps-benchmark/src/main/java/org/apache/baremaps/benchmarks/MBTilesBenchmark.java
+++ b/baremaps-benchmark/src/main/java/org/apache/baremaps/benchmarks/MBTilesBenchmark.java
@@ -15,8 +15,10 @@ package org.apache.baremaps.benchmarks;
import java.io.File;
import java.io.IOException;
import java.nio.ByteBuffer;
+import java.util.ArrayList;
import java.util.Random;
import java.util.concurrent.TimeUnit;
+
import org.apache.baremaps.tilestore.TileCoord;
import org.apache.baremaps.tilestore.TileStoreException;
import org.apache.baremaps.tilestore.mbtiles.MBTiles;
@@ -32,37 +34,59 @@ import org.openjdk.jmh.runner.options.OptionsBuilder;
@Fork(value = 1, warmups = 1)
public class MBTilesBenchmark {
- public Random random = new Random(0);
+ public Random random = new Random(0);
- @Param({"10", "100", "1000"})
- public int iterations;
+ @Param({"10", "100", "1000"})
+ public int iterations;
- private MBTiles mbTiles;
+ private MBTiles mbTiles;
- @Setup
- public void setup() throws IOException, TileStoreException {
- var sqliteFile = File.createTempFile("baremaps", ".sqlite");
- sqliteFile.deleteOnExit();
- var sqliteDataSource = ExportVectorTiles.createDataSource(sqliteFile.toPath());
- mbTiles = new MBTiles(sqliteDataSource);
- mbTiles.initializeDatabase();
- }
+ @Setup
+ public void setup() throws IOException, TileStoreException {
+ var sqliteFile = File.createTempFile("baremaps", ".sqlite");
+ sqliteFile.deleteOnExit();
+ var sqliteDataSource = ExportVectorTiles.createDataSource(sqliteFile.toPath());
+ mbTiles = new MBTiles(sqliteDataSource);
+ mbTiles.initializeDatabase();
+ }
- @Benchmark
- @BenchmarkMode(Mode.SingleShotTime)
- public void writeMBTiles(MBTilesBenchmark benchmark) throws TileStoreException {
- for (int i = 0; i < benchmark.iterations; i++) {
- var bytes = new byte[1 << 16];
- random.nextBytes(bytes);
- mbTiles.put(new TileCoord(0, 0, i), ByteBuffer.wrap(bytes));
+ @Benchmark
+ @BenchmarkMode(Mode.SingleShotTime)
+ public void writeMBTiles(MBTilesBenchmark benchmark) throws TileStoreException {
+ for (int i = 0; i < benchmark.iterations; i++) {
+ var bytes = new byte[1 << 16];
+ random.nextBytes(bytes);
+ mbTiles.put(new TileCoord(0, 0, i), ByteBuffer.wrap(bytes));
+ }
}
- }
- public static void main(String[] args) throws RunnerException {
- Options opt =
- new OptionsBuilder().include(MBTilesBenchmark.class.getSimpleName()).forks(1).build();
- new Runner(opt).run();
- }
+ @Benchmark
+ @BenchmarkMode(Mode.SingleShotTime)
+ public void writeMBTilesBatch(MBTilesBenchmark benchmark) throws TileStoreException {
+ var coords = new ArrayList<TileCoord>();
+ var buffers = new ArrayList<ByteBuffer>();
+ for (int i = 0; i < benchmark.iterations; i++) {
+ var bytes = new byte[1 << 16];
+ random.nextBytes(bytes);
+ coords.add(new TileCoord(0, 0, i));
+ buffers.add(ByteBuffer.wrap(bytes));
+ if (coords.size() == 100) {
+ random.nextBytes(bytes);
+ mbTiles.put(coords, buffers);
+ coords.clear();
+ buffers.clear();
+ }
+ }
+ mbTiles.put(coords, buffers);
+ coords.clear();
+ buffers.clear();
+ }
+
+ public static void main(String[] args) throws RunnerException {
+ Options opt =
+ new OptionsBuilder().include(MBTilesBenchmark.class.getSimpleName()).forks(1).build();
+ new Runner(opt).run();
+ }
}
diff --git a/baremaps-core/src/main/java/org/apache/baremaps/tilestore/mbtiles/MBTiles.java b/baremaps-core/src/main/java/org/apache/baremaps/tilestore/mbtiles/MBTiles.java
index 1a0a5352..fa1fb9d7 100644
--- a/baremaps-core/src/main/java/org/apache/baremaps/tilestore/mbtiles/MBTiles.java
+++ b/baremaps-core/src/main/java/org/apache/baremaps/tilestore/mbtiles/MBTiles.java
@@ -22,6 +22,7 @@ import java.sql.ResultSet;
import java.sql.SQLException;
import java.sql.Statement;
import java.util.HashMap;
+import java.util.List;
import java.util.Map;
import javax.sql.DataSource;
import org.apache.baremaps.tilestore.TileCoord;
@@ -105,6 +106,26 @@ public class MBTiles implements TileStore {
}
}
+ /** {@inheritDoc} */
+ @Override
+ public void put(List<TileCoord> tileCoords, List<ByteBuffer> blobs) throws TileStoreException {
+ try (Connection connection = dataSource.getConnection();
+ PreparedStatement statement = connection.prepareStatement(INSERT_TILE)) {
+ for (int i = 0; i < tileCoords.size(); i++) {
+ TileCoord tileCoord = tileCoords.get(i);
+ ByteBuffer blob = blobs.get(i);
+ statement.setInt(1, tileCoord.z());
+ statement.setInt(2, tileCoord.x());
+ statement.setInt(3, reverseY(tileCoord.y(), tileCoord.z()));
+ statement.setBytes(4, blob.array());
+ statement.addBatch();
+ }
+ statement.executeBatch();
+ } catch (SQLException e) {
+ throw new TileStoreException(e);
+ }
+ }
+
/** {@inheritDoc} */
@Override
public void delete(TileCoord tileCoord) throws TileStoreException {
diff --git a/baremaps-core/src/main/java/org/apache/baremaps/workflow/tasks/ExportVectorTiles.java b/baremaps-core/src/main/java/org/apache/baremaps/workflow/tasks/ExportVectorTiles.java
index f0b8ac83..435556ea 100644
--- a/baremaps-core/src/main/java/org/apache/baremaps/workflow/tasks/ExportVectorTiles.java
+++ b/baremaps-core/src/main/java/org/apache/baremaps/workflow/tasks/ExportVectorTiles.java
@@ -87,14 +87,8 @@ public record ExportVectorTiles(
if (mbtiles) {
Files.deleteIfExists(repository);
- var sqliteDataSource = createDataSource(repository);
-
- var hikariConfig = new HikariConfig();
- hikariConfig.setDataSource(sqliteDataSource);
- hikariConfig.setMaximumPoolSize(1);
- var hikariDataSource = new HikariDataSource(hikariConfig);
-
- var tilesStore = new MBTiles(hikariDataSource);
+ var dataSource = createDataSource(repository);
+ var tilesStore = new MBTiles(dataSource);
tilesStore.initializeDatabase();
tilesStore.writeMetadata(metadata(source));
@@ -104,22 +98,6 @@ public record ExportVectorTiles(
}
}
- public static SQLiteDataSource createDataSource(Path path) {
- var sqliteConfig = new SQLiteConfig();
- sqliteConfig.setCacheSize(1000000);
- sqliteConfig.setPageSize(65536);
- sqliteConfig.setJournalMode(JournalMode.OFF);
- sqliteConfig.setLockingMode(LockingMode.EXCLUSIVE);
- sqliteConfig.setSynchronous(SynchronousMode.OFF);
- sqliteConfig.setTempStore(TempStore.MEMORY);
-
- var sqliteDataSource = new SQLiteDataSource();
- sqliteDataSource.setConfig(sqliteConfig);
- sqliteDataSource.setUrl("jdbc:sqlite:" + path);
-
- return sqliteDataSource;
- }
-
private Map<String, String> metadata(Tileset tileset) throws JsonProcessingException {
var metadata = new HashMap<String, String>();
@@ -161,4 +139,30 @@ public record ExportVectorTiles(
return metadata;
}
+
+ /**
+ * Create a SQLite data source.
+ * @param path the path to the SQLite database
+ * @return the SQLite data source
+ */
+ public static DataSource createDataSource(Path path) {
+ var sqliteConfig = new SQLiteConfig();
+ sqliteConfig.setCacheSize(1000000);
+ sqliteConfig.setPageSize(65536);
+ sqliteConfig.setJournalMode(JournalMode.OFF);
+ sqliteConfig.setLockingMode(LockingMode.EXCLUSIVE);
+ sqliteConfig.setSynchronous(SynchronousMode.OFF);
+ sqliteConfig.setTempStore(TempStore.MEMORY);
+
+ var sqliteDataSource = new SQLiteDataSource();
+ sqliteDataSource.setConfig(sqliteConfig);
+ sqliteDataSource.setUrl("jdbc:sqlite:" + path);
+
+ var hikariConfig = new HikariConfig();
+ hikariConfig.setDataSource(sqliteDataSource);
+ hikariConfig.setMaximumPoolSize(1);
+ var hikariDataSource = new HikariDataSource(hikariConfig);
+
+ return hikariDataSource;
+ }
}
[incubator-baremaps] 07/09: Fix CodeQL and Sonar warnings
Posted by bc...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
bchapuis pushed a commit to branch 688-export
in repository https://gitbox.apache.org/repos/asf/incubator-baremaps.git
commit ed7933a4e7e78d43ff0db5cd38b70b9606b23509
Author: Bertil Chapuis <bc...@gmail.com>
AuthorDate: Sun Jun 18 21:40:03 2023 +0200
Fix CodeQL and Sonar warnings
---
.../baremaps/benchmarks/MBTilesBenchmark.java | 21 ++++++++++++++-------
.../java/org/apache/baremaps/cli/map/MBTiles.java | 18 ++++++------------
.../main/java/org/apache/baremaps/cli/map/Map.java | 3 ++-
.../tilestore/postgres/PostgresTileStore.java | 2 ++
.../baremaps/workflow/tasks/ExportVectorTiles.java | 5 +----
5 files changed, 25 insertions(+), 24 deletions(-)
diff --git a/baremaps-benchmark/src/main/java/org/apache/baremaps/benchmarks/MBTilesBenchmark.java b/baremaps-benchmark/src/main/java/org/apache/baremaps/benchmarks/MBTilesBenchmark.java
index b529eb66..dc333999 100644
--- a/baremaps-benchmark/src/main/java/org/apache/baremaps/benchmarks/MBTilesBenchmark.java
+++ b/baremaps-benchmark/src/main/java/org/apache/baremaps/benchmarks/MBTilesBenchmark.java
@@ -12,11 +12,13 @@
package org.apache.baremaps.benchmarks;
-import java.io.File;
import java.io.IOException;
import java.nio.ByteBuffer;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.Paths;
+import java.security.SecureRandom;
import java.util.ArrayList;
-import java.util.Random;
import java.util.concurrent.TimeUnit;
import org.apache.baremaps.tilestore.TileCoord;
import org.apache.baremaps.tilestore.TileStoreException;
@@ -33,22 +35,27 @@ import org.openjdk.jmh.runner.options.OptionsBuilder;
@Fork(value = 1, warmups = 1)
public class MBTilesBenchmark {
- public Random random = new Random(0);
+ public SecureRandom random = new SecureRandom();
@Param({"10", "100", "1000"})
public int iterations;
+ private Path file;
+
private MBTiles mbTiles;
@Setup
public void setup() throws IOException, TileStoreException {
- var sqliteFile = File.createTempFile("baremaps", ".sqlite");
- sqliteFile.deleteOnExit();
- var sqliteDataSource = ExportVectorTiles.createDataSource(sqliteFile.toPath());
- mbTiles = new MBTiles(sqliteDataSource);
+ file = Files.createTempFile(Paths.get("."), "baremaps", ".mbtiles");
+ mbTiles = new MBTiles(ExportVectorTiles.createDataSource(file));
mbTiles.initializeDatabase();
}
+ @TearDown
+ public void tearDown() throws IOException, TileStoreException {
+ Files.delete(file);
+ }
+
@Benchmark
@BenchmarkMode(Mode.SingleShotTime)
public void writeMBTiles(MBTilesBenchmark benchmark) throws TileStoreException {
diff --git a/baremaps-cli/src/main/java/org/apache/baremaps/cli/map/MBTiles.java b/baremaps-cli/src/main/java/org/apache/baremaps/cli/map/MBTiles.java
index f4478ca9..0f8c7f50 100644
--- a/baremaps-cli/src/main/java/org/apache/baremaps/cli/map/MBTiles.java
+++ b/baremaps-cli/src/main/java/org/apache/baremaps/cli/map/MBTiles.java
@@ -12,19 +12,20 @@
package org.apache.baremaps.cli.map;
+import static io.servicetalk.data.jackson.jersey.ServiceTalkJacksonSerializerFeature.newContextResolver;
+import static org.apache.baremaps.utils.ObjectMapperUtils.objectMapper;
+
import com.fasterxml.jackson.databind.ObjectMapper;
import com.github.benmanes.caffeine.cache.CaffeineSpec;
import io.servicetalk.http.netty.HttpServers;
import io.servicetalk.http.router.jersey.HttpJerseyRouterBuilder;
+import java.nio.file.Path;
+import java.util.concurrent.Callable;
import org.apache.baremaps.cli.Options;
-import org.apache.baremaps.config.ConfigReader;
-import org.apache.baremaps.postgres.PostgresUtils;
import org.apache.baremaps.server.CorsFilter;
import org.apache.baremaps.server.ServerResources;
import org.apache.baremaps.tilestore.TileCache;
import org.apache.baremaps.tilestore.TileStore;
-import org.apache.baremaps.tilestore.postgres.PostgresTileStore;
-import org.apache.baremaps.vectortile.tileset.Tileset;
import org.apache.baremaps.workflow.tasks.ExportVectorTiles;
import org.glassfish.hk2.utilities.binding.AbstractBinder;
import org.glassfish.jersey.server.ResourceConfig;
@@ -34,13 +35,6 @@ import picocli.CommandLine.Command;
import picocli.CommandLine.Mixin;
import picocli.CommandLine.Option;
-import java.nio.file.Path;
-import java.util.concurrent.Callable;
-
-import static io.servicetalk.data.jackson.jersey.ServiceTalkJacksonSerializerFeature.contextResolverFor;
-import static io.servicetalk.data.jackson.jersey.ServiceTalkJacksonSerializerFeature.newContextResolver;
-import static org.apache.baremaps.utils.ObjectMapperUtils.objectMapper;
-
@Command(name = "mbtiles", description = "Start a mbtiles server with caching capabilities.")
public class MBTiles implements Callable<Integer> {
@@ -57,7 +51,7 @@ public class MBTiles implements Callable<Integer> {
private Path mbtiles;
@Option(names = {"--tilejson"}, paramLabel = "TILEJSON", description = "The tileJSON file.",
- required = true)
+ required = true)
private Path tileset;
@Option(names = {"--style"}, paramLabel = "STYLE", description = "The style file.",
diff --git a/baremaps-cli/src/main/java/org/apache/baremaps/cli/map/Map.java b/baremaps-cli/src/main/java/org/apache/baremaps/cli/map/Map.java
index f84dea54..c026d055 100644
--- a/baremaps-cli/src/main/java/org/apache/baremaps/cli/map/Map.java
+++ b/baremaps-cli/src/main/java/org/apache/baremaps/cli/map/Map.java
@@ -18,7 +18,8 @@ import picocli.CommandLine;
import picocli.CommandLine.Command;
@Command(name = "map", description = "Map commands.",
- subcommands = {Init.class, Export.class, Serve.class, Dev.class, MBTiles.class}, sortOptions = false)
+ subcommands = {Init.class, Export.class, Serve.class, Dev.class, MBTiles.class},
+ sortOptions = false)
public class Map implements Runnable {
@Override
diff --git a/baremaps-core/src/main/java/org/apache/baremaps/tilestore/postgres/PostgresTileStore.java b/baremaps-core/src/main/java/org/apache/baremaps/tilestore/postgres/PostgresTileStore.java
index e4d4485f..af8796f2 100644
--- a/baremaps-core/src/main/java/org/apache/baremaps/tilestore/postgres/PostgresTileStore.java
+++ b/baremaps-core/src/main/java/org/apache/baremaps/tilestore/postgres/PostgresTileStore.java
@@ -239,11 +239,13 @@ public class PostgresTileStore implements TileStore {
}
/** This operation is not supported. */
+ @Override
public void put(TileCoord tileCoord, ByteBuffer blob) {
throw new UnsupportedOperationException("The postgis tile store is read only");
}
/** This operation is not supported. */
+ @Override
public void delete(TileCoord tileCoord) {
throw new UnsupportedOperationException("The postgis tile store is read only");
}
diff --git a/baremaps-core/src/main/java/org/apache/baremaps/workflow/tasks/ExportVectorTiles.java b/baremaps-core/src/main/java/org/apache/baremaps/workflow/tasks/ExportVectorTiles.java
index 5cca2bec..43040de8 100644
--- a/baremaps-core/src/main/java/org/apache/baremaps/workflow/tasks/ExportVectorTiles.java
+++ b/baremaps-core/src/main/java/org/apache/baremaps/workflow/tasks/ExportVectorTiles.java
@@ -86,12 +86,10 @@ public record ExportVectorTiles(
private TileStore targetTileStore(Tileset source) throws TileStoreException, IOException {
if (mbtiles) {
Files.deleteIfExists(repository);
-
var dataSource = createDataSource(repository);
var tilesStore = new MBTiles(dataSource);
tilesStore.initializeDatabase();
tilesStore.writeMetadata(metadata(source));
-
return tilesStore;
} else {
return new FileTileStore(repository);
@@ -162,8 +160,7 @@ public record ExportVectorTiles(
var hikariConfig = new HikariConfig();
hikariConfig.setDataSource(sqliteDataSource);
hikariConfig.setMaximumPoolSize(1);
- var hikariDataSource = new HikariDataSource(hikariConfig);
- return hikariDataSource;
+ return new HikariDataSource(hikariConfig);
}
}
[incubator-baremaps] 09/09: Fix minor issues
Posted by bc...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
bchapuis pushed a commit to branch 688-export
in repository https://gitbox.apache.org/repos/asf/incubator-baremaps.git
commit 710c39fd2e950734e0b1371ccd7d96fb0155625c
Author: Bertil Chapuis <bc...@gmail.com>
AuthorDate: Mon Jun 19 12:03:21 2023 +0200
Fix minor issues
---
.run/basemap-mbtiles.run.xml | 2 +-
.../baremaps/benchmarks/MBTilesBenchmark.java | 4 +-
.../main/java/org/apache/baremaps/cli/map/Dev.java | 4 +-
.../java/org/apache/baremaps/cli/map/MBTiles.java | 50 ++++++++++++++++------
.../java/org/apache/baremaps/cli/map/Serve.java | 9 ++--
.../org/apache/baremaps/utils/SqliteUtils.java | 4 +-
.../baremaps/workflow/tasks/ExportVectorTiles.java | 2 +-
.../tilestore/mbtiles/MBTilesStoreTest.java | 7 ++-
.../org/apache/baremaps/server/TileResource.java | 2 +-
9 files changed, 51 insertions(+), 33 deletions(-)
diff --git a/.run/basemap-mbtiles.run.xml b/.run/basemap-mbtiles.run.xml
index 3a08cf9a..6a57530d 100644
--- a/.run/basemap-mbtiles.run.xml
+++ b/.run/basemap-mbtiles.run.xml
@@ -2,7 +2,7 @@
<configuration default="false" name="basemap-mbtiles" type="Application" factoryName="Application">
<option name="MAIN_CLASS_NAME" value="org.apache.baremaps.cli.Baremaps" />
<module name="baremaps-cli" />
- <option name="PROGRAM_PARAMETERS" value="map mbtiles --mbtiles $USER_HOME$/Datasets/Baremaps/tiles.mbtiles --tileset tileset.js --style style.js" />
+ <option name="PROGRAM_PARAMETERS" value="map mbtiles --mbtiles $USER_HOME$/Datasets/Baremaps/tiles.mbtiles --tilejson tileset.js --style style.js" />
<option name="WORKING_DIRECTORY" value="$PROJECT_DIR$/basemap" />
<extension name="coverage">
<pattern>
diff --git a/baremaps-benchmark/src/main/java/org/apache/baremaps/benchmarks/MBTilesBenchmark.java b/baremaps-benchmark/src/main/java/org/apache/baremaps/benchmarks/MBTilesBenchmark.java
index 3754ae6f..75042421 100644
--- a/baremaps-benchmark/src/main/java/org/apache/baremaps/benchmarks/MBTilesBenchmark.java
+++ b/baremaps-benchmark/src/main/java/org/apache/baremaps/benchmarks/MBTilesBenchmark.java
@@ -35,7 +35,7 @@ import org.openjdk.jmh.runner.options.OptionsBuilder;
@Fork(value = 1, warmups = 1)
public class MBTilesBenchmark {
- public SecureRandom random = new SecureRandom();
+ public static SecureRandom random = new SecureRandom();
@Param({"10", "100", "1000"})
public int iterations;
@@ -47,7 +47,7 @@ public class MBTilesBenchmark {
@Setup
public void setup() throws IOException, TileStoreException {
file = Files.createTempFile(Paths.get("."), "baremaps", ".mbtiles");
- mbTilesStore = new MBTilesStore(SqliteUtils.createDataSource(file, false));
+ mbTilesStore = new MBTilesStore(SqliteUtils.createDataSource(file));
mbTilesStore.initializeDatabase();
}
diff --git a/baremaps-cli/src/main/java/org/apache/baremaps/cli/map/Dev.java b/baremaps-cli/src/main/java/org/apache/baremaps/cli/map/Dev.java
index c77b1444..2c54f526 100644
--- a/baremaps-cli/src/main/java/org/apache/baremaps/cli/map/Dev.java
+++ b/baremaps-cli/src/main/java/org/apache/baremaps/cli/map/Dev.java
@@ -24,12 +24,12 @@ import java.util.function.Supplier;
import org.apache.baremaps.cli.Options;
import org.apache.baremaps.config.ConfigReader;
import org.apache.baremaps.server.*;
+import org.apache.baremaps.server.CorsFilter;
import org.apache.baremaps.tilestore.TileStore;
import org.apache.baremaps.tilestore.postgres.PostgresTileStore;
+import org.apache.baremaps.utils.PostgresUtils;
import org.apache.baremaps.vectortile.style.Style;
import org.apache.baremaps.vectortile.tilejson.TileJSON;
-import org.apache.baremaps.server.CorsFilter;
-import org.apache.baremaps.utils.PostgresUtils;
import org.apache.baremaps.vectortile.tileset.Tileset;
import org.glassfish.hk2.api.TypeLiteral;
import org.glassfish.hk2.utilities.binding.AbstractBinder;
diff --git a/baremaps-cli/src/main/java/org/apache/baremaps/cli/map/MBTiles.java b/baremaps-cli/src/main/java/org/apache/baremaps/cli/map/MBTiles.java
index 67924764..69ee4a2c 100644
--- a/baremaps-cli/src/main/java/org/apache/baremaps/cli/map/MBTiles.java
+++ b/baremaps-cli/src/main/java/org/apache/baremaps/cli/map/MBTiles.java
@@ -15,19 +15,23 @@ package org.apache.baremaps.cli.map;
import static io.servicetalk.data.jackson.jersey.ServiceTalkJacksonSerializerFeature.newContextResolver;
import static org.apache.baremaps.utils.ObjectMapperUtils.objectMapper;
-import com.fasterxml.jackson.databind.ObjectMapper;
import com.github.benmanes.caffeine.cache.CaffeineSpec;
import io.servicetalk.http.netty.HttpServers;
import io.servicetalk.http.router.jersey.HttpJerseyRouterBuilder;
import java.nio.file.Path;
import java.util.concurrent.Callable;
+import java.util.function.Supplier;
+
import org.apache.baremaps.cli.Options;
-import org.apache.baremaps.server.CorsFilter;
-import org.apache.baremaps.server.ServerResources;
+import org.apache.baremaps.config.ConfigReader;
+import org.apache.baremaps.server.*;
import org.apache.baremaps.tilestore.TileCache;
import org.apache.baremaps.tilestore.TileStore;
import org.apache.baremaps.tilestore.mbtiles.MBTilesStore;
import org.apache.baremaps.utils.SqliteUtils;
+import org.apache.baremaps.vectortile.style.Style;
+import org.apache.baremaps.vectortile.tilejson.TileJSON;
+import org.glassfish.hk2.api.TypeLiteral;
import org.glassfish.hk2.utilities.binding.AbstractBinder;
import org.glassfish.jersey.server.ResourceConfig;
import org.slf4j.Logger;
@@ -65,24 +69,42 @@ public class MBTiles implements Callable<Integer> {
@Override
public Integer call() throws Exception {
var objectMapper = objectMapper();
+ var configReader = new ConfigReader();
var caffeineSpec = CaffeineSpec.parse(cache);
- var datasource = SqliteUtils.createDataSource(mbtilesPath, true);
+ var datasource = SqliteUtils.createDataSource(mbtilesPath);
+ var tileStoreSupplierType = new TypeLiteral<Supplier<TileStore>>() {};
var tileStore = new MBTilesStore(datasource);
var tileCache = new TileCache(tileStore, caffeineSpec);
+ var tileStoreSupplier = (Supplier<TileStore>) () -> tileCache;
+
+ var styleSupplierType = new TypeLiteral<Supplier<Style>>() {};
+ var style = objectMapper.readValue(configReader.read(stylePath), Style.class);
+ var styleSupplier = (Supplier<Style>) () -> style;
+
+ var tileJSONSupplierType = new TypeLiteral<Supplier<TileJSON>>() {};
+ var tileJSON = objectMapper.readValue(configReader.read(tileJSONPath), TileJSON.class);
+ var tileJSONSupplier = (Supplier<TileJSON>) () -> tileJSON;
// Configure the application
var application =
- new ResourceConfig().register(CorsFilter.class).register(ServerResources.class)
- .register(newContextResolver(objectMapper)).register(new AbstractBinder() {
- @Override
- protected void configure() {
- bind(tileJSONPath).to(Path.class).named("tileset");
- bind(stylePath).to(Path.class).named("style");
- bind(tileCache).to(TileStore.class);
- bind(objectMapper).to(ObjectMapper.class);
- }
- });
+ new ResourceConfig()
+ .register(CorsFilter.class)
+ .register(TileResource.class)
+ .register(StyleResource.class)
+ .register(TilesetResource.class)
+ .register(ClassPathResource.class)
+ .register(newContextResolver(objectMapper))
+ .register(new AbstractBinder() {
+ @Override
+ protected void configure() {
+ bind("assets").to(String.class).named("directory");
+ bind("server.html").to(String.class).named("index");
+ bind(tileStoreSupplier).to(tileStoreSupplierType);
+ bind(styleSupplier).to(styleSupplierType);
+ bind(tileJSONSupplier).to(tileJSONSupplierType);
+ }
+ });
var httpService = new HttpJerseyRouterBuilder().buildBlockingStreaming(application);
var serverContext = HttpServers.forPort(port).listenBlockingStreamingAndAwait(httpService);
diff --git a/baremaps-cli/src/main/java/org/apache/baremaps/cli/map/Serve.java b/baremaps-cli/src/main/java/org/apache/baremaps/cli/map/Serve.java
index 5c1fa5ad..62978214 100644
--- a/baremaps-cli/src/main/java/org/apache/baremaps/cli/map/Serve.java
+++ b/baremaps-cli/src/main/java/org/apache/baremaps/cli/map/Serve.java
@@ -23,15 +23,14 @@ import java.util.concurrent.Callable;
import java.util.function.Supplier;
import org.apache.baremaps.cli.Options;
import org.apache.baremaps.config.ConfigReader;
-
import org.apache.baremaps.server.*;
+import org.apache.baremaps.server.CorsFilter;
import org.apache.baremaps.tilestore.TileCache;
import org.apache.baremaps.tilestore.TileStore;
import org.apache.baremaps.tilestore.postgres.PostgresTileStore;
+import org.apache.baremaps.utils.PostgresUtils;
import org.apache.baremaps.vectortile.style.Style;
import org.apache.baremaps.vectortile.tilejson.TileJSON;
-import org.apache.baremaps.server.CorsFilter;
-import org.apache.baremaps.utils.PostgresUtils;
import org.apache.baremaps.vectortile.tileset.Tileset;
import org.glassfish.hk2.api.TypeLiteral;
import org.glassfish.hk2.utilities.binding.AbstractBinder;
@@ -71,10 +70,10 @@ public class Serve implements Callable<Integer> {
public Integer call() throws Exception {
var objectMapper = objectMapper();
var configReader = new ConfigReader();
- var tileset = objectMapper.readValue(configReader.read(this.tilesetPath), Tileset.class);
var caffeineSpec = CaffeineSpec.parse(cache);
- var datasource = PostgresUtils.createDataSource(tileset.getDatabase());
+ var tileset = objectMapper.readValue(configReader.read(tilesetPath), Tileset.class);
+ var datasource = PostgresUtils.createDataSource(tileset.getDatabase());
var tileStoreSupplierType = new TypeLiteral<Supplier<TileStore>>() {};
var tileStore = new PostgresTileStore(datasource, tileset);
var tileCache = new TileCache(tileStore, caffeineSpec);
diff --git a/baremaps-core/src/main/java/org/apache/baremaps/utils/SqliteUtils.java b/baremaps-core/src/main/java/org/apache/baremaps/utils/SqliteUtils.java
index 1ca6baaa..f6ea9051 100644
--- a/baremaps-core/src/main/java/org/apache/baremaps/utils/SqliteUtils.java
+++ b/baremaps-core/src/main/java/org/apache/baremaps/utils/SqliteUtils.java
@@ -41,10 +41,9 @@ public final class SqliteUtils {
* Create a SQLite data source.
*
* @param path the path to the SQLite database
- * @param readOnly
* @return the SQLite data source
*/
- public static HikariDataSource createDataSource(Path path, boolean readOnly) {
+ public static HikariDataSource createDataSource(Path path) {
var sqliteConfig = new SQLiteConfig();
sqliteConfig.setCacheSize(1000000);
sqliteConfig.setPageSize(65536);
@@ -52,7 +51,6 @@ public final class SqliteUtils {
sqliteConfig.setLockingMode(LockingMode.EXCLUSIVE);
sqliteConfig.setSynchronous(SynchronousMode.OFF);
sqliteConfig.setTempStore(TempStore.MEMORY);
- sqliteConfig.setReadOnly(readOnly);
var sqliteDataSource = new SQLiteDataSource();
sqliteDataSource.setConfig(sqliteConfig);
diff --git a/baremaps-core/src/main/java/org/apache/baremaps/workflow/tasks/ExportVectorTiles.java b/baremaps-core/src/main/java/org/apache/baremaps/workflow/tasks/ExportVectorTiles.java
index b33377e5..5f3f4910 100644
--- a/baremaps-core/src/main/java/org/apache/baremaps/workflow/tasks/ExportVectorTiles.java
+++ b/baremaps-core/src/main/java/org/apache/baremaps/workflow/tasks/ExportVectorTiles.java
@@ -79,7 +79,7 @@ public record ExportVectorTiles(
private TileStore targetTileStore(Tileset source) throws TileStoreException, IOException {
if (mbtiles) {
Files.deleteIfExists(repository);
- var dataSource = SqliteUtils.createDataSource(repository, false);
+ var dataSource = SqliteUtils.createDataSource(repository);
var tilesStore = new MBTilesStore(dataSource);
tilesStore.initializeDatabase();
tilesStore.writeMetadata(metadata(source));
diff --git a/baremaps-core/src/test/java/org/apache/baremaps/tilestore/mbtiles/MBTilesStoreTest.java b/baremaps-core/src/test/java/org/apache/baremaps/tilestore/mbtiles/MBTilesStoreTest.java
index f28154b4..12abdf82 100644
--- a/baremaps-core/src/test/java/org/apache/baremaps/tilestore/mbtiles/MBTilesStoreTest.java
+++ b/baremaps-core/src/test/java/org/apache/baremaps/tilestore/mbtiles/MBTilesStoreTest.java
@@ -13,7 +13,6 @@
package org.apache.baremaps.tilestore.mbtiles;
import static org.junit.jupiter.api.Assertions.assertEquals;
-import static org.junit.jupiter.api.Assertions.assertTrue;
import java.io.IOException;
import java.nio.file.Files;
@@ -56,14 +55,14 @@ class MBTilesStoreTest extends TileStoreTest {
void readWriteMetadata() throws Exception {
MBTilesStore tileStore = createTileStore();
Map<String, String> metadata = tileStore.readMetadata();
- assertTrue(metadata.size() == 0);
+ assertEquals(0, metadata.size());
Map<String, String> m1 = new HashMap<>();
m1.put("test", "test");
tileStore.writeMetadata(m1);
Map<String, String> m2 = tileStore.readMetadata();
- assertTrue(m2.size() == 1);
- assertEquals(m2.get("test"), "test");
+ assertEquals(1, m2.size());
+ assertEquals("test", m2.get("test"));
}
}
diff --git a/baremaps-server/src/main/java/org/apache/baremaps/server/TileResource.java b/baremaps-server/src/main/java/org/apache/baremaps/server/TileResource.java
index cd64435a..4029a556 100644
--- a/baremaps-server/src/main/java/org/apache/baremaps/server/TileResource.java
+++ b/baremaps-server/src/main/java/org/apache/baremaps/server/TileResource.java
@@ -49,7 +49,7 @@ public class TileResource {
TileCoord tileCoord = new TileCoord(x, y, z);
try {
TileStore tileStore = tileStoreSupplier.get();
- ByteBuffer blob = tileStore.read(tileCoord);
+ ByteBuffer blob = tileStore.get(tileCoord);
if (blob != null) {
byte[] bytes = new byte[blob.remaining()];
blob.get(bytes);
[incubator-baremaps] 01/09: Fix null pointer exception related to bounds
Posted by bc...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
bchapuis pushed a commit to branch 688-export
in repository https://gitbox.apache.org/repos/asf/incubator-baremaps.git
commit 09734a955362e67e5fd55d112f73ed900e26f084
Author: Bertil Chapuis <bc...@gmail.com>
AuthorDate: Fri Jun 9 21:14:58 2023 +0200
Fix null pointer exception related to bounds
---
.../apache/baremaps/tilestore/mbtiles/MBTiles.java | 5 +++--
.../baremaps/workflow/tasks/ExportVectorTiles.java | 25 ++++++++++++++++------
2 files changed, 21 insertions(+), 9 deletions(-)
diff --git a/baremaps-core/src/main/java/org/apache/baremaps/tilestore/mbtiles/MBTiles.java b/baremaps-core/src/main/java/org/apache/baremaps/tilestore/mbtiles/MBTiles.java
index 9cfbdad7..06d0e0cb 100644
--- a/baremaps-core/src/main/java/org/apache/baremaps/tilestore/mbtiles/MBTiles.java
+++ b/baremaps-core/src/main/java/org/apache/baremaps/tilestore/mbtiles/MBTiles.java
@@ -23,6 +23,7 @@ import java.sql.SQLException;
import java.sql.Statement;
import java.util.HashMap;
import java.util.Map;
+import javax.sql.DataSource;
import org.apache.baremaps.tilestore.TileCoord;
import org.apache.baremaps.tilestore.TileStore;
import org.apache.baremaps.tilestore.TileStoreException;
@@ -58,14 +59,14 @@ public class MBTiles implements TileStore {
private static final String DELETE_METADATA = "DELETE FROM metadata";
- private final org.sqlite.SQLiteDataSource dataSource;
+ private final DataSource dataSource;
/**
* Constructs an {@code MBTiles} with the provided SQLite datasource.
*
* @param dataSource the SQLite datasource
*/
- public MBTiles(org.sqlite.SQLiteDataSource dataSource) {
+ public MBTiles(DataSource dataSource) {
this.dataSource = dataSource;
}
diff --git a/baremaps-core/src/main/java/org/apache/baremaps/workflow/tasks/ExportVectorTiles.java b/baremaps-core/src/main/java/org/apache/baremaps/workflow/tasks/ExportVectorTiles.java
index 7828a049..32246c2d 100644
--- a/baremaps-core/src/main/java/org/apache/baremaps/workflow/tasks/ExportVectorTiles.java
+++ b/baremaps-core/src/main/java/org/apache/baremaps/workflow/tasks/ExportVectorTiles.java
@@ -16,7 +16,10 @@ import static org.apache.baremaps.utils.ObjectMapperUtils.objectMapper;
import com.fasterxml.jackson.core.JsonProcessingException;
import com.fasterxml.jackson.databind.ObjectMapper;
+import com.zaxxer.hikari.HikariConfig;
+import com.zaxxer.hikari.HikariDataSource;
import java.io.IOException;
+import java.nio.file.Files;
import java.nio.file.Path;
import java.util.HashMap;
import java.util.Map;
@@ -61,10 +64,11 @@ public record ExportVectorTiles(
var sourceTileStore = sourceTileStore(tileset, datasource);
var targetTileStore = targetTileStore(tileset);
- var envelope =
- new Envelope(
+ var envelope = tileset.getBounds().size() == 4
+ ? new Envelope(
tileset.getBounds().get(0), tileset.getBounds().get(2),
- tileset.getBounds().get(1), tileset.getBounds().get(3));
+ tileset.getBounds().get(1), tileset.getBounds().get(3))
+ : new Envelope(-180, 180, -85.0511, 85.0511);
var count = TileCoord.count(envelope, tileset.getMinzoom(), tileset.getMaxzoom());
@@ -81,6 +85,8 @@ public record ExportVectorTiles(
private TileStore targetTileStore(Tileset source) throws TileStoreException, IOException {
if (mbtiles) {
+ Files.deleteIfExists(repository);
+
var sqliteConfig = new SQLiteConfig();
sqliteConfig.setCacheSize(1000000);
sqliteConfig.setJournalMode(JournalMode.OFF);
@@ -88,11 +94,16 @@ public record ExportVectorTiles(
sqliteConfig.setSynchronous(SynchronousMode.OFF);
sqliteConfig.setTempStore(TempStore.MEMORY);
- var dataSource = new SQLiteDataSource();
- dataSource.setConfig(sqliteConfig);
- dataSource.setUrl("jdbc:sqlite:" + repository);
+ var sqliteDataSource = new SQLiteDataSource();
+ sqliteDataSource.setConfig(sqliteConfig);
+ sqliteDataSource.setUrl("jdbc:sqlite:" + repository);
+
+ var hikariConfig = new HikariConfig();
+ hikariConfig.setDataSource(sqliteDataSource);
+ hikariConfig.setMaximumPoolSize(1);
+ var hikariDataSource = new HikariDataSource(hikariConfig);
- var tilesStore = new MBTiles(dataSource);
+ var tilesStore = new MBTiles(hikariDataSource);
tilesStore.initializeDatabase();
tilesStore.writeMetadata(metadata(source));
[incubator-baremaps] 06/09: Add mbtiles command
Posted by bc...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
bchapuis pushed a commit to branch 688-export
in repository https://gitbox.apache.org/repos/asf/incubator-baremaps.git
commit 2498555787347b0fd09c104471f7106ac79643dc
Author: Bertil Chapuis <bc...@gmail.com>
AuthorDate: Fri Jun 16 22:51:27 2023 +0200
Add mbtiles command
---
.run/basemap-mbtiles.run.xml | 17 ++++
.../java/org/apache/baremaps/cli/map/MBTiles.java | 100 +++++++++++++++++++++
.../main/java/org/apache/baremaps/cli/map/Map.java | 2 +-
3 files changed, 118 insertions(+), 1 deletion(-)
diff --git a/.run/basemap-mbtiles.run.xml b/.run/basemap-mbtiles.run.xml
new file mode 100644
index 00000000..3a08cf9a
--- /dev/null
+++ b/.run/basemap-mbtiles.run.xml
@@ -0,0 +1,17 @@
+<component name="ProjectRunConfigurationManager">
+ <configuration default="false" name="basemap-mbtiles" type="Application" factoryName="Application">
+ <option name="MAIN_CLASS_NAME" value="org.apache.baremaps.cli.Baremaps" />
+ <module name="baremaps-cli" />
+ <option name="PROGRAM_PARAMETERS" value="map mbtiles --mbtiles $USER_HOME$/Datasets/Baremaps/tiles.mbtiles --tileset tileset.js --style style.js" />
+ <option name="WORKING_DIRECTORY" value="$PROJECT_DIR$/basemap" />
+ <extension name="coverage">
+ <pattern>
+ <option name="PATTERN" value="org.apache.baremaps.server.ogcapi.*" />
+ <option name="ENABLED" value="true" />
+ </pattern>
+ </extension>
+ <method v="2">
+ <option name="Make" enabled="true" />
+ </method>
+ </configuration>
+</component>
\ No newline at end of file
diff --git a/baremaps-cli/src/main/java/org/apache/baremaps/cli/map/MBTiles.java b/baremaps-cli/src/main/java/org/apache/baremaps/cli/map/MBTiles.java
new file mode 100644
index 00000000..f4478ca9
--- /dev/null
+++ b/baremaps-cli/src/main/java/org/apache/baremaps/cli/map/MBTiles.java
@@ -0,0 +1,100 @@
+/*
+ * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except
+ * in compliance with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software distributed under the License
+ * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
+ * or implied. See the License for the specific language governing permissions and limitations under
+ * the License.
+ */
+
+package org.apache.baremaps.cli.map;
+
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.github.benmanes.caffeine.cache.CaffeineSpec;
+import io.servicetalk.http.netty.HttpServers;
+import io.servicetalk.http.router.jersey.HttpJerseyRouterBuilder;
+import org.apache.baremaps.cli.Options;
+import org.apache.baremaps.config.ConfigReader;
+import org.apache.baremaps.postgres.PostgresUtils;
+import org.apache.baremaps.server.CorsFilter;
+import org.apache.baremaps.server.ServerResources;
+import org.apache.baremaps.tilestore.TileCache;
+import org.apache.baremaps.tilestore.TileStore;
+import org.apache.baremaps.tilestore.postgres.PostgresTileStore;
+import org.apache.baremaps.vectortile.tileset.Tileset;
+import org.apache.baremaps.workflow.tasks.ExportVectorTiles;
+import org.glassfish.hk2.utilities.binding.AbstractBinder;
+import org.glassfish.jersey.server.ResourceConfig;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import picocli.CommandLine.Command;
+import picocli.CommandLine.Mixin;
+import picocli.CommandLine.Option;
+
+import java.nio.file.Path;
+import java.util.concurrent.Callable;
+
+import static io.servicetalk.data.jackson.jersey.ServiceTalkJacksonSerializerFeature.contextResolverFor;
+import static io.servicetalk.data.jackson.jersey.ServiceTalkJacksonSerializerFeature.newContextResolver;
+import static org.apache.baremaps.utils.ObjectMapperUtils.objectMapper;
+
+@Command(name = "mbtiles", description = "Start a mbtiles server with caching capabilities.")
+public class MBTiles implements Callable<Integer> {
+
+ private static final Logger logger = LoggerFactory.getLogger(MBTiles.class);
+
+ @Mixin
+ private Options options;
+
+ @Option(names = {"--cache"}, paramLabel = "CACHE", description = "The caffeine cache directive.")
+ private String cache = "";
+
+ @Option(names = {"--mbtiles"}, paramLabel = "MBTILES", description = "The mbtiles file.",
+ required = true)
+ private Path mbtiles;
+
+ @Option(names = {"--tilejson"}, paramLabel = "TILEJSON", description = "The tileJSON file.",
+ required = true)
+ private Path tileset;
+
+ @Option(names = {"--style"}, paramLabel = "STYLE", description = "The style file.",
+ required = true)
+ private Path style;
+
+ @Option(names = {"--port"}, paramLabel = "PORT", description = "The port of the server.")
+ private int port = 9000;
+
+ @Override
+ public Integer call() throws Exception {
+ var objectMapper = objectMapper();
+ var caffeineSpec = CaffeineSpec.parse(cache);
+ var datasource = ExportVectorTiles.createDataSource(mbtiles);
+
+ var tileStore = new org.apache.baremaps.tilestore.mbtiles.MBTiles(datasource);
+ var tileCache = new TileCache(tileStore, caffeineSpec);
+
+ // Configure the application
+ var application =
+ new ResourceConfig().register(CorsFilter.class).register(ServerResources.class)
+ .register(newContextResolver(objectMapper)).register(new AbstractBinder() {
+ @Override
+ protected void configure() {
+ bind(tileset).to(Path.class).named("tileset");
+ bind(style).to(Path.class).named("style");
+ bind(tileCache).to(TileStore.class);
+ bind(objectMapper).to(ObjectMapper.class);
+ }
+ });
+
+ var httpService = new HttpJerseyRouterBuilder().buildBlockingStreaming(application);
+ var serverContext = HttpServers.forPort(port).listenBlockingStreamingAndAwait(httpService);
+
+ logger.info("Listening on {}", serverContext.listenAddress());
+
+ serverContext.awaitShutdown();
+ return 0;
+ }
+}
diff --git a/baremaps-cli/src/main/java/org/apache/baremaps/cli/map/Map.java b/baremaps-cli/src/main/java/org/apache/baremaps/cli/map/Map.java
index 9ed602d0..f84dea54 100644
--- a/baremaps-cli/src/main/java/org/apache/baremaps/cli/map/Map.java
+++ b/baremaps-cli/src/main/java/org/apache/baremaps/cli/map/Map.java
@@ -18,7 +18,7 @@ import picocli.CommandLine;
import picocli.CommandLine.Command;
@Command(name = "map", description = "Map commands.",
- subcommands = {Init.class, Export.class, Serve.class, Dev.class}, sortOptions = false)
+ subcommands = {Init.class, Export.class, Serve.class, Dev.class, MBTiles.class}, sortOptions = false)
public class Map implements Runnable {
@Override
[incubator-baremaps] 05/09: Format code
Posted by bc...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
bchapuis pushed a commit to branch 688-export
in repository https://gitbox.apache.org/repos/asf/incubator-baremaps.git
commit a6da66f2dafbe6a56c4e45edb37540a7bd496b1b
Author: Bertil Chapuis <bc...@gmail.com>
AuthorDate: Wed Jun 14 00:01:16 2023 +0200
Format code
---
.../baremaps/benchmarks/MBTilesBenchmark.java | 85 +++++++++++-----------
.../apache/baremaps/tilestore/mbtiles/MBTiles.java | 2 +-
.../baremaps/workflow/tasks/ExportVectorTiles.java | 1 +
3 files changed, 44 insertions(+), 44 deletions(-)
diff --git a/baremaps-benchmark/src/main/java/org/apache/baremaps/benchmarks/MBTilesBenchmark.java b/baremaps-benchmark/src/main/java/org/apache/baremaps/benchmarks/MBTilesBenchmark.java
index 9dd817a9..b529eb66 100644
--- a/baremaps-benchmark/src/main/java/org/apache/baremaps/benchmarks/MBTilesBenchmark.java
+++ b/baremaps-benchmark/src/main/java/org/apache/baremaps/benchmarks/MBTilesBenchmark.java
@@ -18,7 +18,6 @@ import java.nio.ByteBuffer;
import java.util.ArrayList;
import java.util.Random;
import java.util.concurrent.TimeUnit;
-
import org.apache.baremaps.tilestore.TileCoord;
import org.apache.baremaps.tilestore.TileStoreException;
import org.apache.baremaps.tilestore.mbtiles.MBTiles;
@@ -34,59 +33,59 @@ import org.openjdk.jmh.runner.options.OptionsBuilder;
@Fork(value = 1, warmups = 1)
public class MBTilesBenchmark {
- public Random random = new Random(0);
+ public Random random = new Random(0);
- @Param({"10", "100", "1000"})
- public int iterations;
+ @Param({"10", "100", "1000"})
+ public int iterations;
- private MBTiles mbTiles;
+ private MBTiles mbTiles;
- @Setup
- public void setup() throws IOException, TileStoreException {
- var sqliteFile = File.createTempFile("baremaps", ".sqlite");
- sqliteFile.deleteOnExit();
- var sqliteDataSource = ExportVectorTiles.createDataSource(sqliteFile.toPath());
- mbTiles = new MBTiles(sqliteDataSource);
- mbTiles.initializeDatabase();
- }
+ @Setup
+ public void setup() throws IOException, TileStoreException {
+ var sqliteFile = File.createTempFile("baremaps", ".sqlite");
+ sqliteFile.deleteOnExit();
+ var sqliteDataSource = ExportVectorTiles.createDataSource(sqliteFile.toPath());
+ mbTiles = new MBTiles(sqliteDataSource);
+ mbTiles.initializeDatabase();
+ }
- @Benchmark
- @BenchmarkMode(Mode.SingleShotTime)
- public void writeMBTiles(MBTilesBenchmark benchmark) throws TileStoreException {
- for (int i = 0; i < benchmark.iterations; i++) {
- var bytes = new byte[1 << 16];
- random.nextBytes(bytes);
- mbTiles.put(new TileCoord(0, 0, i), ByteBuffer.wrap(bytes));
- }
+ @Benchmark
+ @BenchmarkMode(Mode.SingleShotTime)
+ public void writeMBTiles(MBTilesBenchmark benchmark) throws TileStoreException {
+ for (int i = 0; i < benchmark.iterations; i++) {
+ var bytes = new byte[1 << 16];
+ random.nextBytes(bytes);
+ mbTiles.put(new TileCoord(0, 0, i), ByteBuffer.wrap(bytes));
}
+ }
- @Benchmark
- @BenchmarkMode(Mode.SingleShotTime)
- public void writeMBTilesBatch(MBTilesBenchmark benchmark) throws TileStoreException {
- var coords = new ArrayList<TileCoord>();
- var buffers = new ArrayList<ByteBuffer>();
- for (int i = 0; i < benchmark.iterations; i++) {
- var bytes = new byte[1 << 16];
- random.nextBytes(bytes);
- coords.add(new TileCoord(0, 0, i));
- buffers.add(ByteBuffer.wrap(bytes));
- if (coords.size() == 100) {
- random.nextBytes(bytes);
- mbTiles.put(coords, buffers);
- coords.clear();
- buffers.clear();
- }
- }
+ @Benchmark
+ @BenchmarkMode(Mode.SingleShotTime)
+ public void writeMBTilesBatch(MBTilesBenchmark benchmark) throws TileStoreException {
+ var coords = new ArrayList<TileCoord>();
+ var buffers = new ArrayList<ByteBuffer>();
+ for (int i = 0; i < benchmark.iterations; i++) {
+ var bytes = new byte[1 << 16];
+ random.nextBytes(bytes);
+ coords.add(new TileCoord(0, 0, i));
+ buffers.add(ByteBuffer.wrap(bytes));
+ if (coords.size() == 100) {
+ random.nextBytes(bytes);
mbTiles.put(coords, buffers);
coords.clear();
buffers.clear();
+ }
}
+ mbTiles.put(coords, buffers);
+ coords.clear();
+ buffers.clear();
+ }
- public static void main(String[] args) throws RunnerException {
- Options opt =
- new OptionsBuilder().include(MBTilesBenchmark.class.getSimpleName()).forks(1).build();
- new Runner(opt).run();
- }
+ public static void main(String[] args) throws RunnerException {
+ Options opt =
+ new OptionsBuilder().include(MBTilesBenchmark.class.getSimpleName()).forks(1).build();
+ new Runner(opt).run();
+ }
}
diff --git a/baremaps-core/src/main/java/org/apache/baremaps/tilestore/mbtiles/MBTiles.java b/baremaps-core/src/main/java/org/apache/baremaps/tilestore/mbtiles/MBTiles.java
index fa1fb9d7..0344aed5 100644
--- a/baremaps-core/src/main/java/org/apache/baremaps/tilestore/mbtiles/MBTiles.java
+++ b/baremaps-core/src/main/java/org/apache/baremaps/tilestore/mbtiles/MBTiles.java
@@ -110,7 +110,7 @@ public class MBTiles implements TileStore {
@Override
public void put(List<TileCoord> tileCoords, List<ByteBuffer> blobs) throws TileStoreException {
try (Connection connection = dataSource.getConnection();
- PreparedStatement statement = connection.prepareStatement(INSERT_TILE)) {
+ PreparedStatement statement = connection.prepareStatement(INSERT_TILE)) {
for (int i = 0; i < tileCoords.size(); i++) {
TileCoord tileCoord = tileCoords.get(i);
ByteBuffer blob = blobs.get(i);
diff --git a/baremaps-core/src/main/java/org/apache/baremaps/workflow/tasks/ExportVectorTiles.java b/baremaps-core/src/main/java/org/apache/baremaps/workflow/tasks/ExportVectorTiles.java
index 435556ea..5cca2bec 100644
--- a/baremaps-core/src/main/java/org/apache/baremaps/workflow/tasks/ExportVectorTiles.java
+++ b/baremaps-core/src/main/java/org/apache/baremaps/workflow/tasks/ExportVectorTiles.java
@@ -142,6 +142,7 @@ public record ExportVectorTiles(
/**
* Create a SQLite data source.
+ *
* @param path the path to the SQLite database
* @return the SQLite data source
*/
[incubator-baremaps] 08/09: Refactor utility classes
Posted by bc...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
bchapuis pushed a commit to branch 688-export
in repository https://gitbox.apache.org/repos/asf/incubator-baremaps.git
commit 450804d104cdeb0f6009742c5e61696e6fcac43a
Author: Bertil Chapuis <bc...@gmail.com>
AuthorDate: Sun Jun 18 23:24:00 2023 +0200
Refactor utility classes
---
.../baremaps/benchmarks/MBTilesBenchmark.java | 16 ++++-----
.../main/java/org/apache/baremaps/cli/map/Dev.java | 5 +--
.../java/org/apache/baremaps/cli/map/MBTiles.java | 17 ++++-----
.../java/org/apache/baremaps/cli/map/Serve.java | 6 ++--
.../org/apache/baremaps/cli/ogcapi/OgcApi.java | 4 +--
.../mbtiles/{MBTiles.java => MBTilesStore.java} | 4 +--
.../{postgres => utils}/PostgresUtils.java | 15 ++++----
.../org/apache/baremaps/utils/SqliteUtils.java | 39 +++++++++++++++++++++
.../apache/baremaps/workflow/WorkflowContext.java | 4 +--
.../baremaps/workflow/tasks/ExportVectorTiles.java | 40 +++-------------------
.../postgres/PostgresRepositoryTest.java | 2 +-
.../repository/BlockImporterTest.java | 4 +--
.../postgres/metadata/PostgresMetadataTest.java | 2 +-
.../{MBTilesTest.java => MBTilesStoreTest.java} | 8 ++---
.../org/apache/baremaps/ogcapi/OgcApiTest.java | 4 +--
15 files changed, 91 insertions(+), 79 deletions(-)
diff --git a/baremaps-benchmark/src/main/java/org/apache/baremaps/benchmarks/MBTilesBenchmark.java b/baremaps-benchmark/src/main/java/org/apache/baremaps/benchmarks/MBTilesBenchmark.java
index dc333999..3754ae6f 100644
--- a/baremaps-benchmark/src/main/java/org/apache/baremaps/benchmarks/MBTilesBenchmark.java
+++ b/baremaps-benchmark/src/main/java/org/apache/baremaps/benchmarks/MBTilesBenchmark.java
@@ -22,8 +22,8 @@ import java.util.ArrayList;
import java.util.concurrent.TimeUnit;
import org.apache.baremaps.tilestore.TileCoord;
import org.apache.baremaps.tilestore.TileStoreException;
-import org.apache.baremaps.tilestore.mbtiles.MBTiles;
-import org.apache.baremaps.workflow.tasks.ExportVectorTiles;
+import org.apache.baremaps.tilestore.mbtiles.MBTilesStore;
+import org.apache.baremaps.utils.SqliteUtils;
import org.openjdk.jmh.annotations.*;
import org.openjdk.jmh.runner.Runner;
import org.openjdk.jmh.runner.RunnerException;
@@ -42,13 +42,13 @@ public class MBTilesBenchmark {
private Path file;
- private MBTiles mbTiles;
+ private MBTilesStore mbTilesStore;
@Setup
public void setup() throws IOException, TileStoreException {
file = Files.createTempFile(Paths.get("."), "baremaps", ".mbtiles");
- mbTiles = new MBTiles(ExportVectorTiles.createDataSource(file));
- mbTiles.initializeDatabase();
+ mbTilesStore = new MBTilesStore(SqliteUtils.createDataSource(file, false));
+ mbTilesStore.initializeDatabase();
}
@TearDown
@@ -62,7 +62,7 @@ public class MBTilesBenchmark {
for (int i = 0; i < benchmark.iterations; i++) {
var bytes = new byte[1 << 16];
random.nextBytes(bytes);
- mbTiles.put(new TileCoord(0, 0, i), ByteBuffer.wrap(bytes));
+ mbTilesStore.put(new TileCoord(0, 0, i), ByteBuffer.wrap(bytes));
}
}
@@ -78,12 +78,12 @@ public class MBTilesBenchmark {
buffers.add(ByteBuffer.wrap(bytes));
if (coords.size() == 100) {
random.nextBytes(bytes);
- mbTiles.put(coords, buffers);
+ mbTilesStore.put(coords, buffers);
coords.clear();
buffers.clear();
}
}
- mbTiles.put(coords, buffers);
+ mbTilesStore.put(coords, buffers);
coords.clear();
buffers.clear();
}
diff --git a/baremaps-cli/src/main/java/org/apache/baremaps/cli/map/Dev.java b/baremaps-cli/src/main/java/org/apache/baremaps/cli/map/Dev.java
index b0daab3c..c77b1444 100644
--- a/baremaps-cli/src/main/java/org/apache/baremaps/cli/map/Dev.java
+++ b/baremaps-cli/src/main/java/org/apache/baremaps/cli/map/Dev.java
@@ -23,12 +23,13 @@ import java.util.concurrent.Callable;
import java.util.function.Supplier;
import org.apache.baremaps.cli.Options;
import org.apache.baremaps.config.ConfigReader;
-import org.apache.baremaps.postgres.PostgresUtils;
import org.apache.baremaps.server.*;
import org.apache.baremaps.tilestore.TileStore;
import org.apache.baremaps.tilestore.postgres.PostgresTileStore;
import org.apache.baremaps.vectortile.style.Style;
import org.apache.baremaps.vectortile.tilejson.TileJSON;
+import org.apache.baremaps.server.CorsFilter;
+import org.apache.baremaps.utils.PostgresUtils;
import org.apache.baremaps.vectortile.tileset.Tileset;
import org.glassfish.hk2.api.TypeLiteral;
import org.glassfish.hk2.utilities.binding.AbstractBinder;
@@ -69,7 +70,7 @@ public class Dev implements Callable<Integer> {
var configReader = new ConfigReader();
var objectMapper = objectMapper();
var tileset = objectMapper.readValue(configReader.read(this.tilesetPath), Tileset.class);
- var datasource = PostgresUtils.dataSource(tileset.getDatabase());
+ var datasource = PostgresUtils.createDataSource(tileset.getDatabase());
var tileStoreType = new TypeLiteral<Supplier<TileStore>>() {};
var tileStoreSupplier = (Supplier<TileStore>) () -> {
diff --git a/baremaps-cli/src/main/java/org/apache/baremaps/cli/map/MBTiles.java b/baremaps-cli/src/main/java/org/apache/baremaps/cli/map/MBTiles.java
index 0f8c7f50..67924764 100644
--- a/baremaps-cli/src/main/java/org/apache/baremaps/cli/map/MBTiles.java
+++ b/baremaps-cli/src/main/java/org/apache/baremaps/cli/map/MBTiles.java
@@ -26,7 +26,8 @@ import org.apache.baremaps.server.CorsFilter;
import org.apache.baremaps.server.ServerResources;
import org.apache.baremaps.tilestore.TileCache;
import org.apache.baremaps.tilestore.TileStore;
-import org.apache.baremaps.workflow.tasks.ExportVectorTiles;
+import org.apache.baremaps.tilestore.mbtiles.MBTilesStore;
+import org.apache.baremaps.utils.SqliteUtils;
import org.glassfish.hk2.utilities.binding.AbstractBinder;
import org.glassfish.jersey.server.ResourceConfig;
import org.slf4j.Logger;
@@ -48,15 +49,15 @@ public class MBTiles implements Callable<Integer> {
@Option(names = {"--mbtiles"}, paramLabel = "MBTILES", description = "The mbtiles file.",
required = true)
- private Path mbtiles;
+ private Path mbtilesPath;
@Option(names = {"--tilejson"}, paramLabel = "TILEJSON", description = "The tileJSON file.",
required = true)
- private Path tileset;
+ private Path tileJSONPath;
@Option(names = {"--style"}, paramLabel = "STYLE", description = "The style file.",
required = true)
- private Path style;
+ private Path stylePath;
@Option(names = {"--port"}, paramLabel = "PORT", description = "The port of the server.")
private int port = 9000;
@@ -65,9 +66,9 @@ public class MBTiles implements Callable<Integer> {
public Integer call() throws Exception {
var objectMapper = objectMapper();
var caffeineSpec = CaffeineSpec.parse(cache);
- var datasource = ExportVectorTiles.createDataSource(mbtiles);
+ var datasource = SqliteUtils.createDataSource(mbtilesPath, true);
- var tileStore = new org.apache.baremaps.tilestore.mbtiles.MBTiles(datasource);
+ var tileStore = new MBTilesStore(datasource);
var tileCache = new TileCache(tileStore, caffeineSpec);
// Configure the application
@@ -76,8 +77,8 @@ public class MBTiles implements Callable<Integer> {
.register(newContextResolver(objectMapper)).register(new AbstractBinder() {
@Override
protected void configure() {
- bind(tileset).to(Path.class).named("tileset");
- bind(style).to(Path.class).named("style");
+ bind(tileJSONPath).to(Path.class).named("tileset");
+ bind(stylePath).to(Path.class).named("style");
bind(tileCache).to(TileStore.class);
bind(objectMapper).to(ObjectMapper.class);
}
diff --git a/baremaps-cli/src/main/java/org/apache/baremaps/cli/map/Serve.java b/baremaps-cli/src/main/java/org/apache/baremaps/cli/map/Serve.java
index e0002b03..5c1fa5ad 100644
--- a/baremaps-cli/src/main/java/org/apache/baremaps/cli/map/Serve.java
+++ b/baremaps-cli/src/main/java/org/apache/baremaps/cli/map/Serve.java
@@ -23,13 +23,15 @@ import java.util.concurrent.Callable;
import java.util.function.Supplier;
import org.apache.baremaps.cli.Options;
import org.apache.baremaps.config.ConfigReader;
-import org.apache.baremaps.postgres.PostgresUtils;
+
import org.apache.baremaps.server.*;
import org.apache.baremaps.tilestore.TileCache;
import org.apache.baremaps.tilestore.TileStore;
import org.apache.baremaps.tilestore.postgres.PostgresTileStore;
import org.apache.baremaps.vectortile.style.Style;
import org.apache.baremaps.vectortile.tilejson.TileJSON;
+import org.apache.baremaps.server.CorsFilter;
+import org.apache.baremaps.utils.PostgresUtils;
import org.apache.baremaps.vectortile.tileset.Tileset;
import org.glassfish.hk2.api.TypeLiteral;
import org.glassfish.hk2.utilities.binding.AbstractBinder;
@@ -71,7 +73,7 @@ public class Serve implements Callable<Integer> {
var configReader = new ConfigReader();
var tileset = objectMapper.readValue(configReader.read(this.tilesetPath), Tileset.class);
var caffeineSpec = CaffeineSpec.parse(cache);
- var datasource = PostgresUtils.dataSource(tileset.getDatabase());
+ var datasource = PostgresUtils.createDataSource(tileset.getDatabase());
var tileStoreSupplierType = new TypeLiteral<Supplier<TileStore>>() {};
var tileStore = new PostgresTileStore(datasource, tileset);
diff --git a/baremaps-cli/src/main/java/org/apache/baremaps/cli/ogcapi/OgcApi.java b/baremaps-cli/src/main/java/org/apache/baremaps/cli/ogcapi/OgcApi.java
index ea1a7e24..e596edc4 100644
--- a/baremaps-cli/src/main/java/org/apache/baremaps/cli/ogcapi/OgcApi.java
+++ b/baremaps-cli/src/main/java/org/apache/baremaps/cli/ogcapi/OgcApi.java
@@ -27,11 +27,11 @@ import javax.sql.DataSource;
import org.apache.baremaps.cli.Options;
import org.apache.baremaps.config.ConfigReader;
import org.apache.baremaps.ogcapi.*;
-import org.apache.baremaps.postgres.PostgresUtils;
import org.apache.baremaps.server.CorsFilter;
import org.apache.baremaps.tilestore.TileCache;
import org.apache.baremaps.tilestore.TileStore;
import org.apache.baremaps.tilestore.postgres.PostgresTileStore;
+import org.apache.baremaps.utils.PostgresUtils;
import org.apache.baremaps.vectortile.tileset.Tileset;
import org.glassfish.hk2.utilities.binding.AbstractBinder;
import org.glassfish.jersey.media.multipart.MultiPartFeature;
@@ -78,7 +78,7 @@ public class OgcApi implements Callable<Integer> {
var configReader = new ConfigReader();
var config = objectMapper.readValue(configReader.read(this.tileset), Tileset.class);
var caffeineSpec = CaffeineSpec.parse(cache);
- var dataSource = PostgresUtils.dataSource(database);
+ var dataSource = PostgresUtils.createDataSource(database);
var tileStore = new PostgresTileStore(dataSource, config);
var tileCache = new TileCache(tileStore, caffeineSpec);
diff --git a/baremaps-core/src/main/java/org/apache/baremaps/tilestore/mbtiles/MBTiles.java b/baremaps-core/src/main/java/org/apache/baremaps/tilestore/mbtiles/MBTilesStore.java
similarity index 98%
rename from baremaps-core/src/main/java/org/apache/baremaps/tilestore/mbtiles/MBTiles.java
rename to baremaps-core/src/main/java/org/apache/baremaps/tilestore/mbtiles/MBTilesStore.java
index 0344aed5..9539d9a5 100644
--- a/baremaps-core/src/main/java/org/apache/baremaps/tilestore/mbtiles/MBTiles.java
+++ b/baremaps-core/src/main/java/org/apache/baremaps/tilestore/mbtiles/MBTilesStore.java
@@ -34,7 +34,7 @@ import org.apache.baremaps.tilestore.TileStoreException;
* <a href="https://docs.mapbox.com/help/glossary/mbtiles/">MBTiles</a> file format for storing
* tiles.
*/
-public class MBTiles implements TileStore {
+public class MBTilesStore implements TileStore {
private static final String CREATE_TABLE_METADATA =
"CREATE TABLE IF NOT EXISTS metadata (name TEXT, value TEXT, PRIMARY KEY (name))";
@@ -67,7 +67,7 @@ public class MBTiles implements TileStore {
*
* @param dataSource the SQLite datasource
*/
- public MBTiles(DataSource dataSource) {
+ public MBTilesStore(DataSource dataSource) {
this.dataSource = dataSource;
}
diff --git a/baremaps-core/src/main/java/org/apache/baremaps/postgres/PostgresUtils.java b/baremaps-core/src/main/java/org/apache/baremaps/utils/PostgresUtils.java
similarity index 81%
rename from baremaps-core/src/main/java/org/apache/baremaps/postgres/PostgresUtils.java
rename to baremaps-core/src/main/java/org/apache/baremaps/utils/PostgresUtils.java
index 70f0667a..ab5cf6e3 100644
--- a/baremaps-core/src/main/java/org/apache/baremaps/postgres/PostgresUtils.java
+++ b/baremaps-core/src/main/java/org/apache/baremaps/utils/PostgresUtils.java
@@ -10,7 +10,7 @@
* the License.
*/
-package org.apache.baremaps.postgres;
+package org.apache.baremaps.utils;
@@ -29,10 +29,11 @@ public final class PostgresUtils {
private PostgresUtils() {}
- public static HikariDataSource dataSource(String host, Integer port, String database,
+ public static HikariDataSource createDataSource(String host, Integer port, String database,
String username, String password) {
- return dataSource(String.format("jdbc:postgresql://%s:%s/%s?&user=%s&password=%s", host, port,
- database, username, password));
+ return createDataSource(
+ String.format("jdbc:postgresql://%s:%s/%s?&user=%s&password=%s", host, port,
+ database, username, password));
}
/**
@@ -42,8 +43,8 @@ public final class PostgresUtils {
* @param url the JDBC url
* @return the data source
*/
- public static HikariDataSource dataSource(String url) {
- return dataSource(url, Runtime.getRuntime().availableProcessors());
+ public static HikariDataSource createDataSource(String url) {
+ return createDataSource(url, Runtime.getRuntime().availableProcessors());
}
/**
@@ -53,7 +54,7 @@ public final class PostgresUtils {
* @param poolSize the pool size
* @return the data source
*/
- public static HikariDataSource dataSource(String url, int poolSize) {
+ public static HikariDataSource createDataSource(String url, int poolSize) {
if (poolSize < 1) {
throw new IllegalArgumentException("PoolSize cannot be inferior to 1");
}
diff --git a/baremaps-core/src/main/java/org/apache/baremaps/utils/SqliteUtils.java b/baremaps-core/src/main/java/org/apache/baremaps/utils/SqliteUtils.java
index 4b937954..1ca6baaa 100644
--- a/baremaps-core/src/main/java/org/apache/baremaps/utils/SqliteUtils.java
+++ b/baremaps-core/src/main/java/org/apache/baremaps/utils/SqliteUtils.java
@@ -15,17 +15,56 @@ package org.apache.baremaps.utils;
import com.google.common.io.Resources;
+import com.zaxxer.hikari.HikariConfig;
+import com.zaxxer.hikari.HikariDataSource;
import java.io.IOException;
import java.net.URL;
import java.nio.charset.StandardCharsets;
+import java.nio.file.Path;
import java.sql.Connection;
import java.sql.DriverManager;
import java.sql.SQLException;
import java.sql.Statement;
+import org.sqlite.SQLiteConfig;
+import org.sqlite.SQLiteConfig.JournalMode;
+import org.sqlite.SQLiteConfig.LockingMode;
+import org.sqlite.SQLiteConfig.SynchronousMode;
+import org.sqlite.SQLiteConfig.TempStore;
+import org.sqlite.SQLiteDataSource;
/** A helper class for creating executing sql scripts onto a SQLite database */
public final class SqliteUtils {
+ private SqliteUtils() {}
+
+ /**
+ * Create a SQLite data source.
+ *
+ * @param path the path to the SQLite database
+ * @param readOnly
+ * @return the SQLite data source
+ */
+ public static HikariDataSource createDataSource(Path path, boolean readOnly) {
+ var sqliteConfig = new SQLiteConfig();
+ sqliteConfig.setCacheSize(1000000);
+ sqliteConfig.setPageSize(65536);
+ sqliteConfig.setJournalMode(JournalMode.OFF);
+ sqliteConfig.setLockingMode(LockingMode.EXCLUSIVE);
+ sqliteConfig.setSynchronous(SynchronousMode.OFF);
+ sqliteConfig.setTempStore(TempStore.MEMORY);
+ sqliteConfig.setReadOnly(readOnly);
+
+ var sqliteDataSource = new SQLiteDataSource();
+ sqliteDataSource.setConfig(sqliteConfig);
+ sqliteDataSource.setUrl("jdbc:sqlite:" + path);
+
+ var hikariConfig = new HikariConfig();
+ hikariConfig.setDataSource(sqliteDataSource);
+ hikariConfig.setMaximumPoolSize(1);
+
+ return new HikariDataSource(hikariConfig);
+ }
+
/**
* Executes the queries contained in a resource file.
*
diff --git a/baremaps-core/src/main/java/org/apache/baremaps/workflow/WorkflowContext.java b/baremaps-core/src/main/java/org/apache/baremaps/workflow/WorkflowContext.java
index 1cc7fab6..eff85f26 100644
--- a/baremaps-core/src/main/java/org/apache/baremaps/workflow/WorkflowContext.java
+++ b/baremaps-core/src/main/java/org/apache/baremaps/workflow/WorkflowContext.java
@@ -17,7 +17,7 @@ package org.apache.baremaps.workflow;
import java.util.Map;
import java.util.concurrent.ConcurrentHashMap;
import javax.sql.DataSource;
-import org.apache.baremaps.postgres.PostgresUtils;
+import org.apache.baremaps.utils.PostgresUtils;
/**
* A context that is passed to the tasks of a workflow and used to share data between tasks.
@@ -33,7 +33,7 @@ public class WorkflowContext {
* @return the data source
*/
public DataSource getDataSource(String database) {
- return dataSources.computeIfAbsent(database, d -> PostgresUtils.dataSource(d));
+ return dataSources.computeIfAbsent(database, d -> PostgresUtils.createDataSource(d));
}
}
diff --git a/baremaps-core/src/main/java/org/apache/baremaps/workflow/tasks/ExportVectorTiles.java b/baremaps-core/src/main/java/org/apache/baremaps/workflow/tasks/ExportVectorTiles.java
index 43040de8..b33377e5 100644
--- a/baremaps-core/src/main/java/org/apache/baremaps/workflow/tasks/ExportVectorTiles.java
+++ b/baremaps-core/src/main/java/org/apache/baremaps/workflow/tasks/ExportVectorTiles.java
@@ -16,8 +16,6 @@ import static org.apache.baremaps.utils.ObjectMapperUtils.objectMapper;
import com.fasterxml.jackson.core.JsonProcessingException;
import com.fasterxml.jackson.databind.ObjectMapper;
-import com.zaxxer.hikari.HikariConfig;
-import com.zaxxer.hikari.HikariDataSource;
import java.io.IOException;
import java.nio.file.Files;
import java.nio.file.Path;
@@ -30,8 +28,9 @@ import org.apache.baremaps.stream.ProgressLogger;
import org.apache.baremaps.stream.StreamUtils;
import org.apache.baremaps.tilestore.*;
import org.apache.baremaps.tilestore.file.FileTileStore;
-import org.apache.baremaps.tilestore.mbtiles.MBTiles;
+import org.apache.baremaps.tilestore.mbtiles.MBTilesStore;
import org.apache.baremaps.tilestore.postgres.PostgresTileStore;
+import org.apache.baremaps.utils.SqliteUtils;
import org.apache.baremaps.vectortile.tileset.Tileset;
import org.apache.baremaps.vectortile.tileset.TilesetQuery;
import org.apache.baremaps.workflow.Task;
@@ -39,12 +38,6 @@ import org.apache.baremaps.workflow.WorkflowContext;
import org.locationtech.jts.geom.Envelope;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import org.sqlite.SQLiteConfig;
-import org.sqlite.SQLiteConfig.JournalMode;
-import org.sqlite.SQLiteConfig.LockingMode;
-import org.sqlite.SQLiteConfig.SynchronousMode;
-import org.sqlite.SQLiteConfig.TempStore;
-import org.sqlite.SQLiteDataSource;
public record ExportVectorTiles(
Path tileset,
@@ -86,8 +79,8 @@ public record ExportVectorTiles(
private TileStore targetTileStore(Tileset source) throws TileStoreException, IOException {
if (mbtiles) {
Files.deleteIfExists(repository);
- var dataSource = createDataSource(repository);
- var tilesStore = new MBTiles(dataSource);
+ var dataSource = SqliteUtils.createDataSource(repository, false);
+ var tilesStore = new MBTilesStore(dataSource);
tilesStore.initializeDatabase();
tilesStore.writeMetadata(metadata(source));
return tilesStore;
@@ -138,29 +131,4 @@ public record ExportVectorTiles(
return metadata;
}
- /**
- * Create a SQLite data source.
- *
- * @param path the path to the SQLite database
- * @return the SQLite data source
- */
- public static DataSource createDataSource(Path path) {
- var sqliteConfig = new SQLiteConfig();
- sqliteConfig.setCacheSize(1000000);
- sqliteConfig.setPageSize(65536);
- sqliteConfig.setJournalMode(JournalMode.OFF);
- sqliteConfig.setLockingMode(LockingMode.EXCLUSIVE);
- sqliteConfig.setSynchronous(SynchronousMode.OFF);
- sqliteConfig.setTempStore(TempStore.MEMORY);
-
- var sqliteDataSource = new SQLiteDataSource();
- sqliteDataSource.setConfig(sqliteConfig);
- sqliteDataSource.setUrl("jdbc:sqlite:" + path);
-
- var hikariConfig = new HikariConfig();
- hikariConfig.setDataSource(sqliteDataSource);
- hikariConfig.setMaximumPoolSize(1);
-
- return new HikariDataSource(hikariConfig);
- }
}
diff --git a/baremaps-core/src/test/java/org/apache/baremaps/openstreetmap/postgres/PostgresRepositoryTest.java b/baremaps-core/src/test/java/org/apache/baremaps/openstreetmap/postgres/PostgresRepositoryTest.java
index 00a1f888..9da06fdf 100644
--- a/baremaps-core/src/test/java/org/apache/baremaps/openstreetmap/postgres/PostgresRepositoryTest.java
+++ b/baremaps-core/src/test/java/org/apache/baremaps/openstreetmap/postgres/PostgresRepositoryTest.java
@@ -18,8 +18,8 @@ import java.io.IOException;
import java.sql.Connection;
import java.sql.SQLException;
import javax.sql.DataSource;
-import org.apache.baremaps.postgres.PostgresUtils;
import org.apache.baremaps.testing.PostgresContainerTest;
+import org.apache.baremaps.utils.PostgresUtils;
import org.junit.jupiter.api.BeforeEach;
public abstract class PostgresRepositoryTest extends PostgresContainerTest {
diff --git a/baremaps-core/src/test/java/org/apache/baremaps/openstreetmap/repository/BlockImporterTest.java b/baremaps-core/src/test/java/org/apache/baremaps/openstreetmap/repository/BlockImporterTest.java
index 3d3d1440..3b645c43 100644
--- a/baremaps-core/src/test/java/org/apache/baremaps/openstreetmap/repository/BlockImporterTest.java
+++ b/baremaps-core/src/test/java/org/apache/baremaps/openstreetmap/repository/BlockImporterTest.java
@@ -27,9 +27,9 @@ import org.apache.baremaps.openstreetmap.postgres.PostgresHeaderRepository;
import org.apache.baremaps.openstreetmap.postgres.PostgresNodeRepository;
import org.apache.baremaps.openstreetmap.postgres.PostgresRelationRepository;
import org.apache.baremaps.openstreetmap.postgres.PostgresWayRepository;
-import org.apache.baremaps.postgres.PostgresUtils;
import org.apache.baremaps.testing.PostgresContainerTest;
import org.apache.baremaps.testing.TestFiles;
+import org.apache.baremaps.utils.PostgresUtils;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Tag;
import org.junit.jupiter.api.Test;
@@ -44,7 +44,7 @@ class BlockImporterTest extends PostgresContainerTest {
@BeforeEach
void init() throws SQLException, IOException {
- dataSource = PostgresUtils.dataSource(jdbcUrl(), 1);
+ dataSource = PostgresUtils.createDataSource(jdbcUrl(), 1);
headerRepository = new PostgresHeaderRepository(dataSource);
nodeRepository = new PostgresNodeRepository(dataSource);
tableRepository = new PostgresWayRepository(dataSource);
diff --git a/baremaps-core/src/test/java/org/apache/baremaps/postgres/metadata/PostgresMetadataTest.java b/baremaps-core/src/test/java/org/apache/baremaps/postgres/metadata/PostgresMetadataTest.java
index cf3e9367..411318ce 100644
--- a/baremaps-core/src/test/java/org/apache/baremaps/postgres/metadata/PostgresMetadataTest.java
+++ b/baremaps-core/src/test/java/org/apache/baremaps/postgres/metadata/PostgresMetadataTest.java
@@ -20,8 +20,8 @@ import java.sql.DatabaseMetaData;
import java.sql.DriverManager;
import java.sql.ResultSet;
import java.sql.SQLException;
-import org.apache.baremaps.postgres.PostgresUtils;
import org.apache.baremaps.testing.PostgresContainerTest;
+import org.apache.baremaps.utils.PostgresUtils;
import org.junit.jupiter.api.Tag;
import org.junit.jupiter.api.Test;
diff --git a/baremaps-core/src/test/java/org/apache/baremaps/tilestore/mbtiles/MBTilesTest.java b/baremaps-core/src/test/java/org/apache/baremaps/tilestore/mbtiles/MBTilesStoreTest.java
similarity index 90%
rename from baremaps-core/src/test/java/org/apache/baremaps/tilestore/mbtiles/MBTilesTest.java
rename to baremaps-core/src/test/java/org/apache/baremaps/tilestore/mbtiles/MBTilesStoreTest.java
index b677e5d5..f28154b4 100644
--- a/baremaps-core/src/test/java/org/apache/baremaps/tilestore/mbtiles/MBTilesTest.java
+++ b/baremaps-core/src/test/java/org/apache/baremaps/tilestore/mbtiles/MBTilesStoreTest.java
@@ -28,7 +28,7 @@ import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
import org.sqlite.SQLiteDataSource;
-class MBTilesTest extends TileStoreTest {
+class MBTilesStoreTest extends TileStoreTest {
Path file;
@@ -43,18 +43,18 @@ class MBTilesTest extends TileStoreTest {
}
@Override
- public MBTiles createTileStore() throws Exception {
+ public MBTilesStore createTileStore() throws Exception {
SQLiteDataSource dataSource = new SQLiteDataSource();
String url = "jdbc:sqlite:" + file.toAbsolutePath();
dataSource.setUrl(url);
- MBTiles tilesStore = new MBTiles(dataSource);
+ MBTilesStore tilesStore = new MBTilesStore(dataSource);
tilesStore.initializeDatabase();
return tilesStore;
}
@Test
void readWriteMetadata() throws Exception {
- MBTiles tileStore = createTileStore();
+ MBTilesStore tileStore = createTileStore();
Map<String, String> metadata = tileStore.readMetadata();
assertTrue(metadata.size() == 0);
diff --git a/baremaps-ogcapi/src/test/java/org/apache/baremaps/ogcapi/OgcApiTest.java b/baremaps-ogcapi/src/test/java/org/apache/baremaps/ogcapi/OgcApiTest.java
index 6cff8fa5..0b8b0dbd 100644
--- a/baremaps-ogcapi/src/test/java/org/apache/baremaps/ogcapi/OgcApiTest.java
+++ b/baremaps-ogcapi/src/test/java/org/apache/baremaps/ogcapi/OgcApiTest.java
@@ -20,9 +20,9 @@ import java.io.IOException;
import java.nio.file.Path;
import java.nio.file.Paths;
import javax.sql.DataSource;
-import org.apache.baremaps.postgres.PostgresUtils;
import org.apache.baremaps.tilestore.TileStore;
import org.apache.baremaps.tilestore.postgres.PostgresTileStore;
+import org.apache.baremaps.utils.PostgresUtils;
import org.apache.baremaps.vectortile.tileset.Tileset;
import org.glassfish.hk2.utilities.binding.AbstractBinder;
import org.glassfish.jersey.client.ClientConfig;
@@ -40,7 +40,7 @@ public abstract class OgcApiTest extends JerseyTest {
@Override
protected ResourceConfig configure() {
// Create a datasource to a throwaway postgis database
- dataSource = PostgresUtils.dataSource("jdbc:tc:postgis:13-3.1:///baremaps");
+ dataSource = PostgresUtils.createDataSource("jdbc:tc:postgis:13-3.1:///baremaps");
// Initialize the database
try (var connection = dataSource.getConnection()) {