You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@baremaps.apache.org by bc...@apache.org on 2023/09/07 15:26:15 UTC
[incubator-baremaps] 02/06: Implement header serialization and deserialization
This is an automated email from the ASF dual-hosted git repository.
bchapuis pushed a commit to branch pmtiles
in repository https://gitbox.apache.org/repos/asf/incubator-baremaps.git
commit ad8bd43eb805d125dfa2ca1ee37752dc0041e1fa
Author: Bertil Chapuis <bc...@gmail.com>
AuthorDate: Tue Sep 5 23:24:54 2023 +0200
Implement header serialization and deserialization
---
.../apache/baremaps/tilestore/pmtiles/PMTiles.java | 116 +++++++++++++++++++++
.../baremaps/tilestore/pmtiles/PMTilesTest.java | 73 +++++++++++++
.../src/test/resources/pmtiles/empty.pmtiles | 0
.../src/test/resources/pmtiles/invalid.pmtiles | 1 +
.../src/test/resources/pmtiles/invalid_v4.pmtiles | Bin 0 -> 468 bytes
.../test/resources/pmtiles/test_fixture_1.pmtiles | Bin 0 -> 468 bytes
.../test/resources/pmtiles/test_fixture_2.pmtiles | Bin 0 -> 466 bytes
7 files changed, 190 insertions(+)
diff --git a/baremaps-core/src/main/java/org/apache/baremaps/tilestore/pmtiles/PMTiles.java b/baremaps-core/src/main/java/org/apache/baremaps/tilestore/pmtiles/PMTiles.java
index c84b19b2..f773c7eb 100644
--- a/baremaps-core/src/main/java/org/apache/baremaps/tilestore/pmtiles/PMTiles.java
+++ b/baremaps-core/src/main/java/org/apache/baremaps/tilestore/pmtiles/PMTiles.java
@@ -3,6 +3,7 @@ package org.apache.baremaps.tilestore.pmtiles;
import com.google.common.math.LongMath;
import java.nio.ByteBuffer;
+import java.nio.ByteOrder;
public class PMTiles {
@@ -142,4 +143,119 @@ public class PMTiles {
}
throw new RuntimeException("Tile zoom level exceeds max safe number limit (26)");
}
+
+ enum Compression {
+ Unknown,
+ None,
+ Gzip,
+ Brotli,
+ Zstd,
+ }
+
+ enum TileType {
+ Unknown,
+ Mvt,
+ Png,
+ Jpeg,
+ Webp,
+ Avif,
+ }
+
+ public record Header(
+ int specVersion,
+ long rootDirectoryOffset,
+ long rootDirectoryLength,
+ long jsonMetadataOffset,
+ long jsonMetadataLength,
+ long leafDirectoryOffset,
+ long leafDirectoryLength,
+ long tileDataOffset,
+ long tileDataLength,
+ long numAddressedTiles,
+ long numTileEntries,
+ long numTileContents,
+ boolean clustered,
+ Compression internalCompression,
+ Compression tileCompression,
+ TileType tileType,
+ int minZoom,
+ int maxZoom,
+ double minLon,
+ double minLat,
+ double maxLon,
+ double maxLat,
+ int centerZoom,
+ double centerLon,
+ double centerLat,
+ String etag
+ ) {}
+
+ public static Header bytesToHeader(ByteBuffer buf, String etag) {
+ buf.order(ByteOrder.LITTLE_ENDIAN);
+ return new Header(
+ buf.get(7),
+ buf.getLong(8),
+ buf.getLong(16),
+ buf.getLong(24),
+ buf.getLong(32),
+ buf.getLong(40),
+ buf.getLong(48),
+ buf.getLong(56),
+ buf.getLong(64),
+ buf.getLong(72),
+ buf.getLong(80),
+ buf.getLong(88),
+ buf.get(96) == 1,
+ Compression.values()[buf.get(97)],
+ Compression.values()[buf.get(98)],
+ TileType.values()[buf.get(99)],
+ buf.get(100),
+ buf.get(101),
+ (double) buf.getInt(102) / 10000000,
+ (double) buf.getInt(106) / 10000000,
+ (double) buf.getInt(110) / 10000000,
+ (double) buf.getInt(114) / 10000000,
+ buf.get(118),
+ (double) buf.getInt(119) / 10000000,
+ (double) buf.getInt(123) / 10000000,
+ etag
+ );
+ }
+
+ public static void headerToBytes(Header header, ByteBuffer buf) {
+ buf.order(ByteOrder.LITTLE_ENDIAN);
+ buf.put(0, (byte) 0x4d);
+ buf.put(1, (byte) 0x42);
+ buf.put(2, (byte) 0x54);
+ buf.put(3, (byte) 0x42);
+ buf.put(4, (byte) 0x49);
+ buf.put(5, (byte) 0x4e);
+ buf.put(6, (byte) 0x41);
+ buf.put(7, (byte) header.specVersion);
+ buf.putLong(8, header.rootDirectoryOffset);
+ buf.putLong(16, header.rootDirectoryLength);
+ buf.putLong(24, header.jsonMetadataOffset);
+ buf.putLong(32, header.jsonMetadataLength);
+ buf.putLong(40, header.leafDirectoryOffset);
+ buf.putLong(48, header.leafDirectoryLength);
+ buf.putLong(56, header.tileDataOffset);
+ buf.putLong(64, header.tileDataLength);
+ buf.putLong(72, header.numAddressedTiles);
+ buf.putLong(80, header.numTileEntries);
+ buf.putLong(88, header.numTileContents);
+ buf.put(96, (byte) (header.clustered ? 1 : 0));
+ buf.put(97, (byte) header.internalCompression.ordinal());
+ buf.put(98, (byte) header.tileCompression.ordinal());
+ buf.put(99, (byte) header.tileType.ordinal());
+ buf.put(100, (byte) header.minZoom);
+ buf.put(101, (byte) header.maxZoom);
+ buf.putInt(102, (int) (header.minLon * 10000000));
+ buf.putInt(106, (int) (header.minLat * 10000000));
+ buf.putInt(110, (int) (header.maxLon * 10000000));
+ buf.putInt(114, (int) (header.maxLat * 10000000));
+ buf.put(118, (byte) header.centerZoom);
+ buf.putInt(119, (int) (header.centerLon * 10000000));
+ buf.putInt(123, (int) (header.centerLat * 10000000));
+ }
+
}
diff --git a/baremaps-core/src/test/java/org/apache/baremaps/tilestore/pmtiles/PMTilesTest.java b/baremaps-core/src/test/java/org/apache/baremaps/tilestore/pmtiles/PMTilesTest.java
index c119a882..3eda409b 100644
--- a/baremaps-core/src/test/java/org/apache/baremaps/tilestore/pmtiles/PMTilesTest.java
+++ b/baremaps-core/src/test/java/org/apache/baremaps/tilestore/pmtiles/PMTilesTest.java
@@ -1,9 +1,14 @@
package org.apache.baremaps.tilestore.pmtiles;
import com.google.common.math.LongMath;
+import org.apache.baremaps.testing.TestFiles;
+import org.apache.baremaps.tilestore.pmtiles.PMTiles.Compression;
+import org.apache.baremaps.tilestore.pmtiles.PMTiles.TileType;
import org.junit.jupiter.api.Test;
+import java.io.IOException;
import java.nio.ByteBuffer;
+import java.nio.file.Files;
import static org.junit.jupiter.api.Assertions.*;
@@ -85,4 +90,72 @@ class PMTilesTest {
assertThrows(RuntimeException.class, () -> PMTiles.zxyToTileId(0, 1, 1));
}
+ @Test
+ void bytesToHeader() throws IOException {
+ var file = TestFiles.resolve("pmtiles/test_fixture_1.pmtiles");
+ try (var channel = Files.newByteChannel(file)) {
+ var buffer = ByteBuffer.allocate(127);
+ channel.read(buffer);
+ buffer.flip();
+ var header = PMTiles.bytesToHeader(buffer, "1");
+ assertEquals(header.rootDirectoryOffset(), 127);
+ assertEquals(header.rootDirectoryLength(), 25);
+ assertEquals(header.jsonMetadataOffset(), 152);
+ assertEquals(header.jsonMetadataLength(), 247);
+ assertEquals(header.leafDirectoryOffset(), 0);
+ assertEquals(header.leafDirectoryLength(), 0);
+ assertEquals(header.tileDataOffset(), 399);
+ assertEquals(header.tileDataLength(), 69);
+ assertEquals(header.numAddressedTiles(), 1);
+ assertEquals(header.numTileEntries(), 1);
+ assertEquals(header.numTileContents(), 1);
+ assertFalse(header.clustered());
+ assertEquals(header.internalCompression(), Compression.Gzip);
+ assertEquals(header.tileCompression(), Compression.Gzip);
+ assertEquals(header.tileType(), TileType.Mvt);
+ assertEquals(header.minZoom(), 0);
+ assertEquals(header.maxZoom(), 0);
+ assertEquals(header.minLon(), 0);
+ assertEquals(header.minLat(), 0);
+ assertEquals(Math.round(header.maxLon()), 1);
+ assertEquals(Math.round(header.maxLat()), 1);
+ }
+ }
+
+ @Test
+ void headerToBytes() throws IOException {
+ var etag = "1";
+ var buffer = ByteBuffer.allocate(127);
+ var header = new PMTiles.Header(
+ 127,
+ 25,
+ 152,
+ 247,
+ 0,
+ 0,
+ 399,
+ 69,
+ 1,
+ 1,
+ 1,
+ 10,
+ false,
+ Compression.Gzip,
+ Compression.Gzip,
+ TileType.Mvt,
+ 0,
+ 0,
+ 0,
+ 1,
+ 1,
+ 0,
+ 0,
+ 0,
+ 0,
+ etag);
+ PMTiles.headerToBytes(header, buffer);
+ var header2 = PMTiles.bytesToHeader(buffer, etag);
+ assertEquals(header, header2);
+ }
+
}
\ No newline at end of file
diff --git a/baremaps-core/src/test/resources/pmtiles/empty.pmtiles b/baremaps-core/src/test/resources/pmtiles/empty.pmtiles
new file mode 100644
index 00000000..e69de29b
diff --git a/baremaps-core/src/test/resources/pmtiles/invalid.pmtiles b/baremaps-core/src/test/resources/pmtiles/invalid.pmtiles
new file mode 100644
index 00000000..f2b720ba
--- /dev/null
+++ b/baremaps-core/src/test/resources/pmtiles/invalid.pmtiles
@@ -0,0 +1 @@
+This is an invalid tile archive, a test case to make sure that the code throws an error, but it needs to be the minimum size to pass the first test
diff --git a/baremaps-core/src/test/resources/pmtiles/invalid_v4.pmtiles b/baremaps-core/src/test/resources/pmtiles/invalid_v4.pmtiles
new file mode 100644
index 00000000..1871cb27
Binary files /dev/null and b/baremaps-core/src/test/resources/pmtiles/invalid_v4.pmtiles differ
diff --git a/baremaps-core/src/test/resources/pmtiles/test_fixture_1.pmtiles b/baremaps-core/src/test/resources/pmtiles/test_fixture_1.pmtiles
new file mode 100644
index 00000000..c86db1f2
Binary files /dev/null and b/baremaps-core/src/test/resources/pmtiles/test_fixture_1.pmtiles differ
diff --git a/baremaps-core/src/test/resources/pmtiles/test_fixture_2.pmtiles b/baremaps-core/src/test/resources/pmtiles/test_fixture_2.pmtiles
new file mode 100644
index 00000000..cb19dd5f
Binary files /dev/null and b/baremaps-core/src/test/resources/pmtiles/test_fixture_2.pmtiles differ