You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@baremaps.apache.org by bc...@apache.org on 2023/06/23 13:17:29 UTC
[incubator-baremaps] 02/11: Prepare the Table and Store abstractions and remove collection tasks
This is an automated email from the ASF dual-hosted git repository.
bchapuis pushed a commit to branch calcite
in repository https://gitbox.apache.org/repos/asf/incubator-baremaps.git
commit e1f18b12b855c151eb0a16c67cca2821681fde92
Author: Bertil Chapuis <bc...@gmail.com>
AuthorDate: Tue Jun 20 11:58:49 2023 +0200
Prepare the Table and Store abstractions and remove collection tasks
---
baremaps-core/pom.xml | 1 -
.../java/org/apache/baremaps/calcite/Calcite.java | 116 ++++++++++++++++
.../collection/algorithm/ExternalMergeSort.java | 4 +-
.../baremaps/collection/type/BooleanDataType.java | 38 ++++++
.../collection/type/geometry/GeometryDataType.java | 2 +-
.../collection/type/geometry/LonLatDataType.java | 67 +++++++++
.../collection/type/geometry/WKBDataType.java | 54 ++++++++
.../main/java/org/apache/baremaps/storage/Row.java | 16 +++
.../org/apache/baremaps/storage/RowDataType.java | 85 ++++++++++++
.../java/org/apache/baremaps/storage/RowImpl.java | 13 ++
.../org/apache/baremaps/storage/SchemaImpl.java | 29 +++-
.../java/org/apache/baremaps/workflow/Task.java | 2 -
.../workflow/tasks/CreateEntityCollection.java | 93 -------------
.../org/apache/baremaps/workflow/tasks/Entity.java | 83 ------------
.../baremaps/workflow/tasks/EntityDataType.java | 66 ---------
.../workflow/tasks/ImportOpenStreetMap.java | 2 +-
.../workflow/tasks/TransformEntityCollection.java | 150 ---------------------
.../baremaps/collection/type/DataTypeProvider.java | 9 ++
.../baremaps/tilestore/file/FileTileStoreTest.java | 2 +-
.../baremaps/vectortile/ExpressionsTest.java | 33 +++--
.../CreateAndTransformEntityCollectionTest.java | 48 -------
21 files changed, 455 insertions(+), 458 deletions(-)
diff --git a/baremaps-core/pom.xml b/baremaps-core/pom.xml
index 3b370c4b..b23bb1f3 100644
--- a/baremaps-core/pom.xml
+++ b/baremaps-core/pom.xml
@@ -84,7 +84,6 @@
<dependency>
<groupId>org.apache.calcite</groupId>
<artifactId>calcite-core</artifactId>
- <version>${version.lib.calcite}</version>
</dependency>
<dependency>
<groupId>org.apache.commons</groupId>
diff --git a/baremaps-core/src/main/java/org/apache/baremaps/calcite/Calcite.java b/baremaps-core/src/main/java/org/apache/baremaps/calcite/Calcite.java
new file mode 100644
index 00000000..300f9af4
--- /dev/null
+++ b/baremaps-core/src/main/java/org/apache/baremaps/calcite/Calcite.java
@@ -0,0 +1,116 @@
+/*
+ * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except
+ * in compliance with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software distributed under the License
+ * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
+ * or implied. See the License for the specific language governing permissions and limitations under
+ * the License.
+ */
+
+package org.apache.baremaps.calcite;
+
+import java.sql.Connection;
+import java.sql.DriverManager;
+import java.sql.ResultSet;
+import java.sql.SQLException;
+import java.util.Arrays;
+import java.util.List;
+import java.util.Properties;
+import org.apache.calcite.DataContext;
+import org.apache.calcite.jdbc.CalciteConnection;
+import org.apache.calcite.jdbc.JavaTypeFactoryImpl;
+import org.apache.calcite.linq4j.Enumerable;
+import org.apache.calcite.linq4j.Linq4j;
+import org.apache.calcite.rel.type.RelDataType;
+import org.apache.calcite.rel.type.RelDataTypeFactory;
+import org.apache.calcite.schema.ScannableTable;
+import org.apache.calcite.schema.SchemaPlus;
+import org.apache.calcite.schema.impl.AbstractTable;
+import org.apache.calcite.sql.type.SqlTypeName;
+
+public class Calcite {
+
+ public static final List<Object[]> PLAYER_DATA_AS_OBJECT_ARRAY = Arrays.asList(
+ new Object[] {1, "Wizard", 5},
+ new Object[] {2, "Hunter", 7}
+
+ );
+ public static final List<Object[]> EQUIPMENT_DATA_AS_OBJECT_ARRAY = Arrays.asList(
+ new Object[] {1, "fireball", 7, 1},
+ new Object[] {2, "rifle", 4, 2});
+
+
+ public static void main(String[] args) throws SQLException {
+
+
+ RelDataTypeFactory typeFactory = new JavaTypeFactoryImpl();
+ RelDataTypeFactory.Builder playerType = new RelDataTypeFactory.Builder(typeFactory);
+
+ Properties info = new Properties();
+ // https://calcite.apache.org/javadocAggregate/org/apache/calcite/config/Lex.html#JAVA
+ info.setProperty("lex", "MYSQL");
+
+ Connection connection = DriverManager.getConnection("jdbc:calcite:", info);
+ CalciteConnection calciteConnection =
+ connection.unwrap(CalciteConnection.class);
+
+ SchemaPlus rootSchema = calciteConnection.getRootSchema();
+
+ playerType.add("id", SqlTypeName.INTEGER);
+ playerType.add("name", SqlTypeName.VARCHAR);
+ playerType.add("level", SqlTypeName.INTEGER);
+
+ ListTable playerTable = new ListTable(playerType.build(), PLAYER_DATA_AS_OBJECT_ARRAY);
+ rootSchema.add("player", playerTable);
+
+ RelDataTypeFactory.Builder equipmentType = new RelDataTypeFactory.Builder(typeFactory);
+
+ equipmentType.add("id", SqlTypeName.INTEGER);
+ equipmentType.add("name", SqlTypeName.VARCHAR);
+ equipmentType.add("damage", SqlTypeName.INTEGER);
+ equipmentType.add("player_id", SqlTypeName.INTEGER);
+
+ ListTable equipmentTable = new ListTable(equipmentType.build(), EQUIPMENT_DATA_AS_OBJECT_ARRAY);
+ rootSchema.add("equipment", equipmentTable);
+
+
+ String sql =
+ "SELECT player.name, equipment.name FROM player INNER JOIN equipment ON player.id = equipment.player_id ";
+ ResultSet resultSet = connection.createStatement().executeQuery(sql);
+ StringBuilder b = new StringBuilder();
+ while (resultSet.next()) {
+ b.append(resultSet.getString(1)).append(" attacks with ");
+ b.append(resultSet.getString(2)).append(" !\n");
+ }
+ System.out.println(b);
+
+
+ resultSet.close();
+ }
+
+ /**
+ * A simple table based on a list.
+ */
+ private static class ListTable extends AbstractTable implements ScannableTable {
+ private final RelDataType rowType;
+ private final List<Object[]> data;
+
+ ListTable(RelDataType rowType, List<Object[]> data) {
+ this.rowType = rowType;
+ this.data = data;
+ }
+
+ @Override
+ public Enumerable<Object[]> scan(final DataContext root) {
+ return Linq4j.asEnumerable(data);
+ }
+
+ @Override
+ public RelDataType getRowType(final RelDataTypeFactory typeFactory) {
+ return rowType;
+ }
+ }
+}
diff --git a/baremaps-core/src/main/java/org/apache/baremaps/collection/algorithm/ExternalMergeSort.java b/baremaps-core/src/main/java/org/apache/baremaps/collection/algorithm/ExternalMergeSort.java
index cd8190c1..4a86efb9 100644
--- a/baremaps-core/src/main/java/org/apache/baremaps/collection/algorithm/ExternalMergeSort.java
+++ b/baremaps-core/src/main/java/org/apache/baremaps/collection/algorithm/ExternalMergeSort.java
@@ -125,8 +125,8 @@ public class ExternalMergeSort {
}
}
- for (DataList<T> DataStore : batches) {
- DataStore.clear();
+ for (DataList<T> batch : batches) {
+ batch.clear();
}
return counter;
diff --git a/baremaps-core/src/main/java/org/apache/baremaps/collection/type/BooleanDataType.java b/baremaps-core/src/main/java/org/apache/baremaps/collection/type/BooleanDataType.java
new file mode 100644
index 00000000..759ecf68
--- /dev/null
+++ b/baremaps-core/src/main/java/org/apache/baremaps/collection/type/BooleanDataType.java
@@ -0,0 +1,38 @@
+/*
+ * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except
+ * in compliance with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software distributed under the License
+ * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
+ * or implied. See the License for the specific language governing permissions and limitations under
+ * the License.
+ */
+
+package org.apache.baremaps.collection.type;
+
+
+
+import java.nio.ByteBuffer;
+
+/** A {@link DataType} for reading and writing bytes in {@link ByteBuffer}s. */
+public class BooleanDataType extends MemoryAlignedDataType<Boolean> {
+
+ /** Constructs a {@link ByteDataType}. */
+ public BooleanDataType() {
+ super(Byte.BYTES);
+ }
+
+ /** {@inheritDoc} */
+ @Override
+ public void write(ByteBuffer buffer, int position, Boolean value) {
+ buffer.put(position, value ? (byte) 1 : (byte) 0);
+ }
+
+ /** {@inheritDoc} */
+ @Override
+ public Boolean read(ByteBuffer buffer, int position) {
+ return buffer.get(position) == 1;
+ }
+}
diff --git a/baremaps-core/src/main/java/org/apache/baremaps/collection/type/geometry/GeometryDataType.java b/baremaps-core/src/main/java/org/apache/baremaps/collection/type/geometry/GeometryDataType.java
index 376e82de..475ec700 100644
--- a/baremaps-core/src/main/java/org/apache/baremaps/collection/type/geometry/GeometryDataType.java
+++ b/baremaps-core/src/main/java/org/apache/baremaps/collection/type/geometry/GeometryDataType.java
@@ -18,7 +18,7 @@ import org.apache.baremaps.collection.type.DataType;
import org.locationtech.jts.geom.*;
/**
- * A {@code DataType} for {@code MultiPolygon} objects.
+ * A {@code DataType} for {@link Geometry} objects.
*/
public class GeometryDataType implements DataType<Geometry> {
diff --git a/baremaps-core/src/main/java/org/apache/baremaps/collection/type/geometry/LonLatDataType.java b/baremaps-core/src/main/java/org/apache/baremaps/collection/type/geometry/LonLatDataType.java
new file mode 100644
index 00000000..9e2ceb67
--- /dev/null
+++ b/baremaps-core/src/main/java/org/apache/baremaps/collection/type/geometry/LonLatDataType.java
@@ -0,0 +1,67 @@
+/*
+ * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except
+ * in compliance with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software distributed under the License
+ * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
+ * or implied. See the License for the specific language governing permissions and limitations under
+ * the License.
+ */
+
+package org.apache.baremaps.collection.type.geometry;
+
+
+
+import java.nio.ByteBuffer;
+import org.apache.baremaps.collection.type.DataType;
+import org.apache.baremaps.collection.type.MemoryAlignedDataType;
+import org.locationtech.jts.geom.Coordinate;
+
+/**
+ * A {@link DataType} for reading and writing longitude/latitude coordinates in {@link ByteBuffer}s.
+ * An integer is used to compress the coordinates to the detriment of precision (centimeters).
+ */
+public class LonLatDataType extends MemoryAlignedDataType<Coordinate> {
+
+ private static final double BITS = Math.pow(2, 31);
+ private static final long SHIFT = 32;
+ private static final long MASK = (1L << 32) - 1L;
+
+ /** Constructs a {@link LonLatDataType}. */
+ public LonLatDataType() {
+ super(Long.BYTES);
+ }
+
+ public static long encodeLonLat(double lon, double lat) {
+ long x = (long) (((lon + 180) / 360) * BITS);
+ long y = (long) (((lat + 90) / 180) * BITS);
+ long l = (x << SHIFT);
+ long r = (y & MASK);
+ return l | r;
+ }
+
+ public static double decodeLon(long value) {
+ double l = (value >>> 32);
+ return (l / BITS) * 360 - 180;
+ }
+
+ public static double decodeLat(long value) {
+ long r = (value & MASK);
+ return (r / BITS) * 180 - 90;
+ }
+
+ /** {@inheritDoc} */
+ @Override
+ public void write(ByteBuffer buffer, int position, Coordinate value) {
+ buffer.putLong(position, encodeLonLat(value.x, value.y));
+ }
+
+ /** {@inheritDoc} */
+ @Override
+ public Coordinate read(ByteBuffer buffer, int position) {
+ var value = buffer.getLong(position);
+ return new Coordinate(decodeLon(value), decodeLat(value));
+ }
+}
diff --git a/baremaps-core/src/main/java/org/apache/baremaps/collection/type/geometry/WKBDataType.java b/baremaps-core/src/main/java/org/apache/baremaps/collection/type/geometry/WKBDataType.java
new file mode 100644
index 00000000..2b4d9a91
--- /dev/null
+++ b/baremaps-core/src/main/java/org/apache/baremaps/collection/type/geometry/WKBDataType.java
@@ -0,0 +1,54 @@
+/*
+ * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except
+ * in compliance with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software distributed under the License
+ * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
+ * or implied. See the License for the specific language governing permissions and limitations under
+ * the License.
+ */
+
+package org.apache.baremaps.collection.type.geometry;
+
+
+
+import java.nio.ByteBuffer;
+import org.apache.baremaps.collection.type.DataType;
+import org.apache.baremaps.utils.GeometryUtils;
+import org.locationtech.jts.geom.Geometry;
+
+/** A {@link DataType} for reading and writing {@link Geometry} in {@link ByteBuffer}s. */
+public class WKBDataType implements DataType<Geometry> {
+
+ /** {@inheritDoc} */
+ @Override
+ public int size(Geometry value) {
+ byte[] bytes = GeometryUtils.serialize(value);
+ return Integer.BYTES + bytes.length;
+ }
+
+ /** {@inheritDoc} */
+ @Override
+ public int size(ByteBuffer buffer, int position) {
+ return buffer.getInt(position);
+ }
+
+ /** {@inheritDoc} */
+ @Override
+ public void write(ByteBuffer buffer, int position, Geometry value) {
+ byte[] bytes = GeometryUtils.serialize(value);
+ buffer.putInt(position, Integer.BYTES + bytes.length);
+ buffer.put(position + Integer.BYTES, bytes);
+ }
+
+ /** {@inheritDoc} */
+ @Override
+ public Geometry read(ByteBuffer buffer, int position) {
+ int size = buffer.getInt(position);
+ byte[] bytes = new byte[Math.max(size - Integer.BYTES, 0)];
+ buffer.get(position + Integer.BYTES, bytes);
+ return GeometryUtils.deserialize(bytes);
+ }
+}
diff --git a/baremaps-core/src/main/java/org/apache/baremaps/storage/Row.java b/baremaps-core/src/main/java/org/apache/baremaps/storage/Row.java
index 3de90fbf..63bb5582 100644
--- a/baremaps-core/src/main/java/org/apache/baremaps/storage/Row.java
+++ b/baremaps-core/src/main/java/org/apache/baremaps/storage/Row.java
@@ -41,6 +41,14 @@ public interface Row {
*/
Object get(String column);
+ /**
+ * Returns the value of the specified column.
+ *
+ * @param index the index of the column
+ * @return the value of the specified column
+ */
+ Object get(int index);
+
/**
* Sets the value of the specified column.
*
@@ -49,4 +57,12 @@ public interface Row {
*/
void set(String column, Object value);
+ /**
+ * Sets the value of the specified column.
+ *
+ * @param index the index of the column
+ * @param value the value
+ */
+ void set(int index, Object value);
+
}
diff --git a/baremaps-core/src/main/java/org/apache/baremaps/storage/RowDataType.java b/baremaps-core/src/main/java/org/apache/baremaps/storage/RowDataType.java
new file mode 100644
index 00000000..9e13c5b3
--- /dev/null
+++ b/baremaps-core/src/main/java/org/apache/baremaps/storage/RowDataType.java
@@ -0,0 +1,85 @@
+/*
+ * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except
+ * in compliance with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software distributed under the License
+ * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
+ * or implied. See the License for the specific language governing permissions and limitations under
+ * the License.
+ */
+
+package org.apache.baremaps.storage;
+
+import java.nio.ByteBuffer;
+import java.util.ArrayList;
+import java.util.Map;
+import org.apache.baremaps.collection.type.*;
+
+public class RowDataType implements DataType<Row> {
+
+ private static final Map<Class, DataType> types = Map.of(
+ Byte.class, new ByteDataType(),
+ Boolean.class, new BooleanDataType(),
+ Short.class, new ShortDataType(),
+ Integer.class, new IntegerDataType(),
+ Long.class, new LongDataType(),
+ Float.class, new FloatDataType(),
+ Double.class, new DoubleDataType(),
+ String.class, new StringDataType());
+
+ private final Schema schema;
+
+ public RowDataType(Schema schema) {
+ this.schema = schema;
+ }
+
+ @Override
+ public int size(Row row) {
+ var size = Integer.BYTES;
+ var columns = schema.columns();
+ for (int i = 0; i < columns.size(); i++) {
+ var columnType = columns.get(i).type();
+ var dataType = types.get(columnType);
+ var value = row.get(i);
+ size += dataType.size(value);
+ }
+ return size;
+ }
+
+ @Override
+ public int size(ByteBuffer buffer, int position) {
+ return buffer.getInt(position);
+ }
+
+ @Override
+ public void write(final ByteBuffer buffer, final int position, final Row row) {
+ var p = position + Integer.BYTES;
+ var columns = schema.columns();
+ for (int i = 0; i < columns.size(); i++) {
+ var column = columns.get(i);
+ var columnType = column.type();
+ var dataType = types.get(columnType);
+ var value = row.get(i);
+ dataType.write(buffer, position, value);
+ p += dataType.size(buffer, position);
+ }
+ buffer.putInt(position, p - position);
+ }
+
+ @Override
+ public Row read(final ByteBuffer buffer, final int position) {
+ var p = position + Integer.BYTES;
+ var columns = schema.columns();
+ var values = new ArrayList();
+ for (int i = 0; i < columns.size(); i++) {
+ var column = columns.get(i);
+ var columnType = column.type();
+ var dataType = types.get(columnType);
+ values.add(dataType.read(buffer, p));
+ p += dataType.size(buffer, p);
+ }
+ return new RowImpl(schema, values);
+ }
+}
diff --git a/baremaps-core/src/main/java/org/apache/baremaps/storage/RowImpl.java b/baremaps-core/src/main/java/org/apache/baremaps/storage/RowImpl.java
index fd2bc4ad..f3afcf14 100644
--- a/baremaps-core/src/main/java/org/apache/baremaps/storage/RowImpl.java
+++ b/baremaps-core/src/main/java/org/apache/baremaps/storage/RowImpl.java
@@ -32,6 +32,14 @@ public record RowImpl(Schema schema, List values) implements Row {
throw new IllegalArgumentException("Column " + column + " not found.");
}
+ /**
+ * {@inheritDoc}
+ */
+ @Override
+ public Object get(int index) {
+ return values.get(index);
+ }
+
/**
* {@inheritDoc}
*/
@@ -46,4 +54,9 @@ public record RowImpl(Schema schema, List values) implements Row {
throw new IllegalArgumentException("Column " + column + " not found.");
}
+ @Override
+ public void set(int index, Object value) {
+ values.set(index, value);
+ }
+
}
diff --git a/baremaps-core/src/main/java/org/apache/baremaps/storage/SchemaImpl.java b/baremaps-core/src/main/java/org/apache/baremaps/storage/SchemaImpl.java
index 67bba9c8..b3b1bcde 100644
--- a/baremaps-core/src/main/java/org/apache/baremaps/storage/SchemaImpl.java
+++ b/baremaps-core/src/main/java/org/apache/baremaps/storage/SchemaImpl.java
@@ -13,12 +13,39 @@
package org.apache.baremaps.storage;
import java.util.ArrayList;
+import java.util.HashMap;
import java.util.List;
+import java.util.Map;
/**
* A schema defines the structure of a table.
*/
-public record SchemaImpl(String name, List<Column> columns) implements Schema {
+public class SchemaImpl implements Schema {
+
+ private final String name;
+
+ private final List<Column> columns;
+
+ private final Map<String, Integer> index;
+
+ public SchemaImpl(String name, List<Column> columns) {
+ this.name = name;
+ this.columns = columns;
+ this.index = new HashMap<>();
+ for (int i = 0; i < columns.size(); i++) {
+ index.put(columns.get(i).name(), i);
+ }
+ }
+
+ @Override
+ public String name() {
+ return name;
+ }
+
+ @Override
+ public List<Column> columns() {
+ return columns;
+ }
/**
* {@inheritDoc}
diff --git a/baremaps-core/src/main/java/org/apache/baremaps/workflow/Task.java b/baremaps-core/src/main/java/org/apache/baremaps/workflow/Task.java
index 7a5c919f..65676fef 100644
--- a/baremaps-core/src/main/java/org/apache/baremaps/workflow/Task.java
+++ b/baremaps-core/src/main/java/org/apache/baremaps/workflow/Task.java
@@ -39,8 +39,6 @@ import org.apache.baremaps.workflow.tasks.*;
@JsonSubTypes.Type(value = UngzipFile.class, name = "UngzipFile"),
@JsonSubTypes.Type(value = UpdateOpenStreetMap.class, name = "UpdateOpenStreetMap"),
@JsonSubTypes.Type(value = CreateGeonamesIndex.class, name = "CreateGeonamesIndex"),
- @JsonSubTypes.Type(value = CreateEntityCollection.class, name = "CreateEntityCollection"),
- @JsonSubTypes.Type(value = TransformEntityCollection.class, name = "TransformEntityCollection"),
@JsonSubTypes.Type(value = CreateIplocIndex.class, name = "CreateIplocIndex")})
public interface Task {
diff --git a/baremaps-core/src/main/java/org/apache/baremaps/workflow/tasks/CreateEntityCollection.java b/baremaps-core/src/main/java/org/apache/baremaps/workflow/tasks/CreateEntityCollection.java
deleted file mode 100644
index e7976bf9..00000000
--- a/baremaps-core/src/main/java/org/apache/baremaps/workflow/tasks/CreateEntityCollection.java
+++ /dev/null
@@ -1,93 +0,0 @@
-/*
- * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except
- * in compliance with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software distributed under the License
- * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
- * or implied. See the License for the specific language governing permissions and limitations under
- * the License.
- */
-
-package org.apache.baremaps.workflow.tasks;
-
-
-import java.nio.file.Files;
-import java.nio.file.Path;
-import java.nio.file.Paths;
-import java.util.HashMap;
-import java.util.stream.Collectors;
-import org.apache.baremaps.collection.AppendOnlyBuffer;
-import org.apache.baremaps.collection.MemoryAlignedDataList;
-import org.apache.baremaps.collection.MonotonicDataMap;
-import org.apache.baremaps.collection.memory.MemoryMappedFile;
-import org.apache.baremaps.collection.type.*;
-import org.apache.baremaps.openstreetmap.model.*;
-import org.apache.baremaps.openstreetmap.pbf.PbfEntityReader;
-import org.apache.baremaps.utils.FileUtils;
-import org.apache.baremaps.utils.ProjectionTransformer;
-import org.apache.baremaps.workflow.Task;
-import org.apache.baremaps.workflow.WorkflowContext;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-
-public record CreateEntityCollection(Path file, Path collection,
- Integer srid) implements Task {
-
- private static final Logger logger = LoggerFactory.getLogger(ImportOpenStreetMap.class);
-
- @Override
- public void execute(WorkflowContext context) throws Exception {
- var path = file.toAbsolutePath();
- var cacheDir = Files.createTempDirectory(Paths.get("."), "cache_");
-
- var coordinatesKeysFile = Files.createFile(cacheDir.resolve("coordinates_keys"));
- var coordinatesValsFile = Files.createFile(cacheDir.resolve("coordinates_vals"));
- var coordinateMap = new MonotonicDataMap<>(
- new MemoryAlignedDataList<>(new PairDataType<>(new LongDataType(), new LongDataType()),
- new MemoryMappedFile(coordinatesKeysFile)),
- new AppendOnlyBuffer<>(new LonLatDataType(), new MemoryMappedFile(coordinatesValsFile)));
-
- var referencesKeysFile = Files.createFile(cacheDir.resolve("references_keys"));
- var referencesValuesFile = Files.createFile(cacheDir.resolve("references_vals"));
- var referenceMap = new MonotonicDataMap<>(
- new MemoryAlignedDataList<>(new PairDataType<>(new LongDataType(), new LongDataType()),
- new MemoryMappedFile(referencesKeysFile)),
- new AppendOnlyBuffer<>(new LongListDataType(), new MemoryMappedFile(referencesValuesFile)));
-
- Files.deleteIfExists(collection);
-
- var entityCollection =
- new AppendOnlyBuffer<>(new EntityDataType(), new MemoryMappedFile(collection));
-
- var projectionTransformer = new ProjectionTransformer(4326, srid);
-
- // Read the PBF file and dispatch the elements to the consumers
- new PbfEntityReader()
- .geometries(true)
- .coordinateMap(coordinateMap)
- .referenceMap(referenceMap)
- .stream(Files.newInputStream(path))
- .filter(Element.class::isInstance)
- .map(Element.class::cast)
- .filter(element -> !element.getTags().isEmpty())
- .filter(element -> element.getGeometry() != null)
- .map(element -> {
- var geometry = element.getGeometry();
- geometry = projectionTransformer.transform(geometry);
- var tags = new HashMap<String, String>();
- for (var tag : element.getTags().entrySet()) {
- tags.put(tag.getKey(), tag.getValue().toString());
- }
- return new Entity(element.id(), tags, geometry);
- })
- .collect(Collectors.toCollection(() -> entityCollection));
-
- entityCollection.close();
-
- FileUtils.deleteRecursively(cacheDir);
- }
-
-}
diff --git a/baremaps-core/src/main/java/org/apache/baremaps/workflow/tasks/Entity.java b/baremaps-core/src/main/java/org/apache/baremaps/workflow/tasks/Entity.java
deleted file mode 100644
index 05d60f5c..00000000
--- a/baremaps-core/src/main/java/org/apache/baremaps/workflow/tasks/Entity.java
+++ /dev/null
@@ -1,83 +0,0 @@
-/*
- * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except
- * in compliance with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software distributed under the License
- * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
- * or implied. See the License for the specific language governing permissions and limitations under
- * the License.
- */
-
-package org.apache.baremaps.workflow.tasks;
-
-import java.util.ArrayList;
-import java.util.List;
-import java.util.Map;
-import org.apache.baremaps.storage.*;
-import org.locationtech.jts.geom.Geometry;
-
-public class Entity implements Row {
-
- private long id;
-
- private Map<String, String> tags;
-
- private Geometry geometry;
-
- public Entity(long id, Map<String, String> tags, Geometry geometry) {
- this.id = id;
- this.tags = tags;
- this.geometry = geometry;
- }
-
-
- @Override
- public Schema schema() {
- var columns = new ArrayList<Column>();
- columns.add(new ColumnImpl("id", Long.class));
- columns.add(new ColumnImpl("geometry", Geometry.class));
- for (var entry : tags.entrySet()) {
- columns.add(new ColumnImpl(entry.getKey(), String.class));
- }
- return new SchemaImpl("entity", columns);
- }
-
- @Override
- public void set(String column, Object value) {
- tags.put(column, value.toString());
- }
-
- @Override
- public Object get(String column) {
- if (column.equals("id")) {
- return id;
- } else if (column.equals("geometry")) {
- return geometry;
- } else {
- return tags.get(column);
- }
- }
-
- @Override
- public List<Object> values() {
- var map = new ArrayList();
- map.add(id);
- map.add(geometry);
- map.addAll(tags.values());
- return map;
- }
-
- public long getId() {
- return id;
- }
-
- public Map<String, String> getTags() {
- return tags;
- }
-
- public Geometry getGeometry() {
- return geometry;
- }
-}
diff --git a/baremaps-core/src/main/java/org/apache/baremaps/workflow/tasks/EntityDataType.java b/baremaps-core/src/main/java/org/apache/baremaps/workflow/tasks/EntityDataType.java
deleted file mode 100644
index 027762d3..00000000
--- a/baremaps-core/src/main/java/org/apache/baremaps/workflow/tasks/EntityDataType.java
+++ /dev/null
@@ -1,66 +0,0 @@
-/*
- * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except
- * in compliance with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software distributed under the License
- * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
- * or implied. See the License for the specific language governing permissions and limitations under
- * the License.
- */
-
-package org.apache.baremaps.workflow.tasks;
-
-import java.nio.ByteBuffer;
-import java.util.Map;
-import org.apache.baremaps.collection.type.*;
-import org.apache.baremaps.collection.type.geometry.GeometryDataType;
-import org.locationtech.jts.geom.Geometry;
-
-public class EntityDataType implements DataType<Entity> {
-
- private LongDataType idType = new LongDataType();
-
- private MapDataType<String, String> tagsType =
- new MapDataType<>(new StringDataType(), new StringDataType());
-
- private GeometryDataType geometryType = new GeometryDataType();
-
- @Override
- public int size(Entity value) {
- int size = 0;
- size += Integer.BYTES;
- size += idType.size(value.getId());
- size += tagsType.size(value.getTags());
- size += geometryType.size(value.getGeometry());
- return size;
- }
-
- @Override
- public int size(ByteBuffer buffer, int position) {
- return buffer.getInt(position);
- }
-
- @Override
- public void write(ByteBuffer buffer, int position, Entity value) {
- buffer.putInt(position, size(value));
- position += Integer.BYTES;
- idType.write(buffer, position, value.getId());
- position += idType.size();
- tagsType.write(buffer, position, value.getTags());
- position += tagsType.size(value.getTags());
- geometryType.write(buffer, position, value.getGeometry());
- }
-
- @Override
- public Entity read(ByteBuffer buffer, int position) {
- position += Integer.BYTES;
- long id = idType.read(buffer, position);
- position += idType.size(id);
- Map<String, String> tags = tagsType.read(buffer, position);
- position += tagsType.size(tags);
- Geometry geometry = geometryType.read(buffer, position);
- return new Entity(id, tags, geometry);
- }
-}
diff --git a/baremaps-core/src/main/java/org/apache/baremaps/workflow/tasks/ImportOpenStreetMap.java b/baremaps-core/src/main/java/org/apache/baremaps/workflow/tasks/ImportOpenStreetMap.java
index a6a38cc5..54da8b80 100644
--- a/baremaps-core/src/main/java/org/apache/baremaps/workflow/tasks/ImportOpenStreetMap.java
+++ b/baremaps-core/src/main/java/org/apache/baremaps/workflow/tasks/ImportOpenStreetMap.java
@@ -19,10 +19,10 @@ import java.nio.file.Paths;
import java.util.List;
import org.apache.baremaps.collection.*;
import org.apache.baremaps.collection.memory.MemoryMappedFile;
-import org.apache.baremaps.collection.type.LonLatDataType;
import org.apache.baremaps.collection.type.LongDataType;
import org.apache.baremaps.collection.type.LongListDataType;
import org.apache.baremaps.collection.type.PairDataType;
+import org.apache.baremaps.collection.type.geometry.LonLatDataType;
import org.apache.baremaps.openstreetmap.model.Node;
import org.apache.baremaps.openstreetmap.model.Relation;
import org.apache.baremaps.openstreetmap.model.Way;
diff --git a/baremaps-core/src/main/java/org/apache/baremaps/workflow/tasks/TransformEntityCollection.java b/baremaps-core/src/main/java/org/apache/baremaps/workflow/tasks/TransformEntityCollection.java
deleted file mode 100644
index 4bfcffff..00000000
--- a/baremaps-core/src/main/java/org/apache/baremaps/workflow/tasks/TransformEntityCollection.java
+++ /dev/null
@@ -1,150 +0,0 @@
-/*
- * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except
- * in compliance with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software distributed under the License
- * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
- * or implied. See the License for the specific language governing permissions and limitations under
- * the License.
- */
-
-package org.apache.baremaps.workflow.tasks;
-
-
-import java.nio.file.Path;
-import java.util.ArrayList;
-import java.util.Iterator;
-import java.util.List;
-import java.util.stream.Collectors;
-import java.util.stream.IntStream;
-import java.util.stream.Stream;
-import org.apache.baremaps.collection.AppendOnlyBuffer;
-import org.apache.baremaps.collection.algorithm.UnionStream;
-import org.apache.baremaps.collection.memory.MemoryMappedFile;
-import org.apache.baremaps.storage.*;
-import org.apache.baremaps.storage.postgres.PostgresStore;
-import org.apache.baremaps.vectortile.expression.Expressions.Expression;
-import org.apache.baremaps.workflow.Task;
-import org.apache.baremaps.workflow.WorkflowContext;
-import org.locationtech.jts.geom.Geometry;
-import org.locationtech.jts.geom.LineString;
-import org.locationtech.jts.geom.Polygon;
-import org.locationtech.jts.geom.util.GeometryFixer;
-import org.locationtech.jts.operation.linemerge.LineMerger;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-public record TransformEntityCollection(Path collection, String database,
- Recipe recipe) implements Task {
-
- private static final Logger logger = LoggerFactory.getLogger(TransformEntityCollection.class);
-
- record Recipe(String name, Expression<Boolean> filter, List<String> groupBy,
- Operation operation) {
- }
-
- enum Operation {
- union, merge, none
- }
-
- @Override
- public void execute(WorkflowContext context) throws Exception {
- logger.info("Transform {} with {}", collection, recipe);
-
- var schema = new SchemaImpl(recipe.name, columns());
-
- var groups = new AppendOnlyBuffer<>(new EntityDataType(), new MemoryMappedFile(collection))
- .stream()
- .filter(this::filter)
- .collect(Collectors.groupingBy(this::propertyValues));
-
- var table = new AbstractTable() {
-
- @Override
- public Schema schema() {
- return schema;
- }
-
- @Override
- public long sizeAsLong() {
- return 0;
- }
-
- @Override
- public Iterator<Row> iterator() {
- return groups.entrySet().stream().flatMap(entry -> {
- var tags = IntStream.range(0, recipe.groupBy.size()).boxed()
- .collect(Collectors.toMap(i -> recipe.groupBy.get(i), i -> entry.getKey().get(i)));
- var geometries = entry.getValue();
- var simplified = simplify(geometries.stream().map(Entity::getGeometry));
- return simplified.map(geometry -> (Row) new Entity(0, tags, geometry));
- }).iterator();
- }
- };
-
- var dataSource = context.getDataSource(database);
- var postgresDatabase = new PostgresStore(dataSource);
- postgresDatabase.add(table);
- }
-
- private Stream<Geometry> simplify(Stream<Geometry> geometries) {
- return switch (recipe.operation()) {
- case union -> union(geometries);
- case merge -> merge(geometries);
- case none -> geometries;
- };
- }
-
- private List<String> groupBy() {
- return recipe.groupBy() == null ? List.of() : recipe.groupBy();
- }
-
- private List<Column> columns() {
- var list = new ArrayList<Column>();
- for (var property : groupBy()) {
- list.add(new ColumnImpl(property, String.class));
- }
- list.add(new ColumnImpl("geometry", Geometry.class));
- return list;
- }
-
- private List<String> propertyValues(Entity entity) {
- var map = new ArrayList<String>();
- for (var property : groupBy()) {
- map.add(entity.get(property).toString());
- }
- return map;
- }
-
- private Stream<Geometry> union(Stream<Geometry> geometries) {
- var filtered = geometries
- .filter(Polygon.class::isInstance)
- .map(Polygon.class::cast)
- .map(GeometryFixer::fix)
- .filter(Geometry::isValid)
- .collect(Collectors.toCollection(ArrayList::new));
- return new UnionStream(filtered).union();
- }
-
- private Stream<Geometry> merge(Stream<Geometry> geometries) {
- var filtered = geometries
- .filter(LineString.class::isInstance)
- .map(LineString.class::cast)
- .map(GeometryFixer::fix)
- .toList();
-
- var lineMerger = new LineMerger();
- lineMerger.add(filtered);
-
- var mergedGeometries = lineMerger.getMergedLineStrings();
-
- return mergedGeometries.stream();
- }
-
- private boolean filter(Entity entity) {
- return recipe.filter.evaluate(entity);
- }
-
-}
diff --git a/baremaps-core/src/test/java/org/apache/baremaps/collection/type/DataTypeProvider.java b/baremaps-core/src/test/java/org/apache/baremaps/collection/type/DataTypeProvider.java
index 0b8d15f8..e6aa1b06 100644
--- a/baremaps-core/src/test/java/org/apache/baremaps/collection/type/DataTypeProvider.java
+++ b/baremaps-core/src/test/java/org/apache/baremaps/collection/type/DataTypeProvider.java
@@ -17,11 +17,15 @@ import java.util.List;
import java.util.Map;
import java.util.stream.Stream;
import org.apache.baremaps.collection.type.geometry.*;
+import org.apache.baremaps.storage.*;
import org.junit.jupiter.params.provider.Arguments;
import org.locationtech.jts.geom.*;
public class DataTypeProvider {
+ private final Schema schema = new SchemaImpl("row",
+ List.of(new ColumnImpl("integer", Integer.class), new ColumnImpl("string", String.class)));
+
private static Stream<Arguments> dataTypes() {
return Stream.of(
// String
@@ -260,6 +264,10 @@ public class DataTypeProvider {
new Coordinate(3, 2), new Coordinate(4, 2),
new Coordinate(4, 1), new Coordinate(3, 1)})})})),
+ // Row
+ // Arguments.of(new RowDataType(sc), )
+
+
// Geometry
Arguments.of(new GeometryDataType(),
new GeometryFactory().createEmpty(0)),
@@ -267,5 +275,6 @@ public class DataTypeProvider {
new GeometryFactory()
.createPoint(new Coordinate(1, 1))));
+
}
}
diff --git a/baremaps-core/src/test/java/org/apache/baremaps/tilestore/file/FileTileStoreTest.java b/baremaps-core/src/test/java/org/apache/baremaps/tilestore/file/FileTileStoreTest.java
index b89f8833..250397fd 100644
--- a/baremaps-core/src/test/java/org/apache/baremaps/tilestore/file/FileTileStoreTest.java
+++ b/baremaps-core/src/test/java/org/apache/baremaps/tilestore/file/FileTileStoreTest.java
@@ -18,8 +18,8 @@ import java.io.IOException;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.Paths;
-import org.apache.baremaps.tilestore.TileStore;
import org.apache.baremaps.tilestore.TileStoreTest;
+import org.apache.baremaps.tilestore.TileStore;
import org.apache.baremaps.utils.FileUtils;
import org.junit.jupiter.api.AfterEach;
import org.junit.jupiter.api.BeforeEach;
diff --git a/baremaps-core/src/test/java/org/apache/baremaps/vectortile/ExpressionsTest.java b/baremaps-core/src/test/java/org/apache/baremaps/vectortile/ExpressionsTest.java
index da436be0..61db0112 100644
--- a/baremaps-core/src/test/java/org/apache/baremaps/vectortile/ExpressionsTest.java
+++ b/baremaps-core/src/test/java/org/apache/baremaps/vectortile/ExpressionsTest.java
@@ -16,7 +16,6 @@ import static org.junit.jupiter.api.Assertions.*;
import java.io.IOException;
import java.util.List;
-import java.util.Map;
import org.apache.baremaps.storage.Row;
import org.apache.baremaps.storage.Schema;
import org.apache.baremaps.vectortile.expression.Expressions;
@@ -25,7 +24,10 @@ import org.junit.jupiter.api.Test;
class ExpressionsTest {
- record RowMock(Map<String, Object> properties) implements Row {
+ record Property(String name, Object value) {
+ }
+
+ record RowMock(List<Property> properties) implements Row {
@Override
public Schema schema() {
@@ -34,17 +36,30 @@ class ExpressionsTest {
@Override
public List<Object> values() {
- return properties.values().stream().toList();
+ return properties.stream().map(p -> p.value).toList();
}
@Override
public void set(String column, Object value) {
- properties.put(column, value);
+ properties.add(new Property(column, value));
+ }
+
+ @Override
+ public void set(int index, Object value) {
+ properties.set(index, new Property(properties.get(index).name, value));
}
@Override
public Object get(String column) {
- return properties.get(column);
+ return properties.stream()
+ .filter(p -> p.name.equals(column))
+ .findFirst().map(p -> p.value)
+ .orElse(null);
+ }
+
+ @Override
+ public Object get(int index) {
+ return properties.get(index).value;
}
}
@@ -68,15 +83,15 @@ class ExpressionsTest {
@Test
public void get() throws IOException {
assertEquals("value",
- new Get("key").evaluate(new RowMock(Map.of("key", "value"))));
- assertEquals(null, new Get("key").evaluate(new RowMock(Map.of())));
+ new Get("key").evaluate(new RowMock(List.of(new Property("key", "value")))));
+ assertEquals(null, new Get("key").evaluate(new RowMock(List.of())));
}
@Test
public void has() throws IOException {
assertEquals(true,
- new Has("key").evaluate(new RowMock(Map.of("key", "value"))));
- assertEquals(false, new Has("key").evaluate(new RowMock(Map.of())));
+ new Has("key").evaluate(new RowMock(List.of(new Property("key", "value")))));
+ assertEquals(false, new Has("key").evaluate(new RowMock(List.of())));
}
@Test
diff --git a/baremaps-core/src/test/java/org/apache/baremaps/workflow/tasks/CreateAndTransformEntityCollectionTest.java b/baremaps-core/src/test/java/org/apache/baremaps/workflow/tasks/CreateAndTransformEntityCollectionTest.java
deleted file mode 100644
index d2ce68e1..00000000
--- a/baremaps-core/src/test/java/org/apache/baremaps/workflow/tasks/CreateAndTransformEntityCollectionTest.java
+++ /dev/null
@@ -1,48 +0,0 @@
-/*
- * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except
- * in compliance with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software distributed under the License
- * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
- * or implied. See the License for the specific language governing permissions and limitations under
- * the License.
- */
-
-package org.apache.baremaps.workflow.tasks;
-
-
-
-import java.nio.file.Files;
-import java.nio.file.Paths;
-import java.util.List;
-import org.apache.baremaps.testing.PostgresContainerTest;
-import org.apache.baremaps.testing.TestFiles;
-import org.apache.baremaps.vectortile.expression.Expressions.Has;
-import org.apache.baremaps.workflow.WorkflowContext;
-import org.apache.baremaps.workflow.tasks.TransformEntityCollection.Operation;
-import org.apache.baremaps.workflow.tasks.TransformEntityCollection.Recipe;
-import org.junit.jupiter.api.Tag;
-import org.junit.jupiter.api.Test;
-
-class CreateAndTransformEntityCollectionTest extends PostgresContainerTest {
-
- @Test
- @Tag("integration")
- void execute() throws Exception {
- var file = TestFiles.resolve("monaco/monaco.osm.pbf");
- var collection = Paths.get("entity_collection");
- var jdbcUrl = jdbcUrl();
-
- var createTask = new CreateEntityCollection(file, collection, 3857);
- createTask.execute(new WorkflowContext());
-
- var simplifyTask = new TransformEntityCollection(collection, jdbcUrl,
- new Recipe("building", new Has("landuse"), List.of("landuse"),
- Operation.union));
- simplifyTask.execute(new WorkflowContext());
-
- Files.delete(collection);
- }
-}