You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@baremaps.apache.org by bc...@apache.org on 2025/04/14 15:49:49 UTC
(incubator-baremaps) 03/05: Add schema for openstreetmap formats
This is an automated email from the ASF dual-hosted git repository.
bchapuis pushed a commit to branch calcite-schema-ddl
in repository https://gitbox.apache.org/repos/asf/incubator-baremaps.git
commit 19f8856ff4e11918f28db212d885f8ae1862662f
Author: Bertil Chapuis <bc...@gmail.com>
AuthorDate: Mon Apr 14 16:54:37 2025 +0200
Add schema for openstreetmap formats
---
.../baremaps/calcite/BaremapsTableFactory.java | 32 +---
.../apache/baremaps/calcite/data/DataSchema.java | 2 +-
.../calcite/openstreetmap/OpenStreetMapSchema.java | 184 ++++++++++++++++++++
.../calcite/openstreetmap/OpenStreetMapTable.java | 19 ++-
.../openstreetmap/OpenStreetMapSchemaTest.java | 187 +++++++++++++++++++++
.../openstreetmap/OpenStreetMapTableTest.java | 4 +-
6 files changed, 395 insertions(+), 33 deletions(-)
diff --git a/baremaps-calcite/src/main/java/org/apache/baremaps/calcite/BaremapsTableFactory.java b/baremaps-calcite/src/main/java/org/apache/baremaps/calcite/BaremapsTableFactory.java
index 857f46261..c2ddc3ad3 100644
--- a/baremaps-calcite/src/main/java/org/apache/baremaps/calcite/BaremapsTableFactory.java
+++ b/baremaps-calcite/src/main/java/org/apache/baremaps/calcite/BaremapsTableFactory.java
@@ -143,9 +143,9 @@ public class BaremapsTableFactory implements TableFactory<Table> {
// Create an entity reader based on the file extension
if (filePath.endsWith(".pbf") || filePath.endsWith(".osm.pbf")) {
- return createTableFromPbf(inputStream);
+ return createTableFromPbf(Paths.get(filePath));
} else if (filePath.endsWith(".xml") || filePath.endsWith(".osm")) {
- return createTableFromXml(inputStream);
+ return createTableFromXml(Paths.get(filePath));
} else {
throw new IllegalArgumentException(
"Unsupported file format. Supported formats are .pbf, .osm.pbf, .xml, and .osm");
@@ -249,16 +249,6 @@ public class BaremapsTableFactory implements TableFactory<Table> {
}
}
- /**
- * Create a table from a PBF file.
- *
- * @param inputStream the input stream
- * @return the table
- */
- public static OpenStreetMapTable createTableFromPbf(InputStream inputStream) {
- return new OpenStreetMapTable(new PbfEntityReader().setGeometries(true), inputStream);
- }
-
/**
* Create a table from a PBF file.
*
@@ -267,17 +257,9 @@ public class BaremapsTableFactory implements TableFactory<Table> {
* @throws IOException if an I/O error occurs
*/
public static OpenStreetMapTable createTableFromPbf(Path path) throws IOException {
- return createTableFromPbf(new FileInputStream(path.toFile()));
- }
-
- /**
- * Create a table from an XML file.
- *
- * @param inputStream the input stream
- * @return the table
- */
- public static OpenStreetMapTable createTableFromXml(InputStream inputStream) {
- return new OpenStreetMapTable(new XmlEntityReader().setGeometries(true), inputStream);
+ PbfEntityReader reader = new PbfEntityReader();
+ reader.setGeometries(true);
+ return new OpenStreetMapTable(path.toFile(), reader);
}
/**
@@ -288,7 +270,9 @@ public class BaremapsTableFactory implements TableFactory<Table> {
* @throws IOException if an I/O error occurs
*/
public static OpenStreetMapTable createTableFromXml(Path path) throws IOException {
- return createTableFromXml(new FileInputStream(path.toFile()));
+ XmlEntityReader reader = new XmlEntityReader();
+ reader.setGeometries(true);
+ return new OpenStreetMapTable(path.toFile(), reader);
}
private Table createGeoParquetTable(Map<String, Object> operand) {
diff --git a/baremaps-calcite/src/main/java/org/apache/baremaps/calcite/data/DataSchema.java b/baremaps-calcite/src/main/java/org/apache/baremaps/calcite/data/DataSchema.java
index 35e9f4c89..8f3bafe84 100644
--- a/baremaps-calcite/src/main/java/org/apache/baremaps/calcite/data/DataSchema.java
+++ b/baremaps-calcite/src/main/java/org/apache/baremaps/calcite/data/DataSchema.java
@@ -88,4 +88,4 @@ public class DataSchema extends AbstractSchema {
protected Map<String, Table> getTableMap() {
return tableMap;
}
-}
\ No newline at end of file
+}
diff --git a/baremaps-calcite/src/main/java/org/apache/baremaps/calcite/openstreetmap/OpenStreetMapSchema.java b/baremaps-calcite/src/main/java/org/apache/baremaps/calcite/openstreetmap/OpenStreetMapSchema.java
new file mode 100644
index 000000000..322a50592
--- /dev/null
+++ b/baremaps-calcite/src/main/java/org/apache/baremaps/calcite/openstreetmap/OpenStreetMapSchema.java
@@ -0,0 +1,184 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to you under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.baremaps.calcite.openstreetmap;
+
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.IOException;
+import java.io.InputStream;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.Objects;
+import org.apache.baremaps.openstreetmap.OpenStreetMapFormat;
+import org.apache.baremaps.openstreetmap.model.Entity;
+import org.apache.baremaps.openstreetmap.pbf.PbfEntityReader;
+import org.apache.baremaps.openstreetmap.xml.XmlEntityReader;
+import org.apache.calcite.rel.type.RelDataTypeFactory;
+import org.apache.calcite.schema.Table;
+import org.apache.calcite.schema.impl.AbstractSchema;
+
+/**
+ * A Calcite schema implementation for OpenStreetMap data. This schema provides access to OpenStreetMap
+ * files through the Apache Calcite framework for SQL querying.
+ */
+public class OpenStreetMapSchema extends AbstractSchema {
+
+ private final File directory;
+ private final Map<String, Table> tableMap;
+ private final RelDataTypeFactory typeFactory;
+
+ /**
+ * Constructs an OpenStreetMapSchema with the specified directory.
+ *
+ * @param directory the directory containing OpenStreetMap files
+ * @param typeFactory the type factory to use for creating tables
+ * @throws IOException if an I/O error occurs
+ */
+ public OpenStreetMapSchema(File directory, RelDataTypeFactory typeFactory) throws IOException {
+ this.directory = Objects.requireNonNull(directory, "Directory cannot be null");
+ this.typeFactory = Objects.requireNonNull(typeFactory, "Type factory cannot be null");
+ this.tableMap = new HashMap<>();
+
+ // Process files in the directory
+ File[] files = directory.listFiles((dir, name) ->
+ name.toLowerCase().endsWith(".pbf") ||
+ name.toLowerCase().endsWith(".osm.pbf") ||
+ name.toLowerCase().endsWith(".xml") ||
+ name.toLowerCase().endsWith(".osm"));
+
+ if (files != null) {
+ for (File file : files) {
+ // Extract the base name without extension (e.g., "sample" from "sample.osm.pbf")
+ String fileName = file.getName();
+ String tableName = fileName;
+
+ // Remove all extensions (e.g., "sample.osm.pbf" -> "sample")
+ while (tableName.contains(".")) {
+ int lastDotIndex = tableName.lastIndexOf('.');
+ if (lastDotIndex > 0) {
+ tableName = tableName.substring(0, lastDotIndex);
+ } else {
+ break;
+ }
+ }
+
+ // Create the table with the file reference
+ tableMap.put(tableName, createTable(file));
+ }
+ }
+ }
+
+ /**
+ * Constructs an OpenStreetMapSchema with a single file.
+ *
+ * @param file the OpenStreetMap file
+ * @param typeFactory the type factory to use for creating tables
+ * @throws IOException if an I/O error occurs
+ */
+ public OpenStreetMapSchema(File file, RelDataTypeFactory typeFactory, boolean isDirectory) throws IOException {
+ if (isDirectory) {
+ // If isDirectory is true, treat the file as a directory
+ this.directory = Objects.requireNonNull(file, "Directory cannot be null");
+ this.typeFactory = Objects.requireNonNull(typeFactory, "Type factory cannot be null");
+ this.tableMap = new HashMap<>();
+
+ // Process files in the directory
+ File[] files = file.listFiles((dir, name) ->
+ name.toLowerCase().endsWith(".pbf") ||
+ name.toLowerCase().endsWith(".osm.pbf") ||
+ name.toLowerCase().endsWith(".xml") ||
+ name.toLowerCase().endsWith(".osm"));
+
+ if (files != null) {
+ for (File osmFile : files) {
+ // Extract the base name without extension (e.g., "sample" from "sample.osm.pbf")
+ String fileName = osmFile.getName();
+ String tableName = fileName;
+
+ // Remove all extensions (e.g., "sample.osm.pbf" -> "sample")
+ while (tableName.contains(".")) {
+ int lastDotIndex = tableName.lastIndexOf('.');
+ if (lastDotIndex > 0) {
+ tableName = tableName.substring(0, lastDotIndex);
+ } else {
+ break;
+ }
+ }
+
+ // Create the table with the file reference
+ tableMap.put(tableName, createTable(osmFile));
+ }
+ }
+ } else {
+ // If isDirectory is false, treat the file as a single file
+ this.directory = Objects.requireNonNull(file, "File cannot be null");
+ this.typeFactory = Objects.requireNonNull(typeFactory, "Type factory cannot be null");
+ this.tableMap = new HashMap<>();
+
+ // Extract the base name without extension (e.g., "sample" from "sample.osm.pbf")
+ String fileName = file.getName();
+ String tableName = fileName;
+
+ // Remove all extensions (e.g., "sample.osm.pbf" -> "sample")
+ while (tableName.contains(".")) {
+ int lastDotIndex = tableName.lastIndexOf('.');
+ if (lastDotIndex > 0) {
+ tableName = tableName.substring(0, lastDotIndex);
+ } else {
+ break;
+ }
+ }
+
+ // Create the table with the file reference
+ tableMap.put(tableName, createTable(file));
+ }
+ }
+
+ /**
+ * Creates a table for the given file.
+ *
+ * @param file the OpenStreetMap file
+ * @return the created table
+ */
+ private Table createTable(File file) {
+ // Determine the appropriate entity reader based on file extension
+ OpenStreetMapFormat.EntityReader<Entity> entityReader;
+ if (file.getName().toLowerCase().endsWith(".pbf") ||
+ file.getName().toLowerCase().endsWith(".osm.pbf")) {
+ PbfEntityReader pbfReader = new PbfEntityReader();
+ pbfReader.setGeometries(true);
+ pbfReader.setCoordinateMap(new HashMap<>());
+ pbfReader.setReferenceMap(new HashMap<>());
+ entityReader = pbfReader;
+ } else {
+ XmlEntityReader xmlReader = new XmlEntityReader();
+ xmlReader.setGeometries(true);
+ xmlReader.setCoordinateMap(new HashMap<>());
+ xmlReader.setReferenceMap(new HashMap<>());
+ entityReader = xmlReader;
+ }
+
+ // Create the table with the file reference
+ return new OpenStreetMapTable(file, entityReader);
+ }
+
+ @Override
+ protected Map<String, Table> getTableMap() {
+ return tableMap;
+ }
+}
\ No newline at end of file
diff --git a/baremaps-calcite/src/main/java/org/apache/baremaps/calcite/openstreetmap/OpenStreetMapTable.java b/baremaps-calcite/src/main/java/org/apache/baremaps/calcite/openstreetmap/OpenStreetMapTable.java
index 93cd87d44..6be494888 100644
--- a/baremaps-calcite/src/main/java/org/apache/baremaps/calcite/openstreetmap/OpenStreetMapTable.java
+++ b/baremaps-calcite/src/main/java/org/apache/baremaps/calcite/openstreetmap/OpenStreetMapTable.java
@@ -17,6 +17,9 @@
package org.apache.baremaps.calcite.openstreetmap;
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.IOException;
import java.io.InputStream;
import java.util.*;
import java.util.stream.Stream;
@@ -40,19 +43,19 @@ import org.locationtech.jts.geom.Geometry;
*/
public class OpenStreetMapTable extends AbstractTable implements ScannableTable {
+ private final File file;
private final EntityReader<Entity> entityReader;
- private final InputStream inputStream;
private RelDataType rowType;
/**
* Constructs an OpenStreetMapTable with the specified parameters.
*
+ * @param file the OpenStreetMap file
* @param entityReader the EntityReader for parsing the OSM data
- * @param inputStream the input stream containing the OSM data
*/
- public OpenStreetMapTable(EntityReader<Entity> entityReader, InputStream inputStream) {
- this.entityReader = entityReader;
- this.inputStream = inputStream;
+ public OpenStreetMapTable(File file, EntityReader<Entity> entityReader) {
+ this.file = Objects.requireNonNull(file, "File cannot be null");
+ this.entityReader = Objects.requireNonNull(entityReader, "Entity reader cannot be null");
}
@Override
@@ -97,7 +100,11 @@ public class OpenStreetMapTable extends AbstractTable implements ScannableTable
return new AbstractEnumerable<Object[]>() {
@Override
public Enumerator<Object[]> enumerator() {
- return new OpenStreetMapEnumerator(entityReader, inputStream);
+ try {
+ return new OpenStreetMapEnumerator(entityReader, new FileInputStream(file));
+ } catch (IOException e) {
+ throw new RuntimeException("Failed to open input stream", e);
+ }
}
};
}
diff --git a/baremaps-calcite/src/test/java/org/apache/baremaps/calcite/openstreetmap/OpenStreetMapSchemaTest.java b/baremaps-calcite/src/test/java/org/apache/baremaps/calcite/openstreetmap/OpenStreetMapSchemaTest.java
new file mode 100644
index 000000000..8cedb78ad
--- /dev/null
+++ b/baremaps-calcite/src/test/java/org/apache/baremaps/calcite/openstreetmap/OpenStreetMapSchemaTest.java
@@ -0,0 +1,187 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to you under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.baremaps.calcite.openstreetmap;
+
+import static org.junit.jupiter.api.Assertions.*;
+
+import java.io.File;
+import java.io.IOException;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.sql.Connection;
+import java.sql.DriverManager;
+import java.sql.ResultSet;
+import java.sql.SQLException;
+import java.sql.Statement;
+import java.util.HashMap;
+import java.util.Properties;
+import org.apache.baremaps.openstreetmap.pbf.PbfEntityReader;
+import org.apache.baremaps.testing.TestFiles;
+import org.apache.calcite.jdbc.CalciteConnection;
+import org.apache.calcite.rel.type.RelDataTypeFactory;
+import org.apache.calcite.schema.SchemaPlus;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.io.TempDir;
+
+public class OpenStreetMapSchemaTest {
+
+ @TempDir
+ Path tempDir;
+
+ private Path sampleDataDir;
+ private RelDataTypeFactory typeFactory;
+
+ @BeforeEach
+ void setup() throws IOException, SQLException {
+ // Create a temporary directory for test files
+ sampleDataDir = tempDir.resolve("osm-data");
+ Files.createDirectories(sampleDataDir);
+
+ // Get the absolute paths to the sample files
+ Path pbfSourcePath = TestFiles.SAMPLE_OSM_PBF.toAbsolutePath();
+ Path xmlSourcePath = TestFiles.SAMPLE_OSM_XML.toAbsolutePath();
+
+ // Copy sample OSM files to the test directory
+ Path pbfPath = sampleDataDir.resolve("sample.osm.pbf");
+ Path xmlPath = sampleDataDir.resolve("sample.osm.xml");
+
+ // Check if source files exist
+ if (!Files.exists(pbfSourcePath)) {
+ throw new IOException("Sample PBF file not found: " + pbfSourcePath);
+ }
+ if (!Files.exists(xmlSourcePath)) {
+ throw new IOException("Sample XML file not found: " + xmlSourcePath);
+ }
+
+ Files.copy(pbfSourcePath, pbfPath);
+ Files.copy(xmlSourcePath, xmlPath);
+
+ // Set up Calcite connection to get a RelDataTypeFactory
+ Properties info = new Properties();
+ info.setProperty("lex", "MYSQL");
+
+ try (Connection connection = DriverManager.getConnection("jdbc:calcite:", info)) {
+ CalciteConnection calciteConnection = connection.unwrap(CalciteConnection.class);
+ typeFactory = calciteConnection.getTypeFactory();
+ }
+ }
+
+ @Test
+ void testSchemaCreation() throws IOException {
+ // Create an OpenStreetMapSchema with the test directory
+ OpenStreetMapSchema schema = new OpenStreetMapSchema(sampleDataDir.toFile(), typeFactory);
+
+ // Verify that the schema contains the expected tables
+ // The table name is based on the filename without extension
+ assertTrue(schema.getTableMap().containsKey("sample"), "Schema should contain 'sample' table");
+
+ // Verify that the table has the expected structure
+ OpenStreetMapTable table = (OpenStreetMapTable) schema.getTableMap().get("sample");
+ assertNotNull(table, "Table should not be null");
+
+ // Verify the schema structure
+ int fieldCount = table.getRowType(typeFactory).getFieldCount();
+ assertEquals(9, fieldCount, "Schema should have 9 columns");
+ }
+
+ @Test
+ void testSqlQueryWithDirectory() throws Exception {
+ // Create an OpenStreetMapSchema with the test directory
+ OpenStreetMapSchema schema = new OpenStreetMapSchema(sampleDataDir.toFile(), typeFactory);
+
+ // Configure Calcite connection properties
+ Properties info = new Properties();
+ info.setProperty("lex", "MYSQL");
+
+ // Set up a connection and register our schema
+ try (Connection connection = DriverManager.getConnection("jdbc:calcite:", info)) {
+ CalciteConnection calciteConnection = connection.unwrap(CalciteConnection.class);
+ SchemaPlus rootSchema = calciteConnection.getRootSchema();
+
+ // Add the schema to the root schema
+ rootSchema.add("osm", schema);
+
+ // Test a simple query to select a limited number of entities
+ // The table name is based on the filename without extension
+ try (Statement statement = connection.createStatement();
+ ResultSet resultSet = statement.executeQuery("SELECT id, type FROM osm.sample LIMIT 10")) {
+ int rowCount = 0;
+
+ while (resultSet.next()) {
+ rowCount++;
+ long id = resultSet.getLong("id");
+ String type = resultSet.getString("type");
+
+ // Verify basic properties
+ assertTrue(id != 0, "Entity should have non-zero ID");
+ assertNotNull(type, "Entity should have a type");
+ }
+
+ // Verify that we got some rows
+ assertTrue(rowCount > 0, "Should have retrieved at least one entity");
+ }
+ }
+ }
+
+ @Test
+ void testSqlQueryWithSingleFile() throws Exception {
+ // Create a properly configured PbfEntityReader
+ PbfEntityReader entityReader = new PbfEntityReader();
+ entityReader.setGeometries(true);
+ entityReader.setCoordinateMap(new HashMap<>());
+ entityReader.setReferenceMap(new HashMap<>());
+
+ // Create an OpenStreetMapSchema with a single file
+ File pbfFile = sampleDataDir.resolve("sample.osm.pbf").toFile();
+ OpenStreetMapSchema schema = new OpenStreetMapSchema(pbfFile, typeFactory, false);
+
+ // Configure Calcite connection properties
+ Properties info = new Properties();
+ info.setProperty("lex", "MYSQL");
+
+ // Set up a connection and register our schema
+ try (Connection connection = DriverManager.getConnection("jdbc:calcite:", info)) {
+ CalciteConnection calciteConnection = connection.unwrap(CalciteConnection.class);
+ SchemaPlus rootSchema = calciteConnection.getRootSchema();
+
+ // Add the schema to the root schema
+ rootSchema.add("osm", schema);
+
+ // Test a simple query to select a limited number of entities
+ // For a single file, the table name is "sample" (not "osm")
+ try (Statement statement = connection.createStatement();
+ ResultSet resultSet = statement.executeQuery("SELECT id, type FROM osm.sample LIMIT 10")) {
+ int rowCount = 0;
+
+ while (resultSet.next()) {
+ rowCount++;
+ long id = resultSet.getLong("id");
+ String type = resultSet.getString("type");
+
+ // Verify basic properties
+ assertTrue(id != 0, "Entity should have non-zero ID");
+ assertNotNull(type, "Entity should have a type");
+ }
+
+ // Verify that we got some rows
+ assertTrue(rowCount > 0, "Should have retrieved at least one entity");
+ }
+ }
+ }
+}
\ No newline at end of file
diff --git a/baremaps-calcite/src/test/java/org/apache/baremaps/calcite/openstreetmap/OpenStreetMapTableTest.java b/baremaps-calcite/src/test/java/org/apache/baremaps/calcite/openstreetmap/OpenStreetMapTableTest.java
index 73aa021ae..74e2f1eaa 100644
--- a/baremaps-calcite/src/test/java/org/apache/baremaps/calcite/openstreetmap/OpenStreetMapTableTest.java
+++ b/baremaps-calcite/src/test/java/org/apache/baremaps/calcite/openstreetmap/OpenStreetMapTableTest.java
@@ -51,7 +51,7 @@ public class OpenStreetMapTableTest {
entityReader.setGeometries(false); // Don't generate geometries to avoid errors
// Create the OpenStreetMapTable
- OpenStreetMapTable osmTable = new OpenStreetMapTable(entityReader, inputStream);
+ OpenStreetMapTable osmTable = new OpenStreetMapTable(SAMPLE_OSM_PATH.toFile(), entityReader);
// Verify the schema structure
RelDataTypeFactory typeFactory = new JavaTypeFactoryImpl();
@@ -91,7 +91,7 @@ public class OpenStreetMapTableTest {
try (var inputStream = new FileInputStream(SAMPLE_OSM_PATH.toFile())) {
// Create the table with our configured reader
- OpenStreetMapTable osmTable = new OpenStreetMapTable(entityReader, inputStream);
+ OpenStreetMapTable osmTable = new OpenStreetMapTable(SAMPLE_OSM_PATH.toFile(), entityReader);
// Configure Calcite connection properties
Properties info = new Properties();