You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@baremaps.apache.org by bc...@apache.org on 2025/01/26 10:44:13 UTC
(incubator-baremaps) branch calcite-framework created (now 251ea904c)
This is an automated email from the ASF dual-hosted git repository.
bchapuis pushed a change to branch calcite-framework
in repository https://gitbox.apache.org/repos/asf/incubator-baremaps.git
at 251ea904c Add header to memory
This branch includes the following new commits:
new e9983dd21 Experiments with calcite
new 14d2eb9dd Add list
new 97799717b Format code
new fa97d3cb0 Improve GeoPackage store
new d6b161c89 Make data collection closeable
new a6c7315e4 Refactor the sub modules
new 251ea904c Add header to memory
The 7 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
(incubator-baremaps) 01/07: Experiments with calcite
Posted by bc...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
bchapuis pushed a commit to branch calcite-framework
in repository https://gitbox.apache.org/repos/asf/incubator-baremaps.git
commit e9983dd21d1acad16405f0b4676acfd97f84341f
Author: Bertil Chapuis <bc...@gmail.com>
AuthorDate: Mon Jan 13 08:26:18 2025 +0100
Experiments with calcite
---
.../org/apache/baremaps/calcite/CalciteTest.java | 2 +-
baremaps-data/pom.xml | 4 +
.../data/{store => calcite}/BaremapsDataTable.java | 2 +-
.../data/calcite/CalciteMultipleDDLParser.java | 14 ++
.../apache/baremaps/data/calcite/CalciteTest.java | 170 +++++++++++++++++++++
pom.xml | 7 +-
6 files changed, 196 insertions(+), 3 deletions(-)
diff --git a/baremaps-core/src/test/java/org/apache/baremaps/calcite/CalciteTest.java b/baremaps-core/src/test/java/org/apache/baremaps/calcite/CalciteTest.java
index efc13eb4d..1b5e8b3b2 100644
--- a/baremaps-core/src/test/java/org/apache/baremaps/calcite/CalciteTest.java
+++ b/baremaps-core/src/test/java/org/apache/baremaps/calcite/CalciteTest.java
@@ -22,10 +22,10 @@ import com.google.common.collect.ImmutableList;
import java.sql.*;
import java.util.List;
import java.util.Properties;
+import org.apache.baremaps.data.calcite.BaremapsDataTable;
import org.apache.baremaps.data.calcite.SqlDataTable;
import org.apache.baremaps.data.collection.AppendOnlyLog;
import org.apache.baremaps.data.collection.IndexedDataList;
-import org.apache.baremaps.data.store.BaremapsDataTable;
import org.apache.baremaps.data.type.RowDataType;
import org.apache.baremaps.maplibre.vectortile.VectorTileFunctions;
import org.apache.baremaps.store.*;
diff --git a/baremaps-data/pom.xml b/baremaps-data/pom.xml
index 406f2194b..2054a761e 100644
--- a/baremaps-data/pom.xml
+++ b/baremaps-data/pom.xml
@@ -37,6 +37,10 @@ limitations under the License.
<groupId>org.apache.calcite</groupId>
<artifactId>calcite-core</artifactId>
</dependency>
+ <dependency>
+ <groupId>org.apache.calcite</groupId>
+ <artifactId>calcite-server</artifactId>
+ </dependency>
<dependency>
<groupId>org.roaringbitmap</groupId>
<artifactId>RoaringBitmap</artifactId>
diff --git a/baremaps-data/src/main/java/org/apache/baremaps/data/store/BaremapsDataTable.java b/baremaps-data/src/main/java/org/apache/baremaps/data/calcite/BaremapsDataTable.java
similarity index 97%
rename from baremaps-data/src/main/java/org/apache/baremaps/data/store/BaremapsDataTable.java
rename to baremaps-data/src/main/java/org/apache/baremaps/data/calcite/BaremapsDataTable.java
index 89123dab6..55ddee115 100644
--- a/baremaps-data/src/main/java/org/apache/baremaps/data/store/BaremapsDataTable.java
+++ b/baremaps-data/src/main/java/org/apache/baremaps/data/calcite/BaremapsDataTable.java
@@ -15,7 +15,7 @@
* limitations under the License.
*/
-package org.apache.baremaps.data.store;
+package org.apache.baremaps.data.calcite;
import java.util.Iterator;
import org.apache.baremaps.data.collection.DataCollection;
diff --git a/baremaps-data/src/main/java/org/apache/baremaps/data/calcite/CalciteMultipleDDLParser.java b/baremaps-data/src/main/java/org/apache/baremaps/data/calcite/CalciteMultipleDDLParser.java
new file mode 100644
index 000000000..04984fd3e
--- /dev/null
+++ b/baremaps-data/src/main/java/org/apache/baremaps/data/calcite/CalciteMultipleDDLParser.java
@@ -0,0 +1,14 @@
+package org.apache.baremaps.data.calcite;
+
+import org.apache.calcite.config.Lex;
+import org.apache.calcite.sql.SqlNode;
+import org.apache.calcite.sql.SqlNodeList;
+import org.apache.calcite.sql.parser.SqlParser;
+import org.apache.calcite.sql.parser.ddl.SqlDdlParserImpl;
+import org.apache.calcite.sql.validate.SqlConformanceEnum;
+
+public class CalciteMultipleDDLParser {
+ public static void main(String[] args) throws Exception {
+
+ }
+}
diff --git a/baremaps-data/src/test/java/org/apache/baremaps/data/calcite/CalciteTest.java b/baremaps-data/src/test/java/org/apache/baremaps/data/calcite/CalciteTest.java
new file mode 100644
index 000000000..52768a70d
--- /dev/null
+++ b/baremaps-data/src/test/java/org/apache/baremaps/data/calcite/CalciteTest.java
@@ -0,0 +1,170 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to you under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.baremaps.data.calcite;
+
+import com.google.common.collect.ImmutableList;
+import org.apache.baremaps.data.collection.AppendOnlyLog;
+import org.apache.baremaps.data.collection.IndexedDataList;
+import org.apache.baremaps.data.type.RowDataType;
+import org.apache.baremaps.store.DataColumn.Cardinality;
+import org.apache.baremaps.store.DataColumn.Type;
+import org.apache.baremaps.store.*;
+import org.apache.calcite.config.Lex;
+import org.apache.calcite.jdbc.CalciteConnection;
+import org.apache.calcite.model.ModelHandler;
+import org.apache.calcite.runtime.SpatialTypeFunctions;
+import org.apache.calcite.schema.SchemaPlus;
+import org.apache.calcite.schema.impl.ViewTable;
+import org.apache.calcite.schema.impl.ViewTableMacro;
+import org.apache.calcite.sql.SqlNode;
+import org.apache.calcite.sql.SqlNodeList;
+import org.apache.calcite.sql.fun.SqlSpatialTypeFunctions;
+import org.apache.calcite.sql.parser.SqlParseException;
+import org.apache.calcite.sql.parser.SqlParser;
+import org.apache.calcite.sql.parser.ddl.SqlDdlParserImpl;
+import org.apache.calcite.sql.validate.SqlConformanceEnum;
+import org.junit.jupiter.api.Test;
+import org.locationtech.jts.geom.Coordinate;
+import org.locationtech.jts.geom.GeometryFactory;
+
+import java.sql.*;
+import java.util.Collections;
+import java.util.List;
+import java.util.Properties;
+
+public class CalciteTest {
+
+ @Test
+ void sql() throws SQLException {
+ GeometryFactory geometryFactory = new GeometryFactory();
+
+ // Configure Calcite connection properties
+ Properties info = new Properties();
+ info.setProperty("lex", "MYSQL"); // Use MySQL dialect
+ info.setProperty("caseSensitive", "false"); // Disable case sensitivity
+ info.setProperty("unquotedCasing", "TO_LOWER"); // Convert unquoted identifiers to lowercase
+ info.setProperty("quotedCasing", "TO_LOWER");
+
+ try (Connection connection = DriverManager.getConnection("jdbc:calcite:", info)) {
+ CalciteConnection calciteConnection = connection.unwrap(CalciteConnection.class);
+ SchemaPlus rootSchema = calciteConnection.getRootSchema();
+
+ // Create and add 'city' table
+ DataSchema cityRowType = new DataSchemaImpl("city", List.of(
+ new DataColumnFixed("id", Cardinality.OPTIONAL, Type.INTEGER),
+ new DataColumnFixed("name", Cardinality.OPTIONAL, Type.STRING),
+ new DataColumnFixed("geometry", Cardinality.OPTIONAL, Type.GEOMETRY)));
+
+ DataTable cityDataTable = new BaremapsDataTable(
+ cityRowType,
+ new IndexedDataList<>(new AppendOnlyLog<>(new RowDataType(cityRowType))));
+
+ cityDataTable.add(new DataRowImpl(cityDataTable.schema(),
+ List.of(1, "Paris", geometryFactory.createPoint(new Coordinate(2.3522, 48.8566)))));
+ cityDataTable.add(new DataRowImpl(cityDataTable.schema(),
+ List.of(2, "New York", geometryFactory.createPoint(new Coordinate(-74.0060, 40.7128)))));
+
+ SqlDataTable citySqlDataTable = new SqlDataTable(cityDataTable);
+ rootSchema.add("city", citySqlDataTable);
+
+ // Create and add 'population' table
+ DataSchema populationRowType = new DataSchemaImpl("population", List.of(
+ new DataColumnFixed("city_id", Cardinality.OPTIONAL, Type.INTEGER),
+ new DataColumnFixed("population", Cardinality.OPTIONAL, Type.INTEGER)));
+
+ DataTable populationDataTable = new BaremapsDataTable(
+ populationRowType,
+ new IndexedDataList<>(new AppendOnlyLog<>(new RowDataType(populationRowType))));
+
+ populationDataTable.add(new DataRowImpl(populationDataTable.schema(), List.of(1, 2_161_000)));
+ populationDataTable.add(new DataRowImpl(populationDataTable.schema(), List.of(2, 8_336_000)));
+
+ SqlDataTable populationSqlDataTable = new SqlDataTable(populationDataTable);
+ rootSchema.add("population", populationSqlDataTable);
+
+ // Create view 'city_population'
+ String mvSql = "SELECT c.id, c.name, c.geometry, p.population " +
+ "FROM city c " + // lowercase and unquoted
+ "JOIN population p ON c.id = p.city_id";
+
+ ViewTableMacro materializedView = ViewTable.viewMacro(
+ rootSchema,
+ mvSql,
+ Collections.emptyList(), // Schema path
+ List.of("city_population"), // Name parts
+ false); // Not a materialized view
+
+ rootSchema.add("city_population", materializedView);
+
+ // Debug: List all tables in the root schema
+ System.out.println("Available tables in the root schema:");
+ for (String tableName : rootSchema.getTableNames()) {
+ System.out.println(" - " + tableName);
+ }
+
+ String sql = "SELECT * FROM city";
+ try (Statement statement = connection.createStatement();
+ ResultSet resultSet = statement.executeQuery(sql)) {
+ while (resultSet.next()) {
+ System.out.println(resultSet.getString("id") + " " + resultSet.getString("name"));
+ }
+ }
+
+ // Query the view
+ sql = "SELECT * FROM city_population";
+ try (Statement statement = connection.createStatement();
+ ResultSet resultSet = statement.executeQuery(sql)) {
+ while (resultSet.next()) {
+ System.out.println(
+ resultSet.getString("id") + " " + resultSet.getString("name")
+ );
+ }
+ }
+ }
+
+ }
+
+ @Test
+ void ddl() throws SqlParseException {
+
+ // Example SQL script with multiple DDL statements
+ String sqlScript = """
+ CREATE MATERIALIZED VIEW IF NOT EXISTS my_view AS
+ SELECT * FROM my_table;
+ """;
+
+ // Build a parser config that supports DDL
+ SqlParser.Config config = SqlParser.configBuilder()
+ .setParserFactory(SqlDdlParserImpl.FACTORY)
+ .setConformance(SqlConformanceEnum.BABEL)
+ .setLex(Lex.MYSQL)
+ .build();
+
+ // Create the parser from the config
+ SqlParser parser = SqlParser.create(sqlScript, config);
+
+ // Parse the script as a list of statements
+ SqlNodeList sqlNodeList = parser.parseStmtList();
+
+ // Iterate and print each parsed statement
+ for (SqlNode sqlNode : sqlNodeList) {
+ System.out.println("Parsed statement: " + sqlNode.toString());
+ }
+ }
+
+}
diff --git a/pom.xml b/pom.xml
index e8da932e2..edb9af60a 100644
--- a/pom.xml
+++ b/pom.xml
@@ -88,7 +88,7 @@ limitations under the License.
<version.lib.awssdk>2.27.17</version.lib.awssdk>
<version.lib.brotli4j>1.16.0</version.lib.brotli4j>
<version.lib.caffeine>3.1.8</version.lib.caffeine>
- <version.lib.calcite>1.37.0</version.lib.calcite>
+ <version.lib.calcite>1.38.0</version.lib.calcite>
<version.lib.commons-compress>1.27.1</version.lib.commons-compress>
<version.lib.fastutil>8.5.14</version.lib.fastutil>
<version.lib.flatgeobuf>3.26.2</version.lib.flatgeobuf>
@@ -348,6 +348,11 @@ limitations under the License.
<artifactId>calcite-core</artifactId>
<version>${version.lib.calcite}</version>
</dependency>
+ <dependency>
+ <groupId>org.apache.calcite</groupId>
+ <artifactId>calcite-server</artifactId>
+ <version>${version.lib.calcite}</version>
+ </dependency>
<dependency>
<groupId>org.apache.commons</groupId>
<artifactId>commons-compress</artifactId>
(incubator-baremaps) 07/07: Add header to memory
Posted by bc...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
bchapuis pushed a commit to branch calcite-framework
in repository https://gitbox.apache.org/repos/asf/incubator-baremaps.git
commit 251ea904ca7f4a272d05938d7db49f54da2edd65
Author: Bertil Chapuis <bc...@gmail.com>
AuthorDate: Sun Jan 26 11:43:54 2025 +0100
Add header to memory
---
.../baremaps/calcite/BaremapsTableFactory.java | 91 ++++--
.../baremaps/calcite/BaremapsTableFactoryTest.java | 118 +++++++
.../apache/baremaps/calcite/CalciteTest.java | 51 +--
.../java/org/apache/baremaps/csv/CsvDataStore.java | 6 +-
.../java/org/apache/baremaps/csv/CsvDataTable.java | 343 ++++++++++-----------
.../baremaps/csv/CsvDataTableGeonamesTest.java | 2 +-
.../org/apache/baremaps/csv/CsvDataTableTest.java | 15 +-
.../baremaps/data/collection/AppendOnlyLog.java | 7 +-
.../org/apache/baremaps/data/memory/Memory.java | 57 +++-
.../data/memory/MemoryMappedDirectory.java | 21 +-
.../baremaps/data/memory/MemoryMappedFile.java | 21 +-
.../apache/baremaps/data/memory/OffHeapMemory.java | 12 +-
.../apache/baremaps/data/memory/OnHeapMemory.java | 12 +-
.../java/org/apache/baremaps/store/DataSchema.java | 19 +-
14 files changed, 476 insertions(+), 299 deletions(-)
diff --git a/baremaps-calcite/src/main/java/org/apache/baremaps/calcite/BaremapsTableFactory.java b/baremaps-calcite/src/main/java/org/apache/baremaps/calcite/BaremapsTableFactory.java
index bd0c99b31..e53de427b 100644
--- a/baremaps-calcite/src/main/java/org/apache/baremaps/calcite/BaremapsTableFactory.java
+++ b/baremaps-calcite/src/main/java/org/apache/baremaps/calcite/BaremapsTableFactory.java
@@ -17,37 +17,82 @@
package org.apache.baremaps.calcite;
-import java.io.File;
-import java.io.IOException;
-import java.util.Map;
import org.apache.baremaps.csv.CsvDataTable;
+import org.apache.baremaps.data.collection.AppendOnlyLog;
+import org.apache.baremaps.data.collection.DataCollection;
+import org.apache.baremaps.data.collection.IndexedDataList;
+import org.apache.baremaps.data.memory.Memory;
+import org.apache.baremaps.data.memory.MemoryMappedDirectory;
+import org.apache.baremaps.data.store.DataTableImpl;
+import org.apache.baremaps.data.type.DataType;
+import org.apache.baremaps.data.type.DataTypeImpl;
+import org.apache.baremaps.store.DataRow;
+import org.apache.baremaps.store.DataSchema;
+import org.apache.baremaps.store.DataTable;
import org.apache.calcite.rel.type.RelDataType;
import org.apache.calcite.rel.type.RelDataTypeImpl;
import org.apache.calcite.rel.type.RelProtoDataType;
import org.apache.calcite.schema.SchemaPlus;
import org.apache.calcite.schema.TableFactory;
+import java.io.File;
+import java.io.IOException;
+import java.nio.MappedByteBuffer;
+import java.nio.file.Path;
+import java.nio.file.Paths;
+import java.util.Map;
+
public class BaremapsTableFactory implements TableFactory<BaremapsTable> {
- public BaremapsTableFactory() {
-
- }
-
- @Override
- public BaremapsTable create(SchemaPlus schema, String name,
- Map<String, Object> operand, RelDataType rowType) {
- final RelProtoDataType protoRowType =
- rowType != null ? RelDataTypeImpl.proto(rowType) : null;
- String file = (String) operand.get("file");
- if (file != null && file.endsWith(".csv")) {
- try {
- return new BaremapsTable(
- new CsvDataTable(new File(file), true),
- protoRowType);
- } catch (IOException e) {
- throw new RuntimeException(e);
- }
+ public BaremapsTableFactory() {
+
}
- return new BaremapsTable(null, protoRowType);
- }
+
+ @Override
+ public BaremapsTable create(
+ SchemaPlus schema,
+ String name,
+ Map<String, Object> operand,
+ RelDataType rowType) {
+ final RelProtoDataType protoRowType =
+ rowType != null ? RelDataTypeImpl.proto(rowType) : null;
+ String format = (String) operand.get("format");
+ DataTable dataTable = switch (format) {
+ case "mmap" -> createMMapTable(schema, name, operand, rowType);
+ case "csv" -> createCsvTable(schema, name, operand, rowType);
+ default -> throw new RuntimeException("Unsupported format");
+ };
+ return new BaremapsTable(dataTable, protoRowType);
+ }
+
+ private DataTable createMMapTable(SchemaPlus schema, String name, Map<String, Object> operand, RelDataType rowType) {
+ String directory = (String) operand.get("directory");
+ if (directory == null) {
+ throw new RuntimeException("A directory should be specified");
+ }
+ try {
+ Path directoryPath = Paths.get(directory);
+ Path schemaFile = directoryPath.resolve("schema.json");
+ DataSchema dataSchema = DataSchema.read(schemaFile);
+ DataType<DataRow> dataType = new DataTypeImpl(dataSchema);
+ Memory<MappedByteBuffer> memory = new MemoryMappedDirectory(directoryPath);
+ DataCollection<DataRow> dataCollection = new AppendOnlyLog<>(dataType, memory);
+ return new DataTableImpl(dataSchema, dataCollection);
+ } catch (IOException e) {
+ throw new RuntimeException(e);
+ }
+ }
+
+ private DataTable createCsvTable(SchemaPlus schema, String name, Map<String, Object> operand, RelDataType rowType) {
+ String file = (String) operand.get("file");
+ if (file == null) {
+ throw new RuntimeException("A file should be specified");
+ }
+ try {
+ return new CsvDataTable(name, new File(file), true, ',');
+ } catch (IOException e) {
+ throw new RuntimeException(e);
+ }
+ }
+
}
diff --git a/baremaps-calcite/src/test/java/org/apache/baremaps/calcite/BaremapsTableFactoryTest.java b/baremaps-calcite/src/test/java/org/apache/baremaps/calcite/BaremapsTableFactoryTest.java
new file mode 100644
index 000000000..1598163e7
--- /dev/null
+++ b/baremaps-calcite/src/test/java/org/apache/baremaps/calcite/BaremapsTableFactoryTest.java
@@ -0,0 +1,118 @@
+package org.apache.baremaps.calcite;
+
+import org.apache.baremaps.data.collection.AppendOnlyLog;
+import org.apache.baremaps.data.memory.MemoryMappedDirectory;
+import org.apache.baremaps.data.store.DataTableImpl;
+import org.apache.baremaps.data.type.DataTypeImpl;
+import org.apache.baremaps.data.util.FileUtils;
+import org.apache.baremaps.store.*;
+import org.apache.baremaps.store.DataColumn.Cardinality;
+import org.apache.baremaps.store.DataColumn.ColumnType;
+import org.junit.jupiter.api.Test;
+import org.locationtech.jts.geom.Coordinate;
+import org.locationtech.jts.geom.GeometryFactory;
+
+import java.io.File;
+import java.io.PrintWriter;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.sql.Connection;
+import java.sql.DriverManager;
+import java.sql.ResultSet;
+import java.util.List;
+
+class BaremapsTableFactoryTest {
+
+
+ @Test
+ public void createCsvTable() throws Exception {
+ File file = File.createTempFile("test", ".csv");
+ String csv = """
+ ID,NAME,GEOM
+ 1,Paris,POINT(2.3522 48.8566)
+ 2,New York,POINT(-74.0060 40.7128)
+ """;
+ try (PrintWriter writer = new PrintWriter(file)) {
+ writer.write(csv);
+ }
+ String model = """
+ {
+ version: '1.0',
+ defaultSchema: 'TEST',
+ schemas: [
+ {
+ name: 'TEST',
+ tables: [
+ {
+ name: 'TEST',
+ factory: 'org.apache.baremaps.calcite.BaremapsTableFactory',
+ operand: {
+ format: 'csv',
+ file: '%s'
+ }
+ }
+ ]
+ }
+ ]
+ }
+ """.formatted(file.getAbsolutePath());
+ try (Connection connection =
+ DriverManager.getConnection("jdbc:calcite:model=inline:" + model)) {
+
+ ResultSet resultSet = connection.createStatement().executeQuery("SELECT * FROM TEST.TEST");
+ while (resultSet.next()) {
+ System.out.println(resultSet.getString("ID") + " " + resultSet.getString("GEOM"));
+ }
+ } finally {
+ file.delete();
+ }
+ }
+
+ @Test
+ public void createMMapTable() throws Exception {
+ Path path = Files.createTempDirectory("temp");
+
+ DataSchema dataSchema = new DataSchemaImpl("test", List.of(
+ new DataColumnFixed("id", Cardinality.REQUIRED, ColumnType.INTEGER),
+ new DataColumnFixed("name", Cardinality.REQUIRED, ColumnType.STRING),
+ new DataColumnFixed("geom", Cardinality.REQUIRED, ColumnType.GEOMETRY)
+ ));
+
+ DataTable dataTable = new DataTableImpl(dataSchema, new AppendOnlyLog<>(new DataTypeImpl(dataSchema), new MemoryMappedDirectory(path)));
+ dataTable.add(new DataRowImpl(dataSchema, List.of(1, "a", new GeometryFactory().createPoint(new Coordinate(1, 1)))));
+ dataTable.add(new DataRowImpl(dataSchema, List.of(2, "b", new GeometryFactory().createPoint(new Coordinate(2, 2)))));
+ dataTable.close();
+
+ String model = """
+ {
+ version: '1.0',
+ defaultSchema: 'TEST',
+ schemas: [
+ {
+ name: 'TEST',
+ tables: [
+ {
+ name: 'TEST',
+ factory: 'org.apache.baremaps.calcite.BaremapsTableFactory',
+ operand: {
+ format: 'mmap',
+ directory: '%s'
+ }
+ }
+ ]
+ }
+ ]
+ }
+ """.formatted(path.toAbsolutePath());
+ try (Connection connection =
+ DriverManager.getConnection("jdbc:calcite:model=inline:" + model)) {
+
+ ResultSet resultSet = connection.createStatement().executeQuery("SELECT * FROM TEST.TEST");
+ while (resultSet.next()) {
+ System.out.println(resultSet.getString("ID") + " " + resultSet.getString("GEOM"));
+ }
+ } finally {
+ FileUtils.deleteRecursively(path);
+ }
+ }
+}
\ No newline at end of file
diff --git a/baremaps-calcite/src/test/java/com/apache/baremaps/calcite/CalciteTest.java b/baremaps-calcite/src/test/java/org/apache/baremaps/calcite/CalciteTest.java
similarity index 86%
rename from baremaps-calcite/src/test/java/com/apache/baremaps/calcite/CalciteTest.java
rename to baremaps-calcite/src/test/java/org/apache/baremaps/calcite/CalciteTest.java
index 870263c8f..b68d2261c 100644
--- a/baremaps-calcite/src/test/java/com/apache/baremaps/calcite/CalciteTest.java
+++ b/baremaps-calcite/src/test/java/org/apache/baremaps/calcite/CalciteTest.java
@@ -15,13 +15,11 @@
* limitations under the License.
*/
-package com.apache.baremaps.calcite;
+package org.apache.baremaps.calcite;
-import java.io.File;
-import java.io.PrintWriter;
import java.sql.*;
import java.util.*;
-import org.apache.baremaps.calcite.BaremapsTable;
+
import org.apache.baremaps.data.collection.AppendOnlyLog;
import org.apache.baremaps.data.collection.IndexedDataList;
import org.apache.baremaps.data.store.DataTableImpl;
@@ -247,50 +245,5 @@ public class CalciteTest {
System.out.println("List B (after SQL): " + listB);
}
- @Test
- public void testCsvStream() throws Exception {
- File file = File.createTempFile("test", ".csv");
- String csv = """
- ID,NAME,GEOM
- 1,Paris,POINT(2.3522 48.8566)
- 2,New York,POINT(-74.0060 40.7128)
- """;
- try (PrintWriter writer = new PrintWriter(file)) {
- writer.write(csv);
- }
-
- String model = """
- {
- version: '1.0',
- defaultSchema: 'TEST',
- schemas: [
- {
- name: 'TEST',
- tables: [
- {
- name: 'TEST',
- type: 'custom',
- factory: 'org.apache.baremaps.calcite.BaremapsTableFactory',
- operand: {
- file: '%s'
- }
- }
- ]
- }
- ]
- }
- """.formatted(file.getAbsolutePath());
-
- try (Connection connection =
- DriverManager.getConnection("jdbc:calcite:model=inline:" + model)) {
-
- ResultSet resultSet = connection.createStatement().executeQuery("SELECT * FROM TEST.TEST");
- while (resultSet.next()) {
- System.out.println(resultSet.getString("ID") + " " + resultSet.getString("GEOM"));
- }
- } finally {
- file.delete();
- }
- }
}
diff --git a/baremaps-csv/src/main/java/org/apache/baremaps/csv/CsvDataStore.java b/baremaps-csv/src/main/java/org/apache/baremaps/csv/CsvDataStore.java
index 74abb32e3..9bfc41aa2 100644
--- a/baremaps-csv/src/main/java/org/apache/baremaps/csv/CsvDataStore.java
+++ b/baremaps-csv/src/main/java/org/apache/baremaps/csv/CsvDataStore.java
@@ -41,15 +41,15 @@ public class CsvDataStore implements DataStore {
* @param tableName the name of the table
* @param schema the data schema defining the structure
* @param csvFile the CSV file to read data from
- * @param hasHeader whether the CSV file includes a header row
+ * @param header whether the CSV file includes a header row
* @param separator the character used to separate columns in the CSV file
* @throws IOException if an I/O error occurs
*/
- public CsvDataStore(String tableName, DataSchema schema, File csvFile, boolean hasHeader,
+ public CsvDataStore(String tableName, DataSchema schema, File csvFile, boolean header,
char separator) throws IOException {
this.tableName = tableName;
this.schema = schema;
- this.dataTable = new CsvDataTable(csvFile, hasHeader);
+ this.dataTable = new CsvDataTable(tableName, csvFile, header, separator);
}
/**
diff --git a/baremaps-csv/src/main/java/org/apache/baremaps/csv/CsvDataTable.java b/baremaps-csv/src/main/java/org/apache/baremaps/csv/CsvDataTable.java
index 360debfae..03283ec8a 100644
--- a/baremaps-csv/src/main/java/org/apache/baremaps/csv/CsvDataTable.java
+++ b/baremaps-csv/src/main/java/org/apache/baremaps/csv/CsvDataTable.java
@@ -17,208 +17,195 @@
package org.apache.baremaps.csv;
+import com.fasterxml.jackson.core.FormatSchema;
import com.fasterxml.jackson.core.JsonParser;
import com.fasterxml.jackson.core.JsonToken;
+import com.fasterxml.jackson.databind.MappingIterator;
import com.fasterxml.jackson.dataformat.csv.CsvMapper;
import com.fasterxml.jackson.dataformat.csv.CsvSchema;
+import org.apache.baremaps.store.*;
+import org.apache.baremaps.store.DataColumn.Cardinality;
+import org.apache.baremaps.store.DataColumn.ColumnType;
+import org.locationtech.jts.io.WKTReader;
+
import java.io.File;
import java.io.IOException;
import java.util.*;
import java.util.stream.Stream;
import java.util.stream.StreamSupport;
-import org.apache.baremaps.store.*;
-import org.apache.baremaps.store.DataColumn.Cardinality;
-import org.apache.baremaps.store.DataColumn.ColumnType;
-import org.locationtech.jts.io.WKTReader;
/**
* A DataTable implementation that reads data from a CSV file using Jackson.
*/
public class CsvDataTable implements DataTable {
- private final File csvFile;
- private final CsvSchema csvSchema;
- private final DataSchema dataSchema;
-
- private final long size;
- private JsonParser parser;
-
- /**
- * Constructs a CsvDataTable with the specified schema, CSV file, header presence, and separator.
- *
- * @param csvFile the CSV file to read data from
- * @param hasHeader whether the CSV file includes a header row
- * @throws IOException if an I/O error occurs
- */
- public CsvDataTable(File csvFile, boolean hasHeader) throws IOException {
- this.csvFile = csvFile;
- this.csvSchema = inferCsvSchema(csvFile);
- this.dataSchema = createDataSchema(csvFile.getName(), csvSchema);
- this.size = calculateSize();
- }
-
-
- private CsvSchema inferCsvSchema(File csvFile) {
- CsvSchema csvSchema = CsvSchema.emptySchema().withUseHeader(true).withColumnSeparator(',');
- try (var ignored = new CsvMapper().readerFor(Map.class)
- .with(csvSchema)
- .createParser(csvFile)) {
- var test = ignored.readValueAsTree();
- return csvSchema;
- } catch (IOException e) {
- throw new DataStoreException("Error reading CSV file", e);
+ private final File file;
+ private final CsvSchema csvSchema;
+ private final DataSchema dataSchema;
+
+ private final long size;
+ private MappingIterator<Map<String, String>> csvIterator;
+
+ /**
+ * Constructs a CsvDataTable with the specified schema, CSV file, header presence, and separator.
+ *
+ * @param name the name of the table
+ * @param csvFile the CSV file to read data from
+ * @param header whether the CSV file includes a header row
+ * @param separator the separator used in the CSV file
+ * @throws IOException if an I/O error occurs
+ */
+ public CsvDataTable(String name, File csvFile, boolean header, char separator) throws IOException {
+ this.file = csvFile;
+
+ // Iterate over all records to infer the csv schema and calculate the size of the table
+ CsvSchema csvSchema = CsvSchema.emptySchema()
+ .withUseHeader(header)
+ .withColumnSeparator(separator);
+
+ try (MappingIterator<Object> iterator = new CsvMapper()
+ .readerFor(Object.class)
+ .with(csvSchema)
+ .readValues(csvFile)) {
+
+ int count = 0;
+ while (iterator.hasNext()) {
+ iterator.next();
+ count++;
+ }
+
+ this.csvSchema = (CsvSchema) iterator.getParserSchema();
+ this.size = count;
+ } catch (IOException e) {
+ throw new DataStoreException("Error reading CSV file", e);
+ }
+
+ // Map the csv schema to a data schema
+ List<DataColumn> columns = new ArrayList<>();
+ for (String columnName : this.csvSchema.getColumnNames()) {
+ switch (this.csvSchema.column(columnName).getType()) {
+ case STRING -> columns
+ .add(new DataColumnFixed(columnName, Cardinality.REQUIRED, ColumnType.STRING));
+ case NUMBER -> columns
+ .add(new DataColumnFixed(columnName, Cardinality.REQUIRED, ColumnType.DOUBLE));
+ case BOOLEAN -> columns
+ .add(new DataColumnFixed(columnName, Cardinality.REQUIRED, ColumnType.BOOLEAN));
+ case ARRAY -> columns
+ .add(new DataColumnFixed(columnName, Cardinality.REPEATED, ColumnType.STRING));
+ default -> throw new IllegalArgumentException(
+ "Unsupported column type: " + csvSchema.column(columnName).getType());
+ }
+ }
+ this.dataSchema = new DataSchemaImpl(name, columns);
}
- }
-
- private DataSchema createDataSchema(String name, CsvSchema csvSchema) {
- List<DataColumn> columns = new ArrayList<>();
- for (String columnName : csvSchema.getColumnNames()) {
- switch (csvSchema.column(columnName).getType()) {
- case STRING -> columns
- .add(new DataColumnFixed(columnName, Cardinality.REQUIRED, ColumnType.STRING));
- case NUMBER -> columns
- .add(new DataColumnFixed(columnName, Cardinality.REQUIRED, ColumnType.DOUBLE));
- case BOOLEAN -> columns
- .add(new DataColumnFixed(columnName, Cardinality.REQUIRED, ColumnType.BOOLEAN));
- case ARRAY -> columns
- .add(new DataColumnFixed(columnName, Cardinality.REPEATED, ColumnType.STRING));
- default -> throw new IllegalArgumentException(
- "Unsupported column type: " + csvSchema.column(columnName).getType());
- }
+
+ @Override
+ public DataSchema schema() {
+ return dataSchema;
}
- return new DataSchemaImpl(name, columns);
- }
-
- /**
- * Calculates the number of rows in the CSV file.
- *
- * @return the number of rows
- * @throws IOException if an I/O error occurs
- */
- private long calculateSize() throws IOException {
- try (var parser = new CsvMapper().readerFor(Map.class)
- .with(csvSchema)
- .createParser(csvFile)) {
- long rowCount = 0;
- while (parser.nextToken() != null) {
- if (parser.currentToken() == JsonToken.START_OBJECT) {
- rowCount++;
- }
- }
- return rowCount;
+
+ @Override
+ public boolean add(DataRow row) {
+ throw new UnsupportedOperationException("Adding rows is not supported.");
+ }
+
+ @Override
+ public void clear() {
+ throw new UnsupportedOperationException("Clearing rows is not supported.");
+ }
+
+ @Override
+ public long size() {
+ return size;
}
- }
-
- @Override
- public DataSchema schema() {
- return dataSchema;
- }
-
- @Override
- public boolean add(DataRow row) {
- throw new UnsupportedOperationException("Adding rows is not supported.");
- }
-
- @Override
- public void clear() {
- throw new UnsupportedOperationException("Clearing rows is not supported.");
- }
-
- @Override
- public long size() {
- return size;
- }
-
- @Override
- public Iterator<DataRow> iterator() {
- try {
- CsvMapper csvMapper = new CsvMapper();
- parser = csvMapper.readerFor(Map.class)
- .with(csvSchema)
- .createParser(csvFile);
-
- Iterator<Map<String, String>> csvIterator = csvMapper.readerFor(Map.class)
- .with(csvSchema)
- .readValues(parser);
-
- return new Iterator<>() {
- @Override
- public boolean hasNext() {
- return csvIterator.hasNext();
+
+ @Override
+ public Iterator<DataRow> iterator() {
+ try {
+
+ csvIterator = new CsvMapper()
+ .readerFor(Map.class)
+ .with(csvSchema)
+ .readValues(file);
+
+ return new Iterator<>() {
+ @Override
+ public boolean hasNext() {
+ return csvIterator.hasNext();
+ }
+
+ @Override
+ public DataRow next() {
+ Map<String, String> csvRow = csvIterator.next();
+ DataRow dataRow = dataSchema.createRow();
+
+ for (int i = 0; i < dataSchema.columns().size(); i++) {
+ DataColumn column = dataSchema.columns().get(i);
+ String columnName = column.name();
+ String value = csvRow.get(columnName);
+
+ if (value != null) {
+ Object parsedValue = parseValue(column, value);
+ dataRow.set(i, parsedValue);
+ } else {
+ dataRow.set(i, null);
+ }
+ }
+ return dataRow;
+ }
+ };
+
+ } catch (IOException e) {
+ throw new DataStoreException("Error reading CSV file", e);
}
+ }
- @Override
- public DataRow next() {
- Map<String, String> csvRow = csvIterator.next();
- DataRow dataRow = dataSchema.createRow();
-
- for (int i = 0; i < dataSchema.columns().size(); i++) {
- DataColumn column = dataSchema.columns().get(i);
- String columnName = column.name();
- String value = csvRow.get(columnName);
-
- if (value != null) {
- Object parsedValue = parseValue(column, value);
- dataRow.set(i, parsedValue);
- } else {
- dataRow.set(i, null);
+ /**
+ * Parses the string value from the CSV according to the column type.
+ *
+ * @param column the data column
+ * @param value the string value from the CSV
+ * @return the parsed value
+ */
+ private Object parseValue(DataColumn column, String value) {
+ ColumnType type = column.type();
+ try {
+ if (value == null || value.isEmpty()) {
+ return null;
}
- }
- return dataRow;
+ return switch (type) {
+ case STRING -> value;
+ case INTEGER -> Integer.parseInt(value);
+ case LONG -> Long.parseLong(value);
+ case FLOAT -> Float.parseFloat(value);
+ case DOUBLE -> Double.parseDouble(value);
+ case BOOLEAN -> Boolean.parseBoolean(value);
+ case GEOMETRY, POINT, LINESTRING, POLYGON, MULTIPOINT, MULTILINESTRING, MULTIPOLYGON,
+ GEOMETRYCOLLECTION -> {
+ WKTReader reader = new WKTReader();
+ yield reader.read(value);
+ }
+ default -> throw new IllegalArgumentException("Unsupported column type: " + type);
+ };
+ } catch (Exception e) {
+ throw new DataStoreException("Error parsing value for column " + column.name(), e);
}
- };
+ }
- } catch (IOException e) {
- throw new DataStoreException("Error reading CSV file", e);
+ @Override
+ public Spliterator<DataRow> spliterator() {
+ return Spliterators.spliteratorUnknownSize(iterator(), Spliterator.ORDERED);
}
- }
-
- /**
- * Parses the string value from the CSV according to the column type.
- *
- * @param column the data column
- * @param value the string value from the CSV
- * @return the parsed value
- */
- private Object parseValue(DataColumn column, String value) {
- ColumnType type = column.type();
- try {
- if (value == null || value.isEmpty()) {
- return null;
- }
- return switch (type) {
- case STRING -> value;
- case INTEGER -> Integer.parseInt(value);
- case LONG -> Long.parseLong(value);
- case FLOAT -> Float.parseFloat(value);
- case DOUBLE -> Double.parseDouble(value);
- case BOOLEAN -> Boolean.parseBoolean(value);
- case GEOMETRY, POINT, LINESTRING, POLYGON, MULTIPOINT, MULTILINESTRING, MULTIPOLYGON,
- GEOMETRYCOLLECTION -> {
- WKTReader reader = new WKTReader();
- yield reader.read(value);
- }
- default -> throw new IllegalArgumentException("Unsupported column type: " + type);
- };
- } catch (Exception e) {
- throw new DataStoreException("Error parsing value for column " + column.name(), e);
+
+ @Override
+ public Stream<DataRow> stream() {
+ return StreamSupport.stream(spliterator(), false);
}
- }
-
- @Override
- public Spliterator<DataRow> spliterator() {
- return Spliterators.spliteratorUnknownSize(iterator(), Spliterator.ORDERED);
- }
-
- @Override
- public Stream<DataRow> stream() {
- return StreamSupport.stream(spliterator(), false);
- }
-
- @Override
- public void close() throws IOException {
- if (parser != null) {
- parser.close();
+
+ @Override
+ public void close() throws IOException {
+ if (csvIterator != null) {
+ csvIterator.close();
+ }
}
- }
}
diff --git a/baremaps-csv/src/test/java/org/apache/baremaps/csv/CsvDataTableGeonamesTest.java b/baremaps-csv/src/test/java/org/apache/baremaps/csv/CsvDataTableGeonamesTest.java
index 489634505..71776b724 100644
--- a/baremaps-csv/src/test/java/org/apache/baremaps/csv/CsvDataTableGeonamesTest.java
+++ b/baremaps-csv/src/test/java/org/apache/baremaps/csv/CsvDataTableGeonamesTest.java
@@ -29,7 +29,7 @@ class CsvDataTableGeonamesTest {
@Test
void testGeonamesCsvDataTable() throws IOException {
- DataTable dataTable = new CsvDataTable(GEONAMES_CSV.toFile(), false);
+ DataTable dataTable = new CsvDataTable("sample", GEONAMES_CSV.toFile(), false, '\t');
assertEquals(5, dataTable.size(), "DataTable should have 5 rows.");
diff --git a/baremaps-csv/src/test/java/org/apache/baremaps/csv/CsvDataTableTest.java b/baremaps-csv/src/test/java/org/apache/baremaps/csv/CsvDataTableTest.java
index 444b37996..bb67934df 100644
--- a/baremaps-csv/src/test/java/org/apache/baremaps/csv/CsvDataTableTest.java
+++ b/baremaps-csv/src/test/java/org/apache/baremaps/csv/CsvDataTableTest.java
@@ -58,7 +58,7 @@ class CsvDataTableTest {
2,PointB,"POINT(2 2)"
""";
Files.writeString(tempCsvFile.toPath(), csvContent);
- DataTable dataTable = new CsvDataTable(tempCsvFile, true);
+ DataTable dataTable = new CsvDataTable("test_table", tempCsvFile, true, ',');
assertEquals(2, dataTable.size());
int rowCount = 0;
for (DataRow row : dataTable) {
@@ -86,8 +86,7 @@ class CsvDataTableTest {
new DataColumnFixed("column1", Cardinality.REQUIRED, ColumnType.INTEGER),
new DataColumnFixed("column2", Cardinality.OPTIONAL, ColumnType.STRING),
new DataColumnFixed("column3", Cardinality.OPTIONAL, ColumnType.GEOMETRY));
- DataSchema schema = new DataSchemaImpl("test_table", columns);
- DataTable dataTable = new CsvDataTable(tempCsvFile, false);
+ DataTable dataTable = new CsvDataTable("test_table", tempCsvFile, false, ';');
assertEquals(2, dataTable.size());
int rowCount = 0;
for (DataRow row : dataTable) {
@@ -120,8 +119,7 @@ class CsvDataTableTest {
new DataColumnFixed("double_col", Cardinality.REQUIRED, ColumnType.DOUBLE),
new DataColumnFixed("bool_col", Cardinality.REQUIRED, ColumnType.BOOLEAN),
new DataColumnFixed("string_col", Cardinality.REQUIRED, ColumnType.STRING));
- DataSchema schema = new DataSchemaImpl("test_table", columns);
- DataTable dataTable = new CsvDataTable(tempCsvFile, true);
+ DataTable dataTable = new CsvDataTable("test_table", tempCsvFile, true, ',');
assertEquals(2, dataTable.size());
int rowCount = 0;
for (DataRow row : dataTable) {
@@ -150,8 +148,7 @@ class CsvDataTableTest {
List<DataColumn> columns = List.of(
new DataColumnFixed("id", Cardinality.REQUIRED, ColumnType.INTEGER),
new DataColumnFixed("name", Cardinality.OPTIONAL, ColumnType.STRING));
- DataSchema schema = new DataSchemaImpl("test_table", columns);
- DataTable dataTable = new CsvDataTable(tempCsvFile, true);
+ DataTable dataTable = new CsvDataTable("test_table", tempCsvFile, true, ',');
assertThrows(RuntimeException.class, () -> {
for (DataRow row : dataTable) {
// This line should throw an exception because abc is not a valid integer
@@ -164,7 +161,7 @@ class CsvDataTableTest {
void testAddAndClearUnsupportedOperations() throws IOException {
String csvContent = "";
Files.writeString(tempCsvFile.toPath(), csvContent);
- DataTable dataTable = new CsvDataTable(tempCsvFile, true);
+ DataTable dataTable = new CsvDataTable("test_table", tempCsvFile, true, ',');
assertThrows(UnsupportedOperationException.class, () -> dataTable.add(null));
assertThrows(UnsupportedOperationException.class, dataTable::clear);
}
@@ -178,7 +175,7 @@ class CsvDataTableTest {
3,Name3
""";
Files.writeString(tempCsvFile.toPath(), csvContent);
- DataTable dataTable = new CsvDataTable(tempCsvFile, true);
+ DataTable dataTable = new CsvDataTable("test_table", tempCsvFile, true, ',');
assertEquals(3, dataTable.size());
}
}
diff --git a/baremaps-data/src/main/java/org/apache/baremaps/data/collection/AppendOnlyLog.java b/baremaps-data/src/main/java/org/apache/baremaps/data/collection/AppendOnlyLog.java
index 1a9830b1b..59a13f6e4 100644
--- a/baremaps-data/src/main/java/org/apache/baremaps/data/collection/AppendOnlyLog.java
+++ b/baremaps-data/src/main/java/org/apache/baremaps/data/collection/AppendOnlyLog.java
@@ -65,8 +65,8 @@ public class AppendOnlyLog<E> implements DataCollection<E> {
this.dataType = dataType;
this.memory = memory;
this.segmentSize = memory.segmentSize();
- this.offset = Long.BYTES;
- this.size = memory.segment(0).getLong(0);
+ this.size = memory.header().getLong(0);
+ this.offset = 0;
}
/**
@@ -148,6 +148,7 @@ public class AppendOnlyLog<E> implements DataCollection<E> {
@Override
public void close() throws IOException {
+ memory.header().putLong(0, size);
memory.close();
}
@@ -166,7 +167,7 @@ public class AppendOnlyLog<E> implements DataCollection<E> {
private AppendOnlyLogIterator(long size) {
this.size = size;
index = 0;
- position = Long.BYTES;
+ position = 0;
}
@Override
diff --git a/baremaps-data/src/main/java/org/apache/baremaps/data/memory/Memory.java b/baremaps-data/src/main/java/org/apache/baremaps/data/memory/Memory.java
index d0a2580bf..a8f159cc6 100644
--- a/baremaps-data/src/main/java/org/apache/baremaps/data/memory/Memory.java
+++ b/baremaps-data/src/main/java/org/apache/baremaps/data/memory/Memory.java
@@ -28,28 +28,42 @@ import java.util.List;
/** A base class to manage segments of on-heap, off-heap, or on-disk memory. */
public abstract class Memory<T extends ByteBuffer> implements Closeable {
+ private final int headerSize;
+
private final int segmentSize;
private final long segmentShift;
private final long segmentMask;
- protected final List<T> segments = new ArrayList<>();
+ protected T header;
+
+ protected List<T> segments = new ArrayList<>();
/**
* Constructs a memory with a given segment size.
*
* @param segmentSize the size of the segments
*/
- protected Memory(int segmentSize) {
+ protected Memory(int headerSize, int segmentSize) {
if ((segmentSize & -segmentSize) != segmentSize) {
throw new IllegalArgumentException("The segment size must be a power of 2");
}
+ this.headerSize = headerSize;
this.segmentSize = segmentSize;
this.segmentShift = (long) (Math.log(this.segmentSize) / Math.log(2));
this.segmentMask = this.segmentSize - 1l;
}
+ /**
+ * Returns the size of the header.
+ *
+ * @return the size of the header
+ */
+ public int headerSize() {
+ return headerSize;
+ }
+
/**
* Returns the size of the segments.
*
@@ -77,6 +91,22 @@ public abstract class Memory<T extends ByteBuffer> implements Closeable {
return segmentMask;
}
+ public ByteBuffer header() {
+ if (header == null) {
+ synchronized (this) {
+ header = allocateHeader();
+ }
+ }
+ return header;
+ }
+
+ /**
+ * Allocates a header.
+ *
+ * @return the header
+ */
+ protected abstract T allocateHeader();
+
/**
* Returns a segment of the memory.
*
@@ -85,47 +115,44 @@ public abstract class Memory<T extends ByteBuffer> implements Closeable {
*/
public ByteBuffer segment(int index) {
if (segments.size() <= index) {
- return allocate(index);
+ return allocateSegmentInternal(index);
}
ByteBuffer segment = segments.get(index);
if (segment == null) {
- return allocate(index);
+ return allocateSegmentInternal(index);
}
return segment;
}
+ /** Returns the size of the allocated memory. */
+ public long size() {
+ return (long) segments.size() * (long) segmentSize;
+ }
+
/** The allocation of segments is synchronized to enable access by multiple threads. */
- private synchronized ByteBuffer allocate(int index) {
+ private synchronized ByteBuffer allocateSegmentInternal(int index) {
while (segments.size() <= index) {
segments.add(null);
}
T segment = segments.get(index);
if (segment == null) {
- segment = allocate(index, segmentSize);
+ segment = allocateSegment(index);
segments.set(index, segment);
}
return segment;
}
- /** Returns the size of the allocated memory. */
- public long size() {
- return (long) segments.size() * (long) segmentSize;
- }
-
/**
* Allocates a segment for a given index and size.
*
* @param index the index of the segment
- * @param size the size of the segment
* @return the segment
*/
- protected abstract T allocate(int index, int size);
+ protected abstract T allocateSegment(int index);
/**
* Clears the memory and the underlying resources.
*/
public abstract void clear() throws IOException;
-
-
}
diff --git a/baremaps-data/src/main/java/org/apache/baremaps/data/memory/MemoryMappedDirectory.java b/baremaps-data/src/main/java/org/apache/baremaps/data/memory/MemoryMappedDirectory.java
index 82d2e0af7..1b9a18dfa 100644
--- a/baremaps-data/src/main/java/org/apache/baremaps/data/memory/MemoryMappedDirectory.java
+++ b/baremaps-data/src/main/java/org/apache/baremaps/data/memory/MemoryMappedDirectory.java
@@ -53,18 +53,31 @@ public class MemoryMappedDirectory extends Memory<MappedByteBuffer> {
* @param segmentBytes the size of the segments in bytes
*/
public MemoryMappedDirectory(Path directory, int segmentBytes) {
- super(segmentBytes);
+ super(1024, segmentBytes);
this.directory = directory;
}
+ @Override
+ protected MappedByteBuffer allocateHeader() {
+ try {
+ Path file = directory.resolve("header");
+ try (FileChannel channel = FileChannel.open(file, StandardOpenOption.CREATE,
+ StandardOpenOption.READ, StandardOpenOption.WRITE)) {
+ return channel.map(MapMode.READ_WRITE, 0, 1024);
+ }
+ } catch (IOException e) {
+ throw new MemoryException(e);
+ }
+ }
+
/** {@inheritDoc} */
@Override
- protected MappedByteBuffer allocate(int index, int size) {
+ protected MappedByteBuffer allocateSegment(int index) {
try {
Path file = directory.resolve(String.format("%s.part", index));
try (FileChannel channel = FileChannel.open(file, StandardOpenOption.CREATE,
StandardOpenOption.READ, StandardOpenOption.WRITE)) {
- return channel.map(MapMode.READ_WRITE, 0, size);
+ return channel.map(MapMode.READ_WRITE, 0, segmentSize());
}
} catch (IOException e) {
throw new MemoryException(e);
@@ -74,6 +87,7 @@ public class MemoryMappedDirectory extends Memory<MappedByteBuffer> {
/** {@inheritDoc} */
@Override
public void close() throws IOException {
+ MappedByteBufferUtils.unmap(header);
for (MappedByteBuffer buffer : segments) {
MappedByteBufferUtils.unmap(buffer);
}
@@ -83,6 +97,7 @@ public class MemoryMappedDirectory extends Memory<MappedByteBuffer> {
@Override
public void clear() throws IOException {
close();
+ header.clear();
segments.clear();
FileUtils.deleteRecursively(directory);
}
diff --git a/baremaps-data/src/main/java/org/apache/baremaps/data/memory/MemoryMappedFile.java b/baremaps-data/src/main/java/org/apache/baremaps/data/memory/MemoryMappedFile.java
index 5db613d34..7c2de6b8b 100644
--- a/baremaps-data/src/main/java/org/apache/baremaps/data/memory/MemoryMappedFile.java
+++ b/baremaps-data/src/main/java/org/apache/baremaps/data/memory/MemoryMappedFile.java
@@ -49,17 +49,32 @@ public class MemoryMappedFile extends Memory<MappedByteBuffer> {
* @param segmentBytes the size of the segments in bytes
*/
public MemoryMappedFile(Path file, int segmentBytes) {
- super(segmentBytes);
+ super(1024, segmentBytes);
this.file = file;
}
+ @Override
+ protected MappedByteBuffer allocateHeader() {
+ try {
+ try (FileChannel channel = FileChannel.open(file, StandardOpenOption.CREATE,
+ StandardOpenOption.READ, StandardOpenOption.WRITE)) {
+ return channel.map(MapMode.READ_WRITE, 0, headerSize());
+ }
+ } catch (IOException e) {
+ throw new MemoryException(e);
+ }
+ }
+
/** {@inheritDoc} */
@Override
- protected MappedByteBuffer allocate(int index, int size) {
+ protected MappedByteBuffer allocateSegment(int index) {
try {
try (FileChannel channel = FileChannel.open(file, StandardOpenOption.CREATE,
StandardOpenOption.READ, StandardOpenOption.WRITE)) {
- return channel.map(MapMode.READ_WRITE, (long) index * size, size);
+ int headerSize = headerSize();
+ int segmentSize = segmentSize();
+ int segmentPosition = headerSize + index * segmentSize;
+ return channel.map(MapMode.READ_WRITE, segmentPosition, segmentSize);
}
} catch (IOException e) {
throw new MemoryException(e);
diff --git a/baremaps-data/src/main/java/org/apache/baremaps/data/memory/OffHeapMemory.java b/baremaps-data/src/main/java/org/apache/baremaps/data/memory/OffHeapMemory.java
index f95f03b51..8294b68be 100644
--- a/baremaps-data/src/main/java/org/apache/baremaps/data/memory/OffHeapMemory.java
+++ b/baremaps-data/src/main/java/org/apache/baremaps/data/memory/OffHeapMemory.java
@@ -36,13 +36,19 @@ public class OffHeapMemory extends Memory<ByteBuffer> {
* @param segmentSize the size of the segments in bytes
*/
public OffHeapMemory(int segmentSize) {
- super(segmentSize);
+ super(1024, segmentSize);
}
/** {@inheritDoc} */
@Override
- protected ByteBuffer allocate(int index, int size) {
- return ByteBuffer.allocateDirect(size);
+ protected ByteBuffer allocateHeader() {
+ return ByteBuffer.allocateDirect(headerSize());
+ }
+
+ /** {@inheritDoc} */
+ @Override
+ protected ByteBuffer allocateSegment(int index) {
+ return ByteBuffer.allocateDirect(segmentSize());
}
/** {@inheritDoc} */
diff --git a/baremaps-data/src/main/java/org/apache/baremaps/data/memory/OnHeapMemory.java b/baremaps-data/src/main/java/org/apache/baremaps/data/memory/OnHeapMemory.java
index 4b0fff757..5d18bbe40 100644
--- a/baremaps-data/src/main/java/org/apache/baremaps/data/memory/OnHeapMemory.java
+++ b/baremaps-data/src/main/java/org/apache/baremaps/data/memory/OnHeapMemory.java
@@ -36,13 +36,19 @@ public class OnHeapMemory extends Memory<ByteBuffer> {
* @param segmentSize the size of the segments in bytes
*/
public OnHeapMemory(int segmentSize) {
- super(segmentSize);
+ super(1024, segmentSize);
}
/** {@inheritDoc} */
@Override
- protected ByteBuffer allocate(int index, int size) {
- return ByteBuffer.allocate(size);
+ protected ByteBuffer allocateHeader() {
+ return ByteBuffer.allocate(headerSize());
+ }
+
+ /** {@inheritDoc} */
+ @Override
+ protected ByteBuffer allocateSegment(int index) {
+ return ByteBuffer.allocate(segmentSize());
}
/** {@inheritDoc} */
diff --git a/baremaps-store/src/main/java/org/apache/baremaps/store/DataSchema.java b/baremaps-store/src/main/java/org/apache/baremaps/store/DataSchema.java
index aef70f1e7..a2dcfecd8 100644
--- a/baremaps-store/src/main/java/org/apache/baremaps/store/DataSchema.java
+++ b/baremaps-store/src/main/java/org/apache/baremaps/store/DataSchema.java
@@ -27,6 +27,8 @@ import com.fasterxml.jackson.databind.module.SimpleModule;
import com.fasterxml.jackson.databind.node.ObjectNode;
import java.io.IOException;
import java.io.Serializable;
+import java.nio.ByteBuffer;
+import java.nio.charset.StandardCharsets;
import java.nio.file.Path;
import java.util.ArrayList;
import java.util.List;
@@ -36,7 +38,7 @@ import org.apache.baremaps.store.DataColumn.ColumnType;
/**
* A {@link DataSchema} is a description of the structure of a row in a {@link DataTable}.
*/
-public interface DataSchema extends Serializable {
+public interface DataSchema {
/**
* Returns the name of the schema.
@@ -100,14 +102,19 @@ public interface DataSchema extends Serializable {
return mapper;
}
- static DataSchema read(Path path) throws IOException {
+ static DataSchema read(ByteBuffer buffer) throws IOException {
var mapper = configureObjectMapper();
- return mapper.readValue(path.toFile(), DataSchema.class);
+ int length = buffer.getInt();
+ byte[] bytes = new byte[length];
+ buffer.get(bytes);
+ return mapper.readValue(bytes, DataSchema.class);
}
- static void write(Path path, DataSchema schema) throws IOException {
+ static void write(ByteBuffer buffer, DataSchema schema) throws IOException {
var mapper = configureObjectMapper();
- mapper.writerWithDefaultPrettyPrinter().writeValue(path.toFile(), schema);
+ var bytes = mapper.writerWithDefaultPrettyPrinter().writeValueAsBytes(schema);
+ buffer.putInt(bytes.length);
+ buffer.put(bytes);
}
-
+
}
(incubator-baremaps) 06/07: Refactor the sub modules
Posted by bc...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
bchapuis pushed a commit to branch calcite-framework
in repository https://gitbox.apache.org/repos/asf/incubator-baremaps.git
commit a6c7315e4360e4a95c88591e4ab73f23ad9bc9a3
Author: Bertil Chapuis <bc...@gmail.com>
AuthorDate: Tue Jan 21 08:42:41 2025 +0100
Refactor the sub modules
---
baremaps-calcite/pom.xml | 49 ++++++
.../org/apache/baremaps/calcite/BaremapsTable.java | 113 ++++++++++++
.../baremaps/calcite/BaremapsTableFactory.java | 53 ++++++
.../com/apache/baremaps}/calcite/CalciteTest.java | 102 ++++++++---
.../apache/baremaps/geocoder/DataRowMapper.java | 3 +-
.../apache/baremaps/tasks/ImportGeoPackage.java | 26 ++-
.../org/apache/baremaps/calcite/CalciteTest.java | 196 ++++++++++-----------
.../java/org/apache/baremaps/csv/CsvDataStore.java | 2 +-
.../java/org/apache/baremaps/csv/CsvDataTable.java | 77 +++++---
.../baremaps/csv/CsvDataTableGeonamesTest.java | 29 +--
.../org/apache/baremaps/csv/CsvDataTableTest.java | 45 ++---
.../baremaps/data/calcite/SqlDataSchema.java | 38 ----
.../apache/baremaps/data/calcite/SqlDataTable.java | 65 -------
.../baremaps/data/calcite/SqlTypeConversion.java | 69 --------
.../baremaps/data/collection/DataConversions.java | 12 +-
.../data/collection/FixedSizeDataList.java | 6 +
.../baremaps/data/collection/IndexedDataList.java | 7 +
.../data/collection/MemoryAlignedDataList.java | 4 +
.../apache/baremaps/data/store/DataStoreImpl.java | 59 +++++++
.../{BaremapsDataTable.java => DataTableImpl.java} | 24 ++-
.../org/apache/baremaps/data/type/DataType.java | 1 +
.../type/{RowDataType.java => DataTypeImpl.java} | 42 ++---
.../baremaps/data/type/DataTypeProvider.java | 39 ++--
.../baremaps/flatgeobuf/FlatGeoBufDataTable.java | 41 +++--
.../flatgeobuf/FlatGeoBufTypeConversion.java | 23 ++-
.../baremaps/geopackage/GeoPackageDataTable.java | 16 +-
.../baremaps/geoparquet/GeoParquetDataTable.java | 8 +
.../baremaps/geoparquet/GeoParquetReader.java | 8 +-
.../baremaps/geoparquet/GeoParquetSpliterator.java | 1 +
.../geoparquet/GeoParquetTypeConversion.java | 20 +--
baremaps-openstreetmap/pom.xml | 5 +
.../openstreetmap/OpenStreetMapDataTable.java | 24 ++-
...> EntityDataColumnTypeGeometryBuilderTest.java} | 2 +-
.../baremaps/postgres/store/PostgresDataStore.java | 4 +-
.../baremaps/postgres/store/PostgresDataTable.java | 5 +
.../postgres/store/PostgresTypeConversion.java | 74 ++++----
.../apache/baremaps/postgres/MockDataTable.java | 17 +-
.../org/apache/baremaps/rpsl/RpslDataTable.java | 35 ++--
.../baremaps/shapefile/ShapefileByteReader.java | 34 ++--
.../baremaps/shapefile/ShapefileDataTable.java | 24 ++-
.../apache/baremaps/shapefile/ShapefileReader.java | 1 -
baremaps-store/pom.xml | 10 ++
.../java/org/apache/baremaps/store/DataColumn.java | 14 +-
.../org/apache/baremaps/store/DataColumnFixed.java | 3 +-
.../apache/baremaps/store/DataColumnNested.java | 4 +-
.../java/org/apache/baremaps/store/DataSchema.java | 73 +++++++-
.../org/apache/baremaps/store/DataSchemaImpl.java | 4 +-
.../java/org/apache/baremaps/store/DataTable.java | 2 +-
.../org/apache/baremaps/store/DataTableMapper.java | 4 +
pom.xml | 1 +
50 files changed, 917 insertions(+), 601 deletions(-)
diff --git a/baremaps-calcite/pom.xml b/baremaps-calcite/pom.xml
new file mode 100644
index 000000000..3653d0f69
--- /dev/null
+++ b/baremaps-calcite/pom.xml
@@ -0,0 +1,49 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+ <modelVersion>4.0.0</modelVersion>
+ <parent>
+ <groupId>org.apache.baremaps</groupId>
+ <artifactId>baremaps</artifactId>
+ <version>0.8.2-SNAPSHOT</version>
+ </parent>
+
+ <artifactId>baremaps-calcite</artifactId>
+
+ <properties>
+ <maven.compiler.source>21</maven.compiler.source>
+ <maven.compiler.target>21</maven.compiler.target>
+ <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
+ </properties>
+
+ <dependencies>
+ <dependency>
+ <groupId>org.apache.baremaps</groupId>
+ <artifactId>baremaps-csv</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.baremaps</groupId>
+ <artifactId>baremaps-data</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.calcite</groupId>
+ <artifactId>calcite-core</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.calcite</groupId>
+ <artifactId>calcite-server</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.commons</groupId>
+ <artifactId>commons-lang3</artifactId>
+ <version>3.16.0</version>
+ <scope>compile</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.immutables</groupId>
+ <artifactId>value</artifactId>
+ <version>2.10.1</version>
+ <scope>provided</scope>
+ </dependency>
+ </dependencies>
+
+</project>
diff --git a/baremaps-calcite/src/main/java/org/apache/baremaps/calcite/BaremapsTable.java b/baremaps-calcite/src/main/java/org/apache/baremaps/calcite/BaremapsTable.java
new file mode 100644
index 000000000..1d7caf7a4
--- /dev/null
+++ b/baremaps-calcite/src/main/java/org/apache/baremaps/calcite/BaremapsTable.java
@@ -0,0 +1,113 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to you under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.baremaps.calcite;
+
+import java.util.EnumMap;
+import org.apache.baremaps.store.DataColumn;
+import org.apache.baremaps.store.DataColumn.ColumnType;
+import org.apache.baremaps.store.DataTable;
+import org.apache.calcite.DataContext;
+import org.apache.calcite.jdbc.JavaTypeFactoryImpl;
+import org.apache.calcite.linq4j.Enumerable;
+import org.apache.calcite.linq4j.Linq4j;
+import org.apache.calcite.rel.type.RelDataType;
+import org.apache.calcite.rel.type.RelDataTypeFactory;
+import org.apache.calcite.rel.type.RelProtoDataType;
+import org.apache.calcite.schema.ScannableTable;
+import org.apache.calcite.schema.impl.AbstractTable;
+import org.apache.calcite.sql.type.SqlTypeName;
+import org.locationtech.jts.geom.*;
+
+public class BaremapsTable extends AbstractTable implements ScannableTable {
+
+ private static final EnumMap<ColumnType, RelDataType> types = new EnumMap<>(ColumnType.class);
+
+ static {
+ types.put(ColumnType.BYTE, new JavaTypeFactoryImpl()
+ .createSqlType(SqlTypeName.TINYINT));
+ types.put(ColumnType.BOOLEAN, new JavaTypeFactoryImpl()
+ .createSqlType(SqlTypeName.BOOLEAN));
+ types.put(ColumnType.SHORT, new JavaTypeFactoryImpl()
+ .createSqlType(SqlTypeName.SMALLINT));
+ types.put(ColumnType.INTEGER, new JavaTypeFactoryImpl()
+ .createSqlType(SqlTypeName.INTEGER));
+ types.put(ColumnType.LONG, new JavaTypeFactoryImpl()
+ .createSqlType(SqlTypeName.BIGINT));
+ types.put(ColumnType.FLOAT, new JavaTypeFactoryImpl()
+ .createSqlType(SqlTypeName.FLOAT));
+ types.put(ColumnType.DOUBLE, new JavaTypeFactoryImpl()
+ .createSqlType(SqlTypeName.DOUBLE));
+ types.put(ColumnType.STRING, new JavaTypeFactoryImpl()
+ .createSqlType(SqlTypeName.VARCHAR));
+ types.put(ColumnType.GEOMETRY, new JavaTypeFactoryImpl()
+ .createJavaType(Geometry.class));
+ types.put(ColumnType.POINT, new JavaTypeFactoryImpl()
+ .createJavaType(Point.class));
+ types.put(ColumnType.LINESTRING, new JavaTypeFactoryImpl()
+ .createJavaType(LineString.class));
+ types.put(ColumnType.POLYGON, new JavaTypeFactoryImpl()
+ .createJavaType(Polygon.class));
+ types.put(ColumnType.MULTIPOINT, new JavaTypeFactoryImpl()
+ .createJavaType(MultiPoint.class));
+ types.put(ColumnType.MULTILINESTRING, new JavaTypeFactoryImpl()
+ .createJavaType(MultiLineString.class));
+ types.put(ColumnType.MULTIPOLYGON, new JavaTypeFactoryImpl()
+ .createJavaType(MultiPolygon.class));
+ types.put(ColumnType.GEOMETRYCOLLECTION, new JavaTypeFactoryImpl()
+ .createJavaType(GeometryCollection.class));
+ }
+
+ private final DataTable table;
+
+ private final RelProtoDataType protoRowType;
+
+ private RelDataType rowType;
+
+ public BaremapsTable(DataTable table) {
+ this.table = table;
+ this.protoRowType = null;
+ }
+
+ public BaremapsTable(DataTable table, RelProtoDataType protoRowType) {
+ this.table = table;
+ this.protoRowType = protoRowType;
+ }
+
+ @Override
+ public RelDataType getRowType(final RelDataTypeFactory typeFactory) {
+ if (rowType == null) {
+ rowType = createRowType(typeFactory);
+ }
+ return rowType;
+ }
+
+ private RelDataType createRowType(RelDataTypeFactory typeFactory) {
+ var rowTypeBuilder = new RelDataTypeFactory.Builder(typeFactory);
+ for (DataColumn column : table.schema().columns()) {
+ rowTypeBuilder.add(column.name(), types.get(column.type()));
+ }
+ return rowTypeBuilder.build();
+ }
+
+ @Override
+ public Enumerable<Object[]> scan(final DataContext root) {
+ return Linq4j.asEnumerable(() -> table.stream()
+ .map(row -> row.values().toArray())
+ .iterator());
+ }
+}
diff --git a/baremaps-calcite/src/main/java/org/apache/baremaps/calcite/BaremapsTableFactory.java b/baremaps-calcite/src/main/java/org/apache/baremaps/calcite/BaremapsTableFactory.java
new file mode 100644
index 000000000..bd0c99b31
--- /dev/null
+++ b/baremaps-calcite/src/main/java/org/apache/baremaps/calcite/BaremapsTableFactory.java
@@ -0,0 +1,53 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to you under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.baremaps.calcite;
+
+import java.io.File;
+import java.io.IOException;
+import java.util.Map;
+import org.apache.baremaps.csv.CsvDataTable;
+import org.apache.calcite.rel.type.RelDataType;
+import org.apache.calcite.rel.type.RelDataTypeImpl;
+import org.apache.calcite.rel.type.RelProtoDataType;
+import org.apache.calcite.schema.SchemaPlus;
+import org.apache.calcite.schema.TableFactory;
+
+public class BaremapsTableFactory implements TableFactory<BaremapsTable> {
+
+ public BaremapsTableFactory() {
+
+ }
+
+ @Override
+ public BaremapsTable create(SchemaPlus schema, String name,
+ Map<String, Object> operand, RelDataType rowType) {
+ final RelProtoDataType protoRowType =
+ rowType != null ? RelDataTypeImpl.proto(rowType) : null;
+ String file = (String) operand.get("file");
+ if (file != null && file.endsWith(".csv")) {
+ try {
+ return new BaremapsTable(
+ new CsvDataTable(new File(file), true),
+ protoRowType);
+ } catch (IOException e) {
+ throw new RuntimeException(e);
+ }
+ }
+ return new BaremapsTable(null, protoRowType);
+ }
+}
diff --git a/baremaps-data/src/test/java/org/apache/baremaps/data/calcite/CalciteTest.java b/baremaps-calcite/src/test/java/com/apache/baremaps/calcite/CalciteTest.java
similarity index 72%
rename from baremaps-data/src/test/java/org/apache/baremaps/data/calcite/CalciteTest.java
rename to baremaps-calcite/src/test/java/com/apache/baremaps/calcite/CalciteTest.java
index fd645d163..870263c8f 100644
--- a/baremaps-data/src/test/java/org/apache/baremaps/data/calcite/CalciteTest.java
+++ b/baremaps-calcite/src/test/java/com/apache/baremaps/calcite/CalciteTest.java
@@ -15,17 +15,20 @@
* limitations under the License.
*/
-package org.apache.baremaps.data.calcite;
+package com.apache.baremaps.calcite;
+import java.io.File;
+import java.io.PrintWriter;
import java.sql.*;
import java.util.*;
+import org.apache.baremaps.calcite.BaremapsTable;
import org.apache.baremaps.data.collection.AppendOnlyLog;
import org.apache.baremaps.data.collection.IndexedDataList;
-import org.apache.baremaps.data.store.BaremapsDataTable;
-import org.apache.baremaps.data.type.RowDataType;
+import org.apache.baremaps.data.store.DataTableImpl;
+import org.apache.baremaps.data.type.DataTypeImpl;
import org.apache.baremaps.store.*;
import org.apache.baremaps.store.DataColumn.Cardinality;
-import org.apache.baremaps.store.DataColumn.Type;
+import org.apache.baremaps.store.DataColumn.ColumnType;
import org.apache.calcite.DataContext;
import org.apache.calcite.DataContexts;
import org.apache.calcite.interpreter.Interpreter;
@@ -43,6 +46,7 @@ import org.apache.calcite.sql.type.SqlTypeName;
import org.apache.calcite.tools.FrameworkConfig;
import org.apache.calcite.tools.Frameworks;
import org.apache.calcite.tools.Planner;
+import org.junit.jupiter.api.Disabled;
import org.junit.jupiter.api.Test;
import org.locationtech.jts.geom.Coordinate;
import org.locationtech.jts.geom.GeometryFactory;
@@ -50,6 +54,7 @@ import org.locationtech.jts.geom.GeometryFactory;
public class CalciteTest {
@Test
+ @Disabled
void sql() throws SQLException {
GeometryFactory geometryFactory = new GeometryFactory();
@@ -66,36 +71,36 @@ public class CalciteTest {
// Create and add 'city' table
DataSchema cityRowType = new DataSchemaImpl("city", List.of(
- new DataColumnFixed("id", Cardinality.OPTIONAL, Type.INTEGER),
- new DataColumnFixed("name", Cardinality.OPTIONAL, Type.STRING),
- new DataColumnFixed("geometry", Cardinality.OPTIONAL, Type.GEOMETRY)));
+ new DataColumnFixed("id", Cardinality.OPTIONAL, ColumnType.INTEGER),
+ new DataColumnFixed("name", Cardinality.OPTIONAL, ColumnType.STRING),
+ new DataColumnFixed("geometry", Cardinality.OPTIONAL, ColumnType.GEOMETRY)));
- DataTable cityDataTable = new BaremapsDataTable(
+ DataTable cityDataTable = new DataTableImpl(
cityRowType,
- new IndexedDataList<>(new AppendOnlyLog<>(new RowDataType(cityRowType))));
+ new IndexedDataList<>(new AppendOnlyLog<>(new DataTypeImpl(cityRowType))));
cityDataTable.add(new DataRowImpl(cityDataTable.schema(),
List.of(1, "Paris", geometryFactory.createPoint(new Coordinate(2.3522, 48.8566)))));
cityDataTable.add(new DataRowImpl(cityDataTable.schema(),
List.of(2, "New York", geometryFactory.createPoint(new Coordinate(-74.0060, 40.7128)))));
- SqlDataTable citySqlDataTable = new SqlDataTable(cityDataTable);
- rootSchema.add("city", citySqlDataTable);
+ BaremapsTable cityBaremapsTable = new BaremapsTable(cityDataTable);
+ rootSchema.add("city", cityBaremapsTable);
// Create and add 'population' table
DataSchema populationRowType = new DataSchemaImpl("population", List.of(
- new DataColumnFixed("city_id", Cardinality.OPTIONAL, Type.INTEGER),
- new DataColumnFixed("population", Cardinality.OPTIONAL, Type.INTEGER)));
+ new DataColumnFixed("city_id", Cardinality.OPTIONAL, ColumnType.INTEGER),
+ new DataColumnFixed("population", Cardinality.OPTIONAL, ColumnType.INTEGER)));
- DataTable populationDataTable = new BaremapsDataTable(
+ DataTable populationDataTable = new DataTableImpl(
populationRowType,
- new IndexedDataList<>(new AppendOnlyLog<>(new RowDataType(populationRowType))));
+ new IndexedDataList<>(new AppendOnlyLog<>(new DataTypeImpl(populationRowType))));
populationDataTable.add(new DataRowImpl(populationDataTable.schema(), List.of(1, 2_161_000)));
populationDataTable.add(new DataRowImpl(populationDataTable.schema(), List.of(2, 8_336_000)));
- SqlDataTable populationSqlDataTable = new SqlDataTable(populationDataTable);
- rootSchema.add("population", populationSqlDataTable);
+ BaremapsTable populationBaremapsTable = new BaremapsTable(populationDataTable);
+ rootSchema.add("population", populationBaremapsTable);
// Create view 'city_population'
String mvSql = "SELECT c.id, c.name, c.geometry, p.population " +
@@ -181,6 +186,7 @@ public class CalciteTest {
}
@Test
+ @Disabled
void list() throws Exception {
// Initialize your Java lists
List<Integer> listA = List.of(1, 2, 3, 4, 5);
@@ -192,13 +198,13 @@ public class CalciteTest {
// Create and add 'city' table
DataSchema cityRowType = new DataSchemaImpl("city", List.of(
- new DataColumnFixed("id", Cardinality.OPTIONAL, Type.INTEGER),
- new DataColumnFixed("name", Cardinality.OPTIONAL, Type.STRING),
- new DataColumnFixed("geometry", Cardinality.OPTIONAL, Type.GEOMETRY)));
+ new DataColumnFixed("id", Cardinality.OPTIONAL, ColumnType.INTEGER),
+ new DataColumnFixed("name", Cardinality.OPTIONAL, ColumnType.STRING),
+ new DataColumnFixed("geometry", Cardinality.OPTIONAL, ColumnType.GEOMETRY)));
- DataTable cityDataTable = new BaremapsDataTable(
+ DataTable cityDataTable = new DataTableImpl(
cityRowType,
- new IndexedDataList<>(new AppendOnlyLog<>(new RowDataType(cityRowType))));
+ new IndexedDataList<>(new AppendOnlyLog<>(new DataTypeImpl(cityRowType))));
GeometryFactory geometryFactory = new GeometryFactory();
cityDataTable.add(new DataRowImpl(cityDataTable.schema(),
@@ -206,8 +212,8 @@ public class CalciteTest {
cityDataTable.add(new DataRowImpl(cityDataTable.schema(),
List.of(2, "New York", geometryFactory.createPoint(new Coordinate(-74.0060, 40.7128)))));
- SqlDataTable citySqlDataTable = new SqlDataTable(cityDataTable);
- rootSchema.add("CITY", citySqlDataTable);
+ BaremapsTable cityBaremapsTable = new BaremapsTable(cityDataTable);
+ rootSchema.add("CITY", cityBaremapsTable);
// Configure the framework
FrameworkConfig config = Frameworks.newConfigBuilder()
@@ -232,7 +238,7 @@ public class CalciteTest {
Interpreter interpreter = new Interpreter(DataContexts.EMPTY, rel);
// Create an interpreter to execute the RelNode
- for (Object[] row : interpreter.asEnumerable()) {
+ for (Object[] row : interpreter) {
listB.add((Integer) row[0]);
}
@@ -241,4 +247,50 @@ public class CalciteTest {
System.out.println("List B (after SQL): " + listB);
}
+ @Test
+ public void testCsvStream() throws Exception {
+ File file = File.createTempFile("test", ".csv");
+ String csv = """
+ ID,NAME,GEOM
+ 1,Paris,POINT(2.3522 48.8566)
+ 2,New York,POINT(-74.0060 40.7128)
+ """;
+ try (PrintWriter writer = new PrintWriter(file)) {
+ writer.write(csv);
+ }
+
+ String model = """
+ {
+ version: '1.0',
+ defaultSchema: 'TEST',
+ schemas: [
+ {
+ name: 'TEST',
+ tables: [
+ {
+ name: 'TEST',
+ type: 'custom',
+ factory: 'org.apache.baremaps.calcite.BaremapsTableFactory',
+ operand: {
+ file: '%s'
+ }
+ }
+ ]
+ }
+ ]
+ }
+ """.formatted(file.getAbsolutePath());
+
+ try (Connection connection =
+ DriverManager.getConnection("jdbc:calcite:model=inline:" + model)) {
+
+ ResultSet resultSet = connection.createStatement().executeQuery("SELECT * FROM TEST.TEST");
+ while (resultSet.next()) {
+ System.out.println(resultSet.getString("ID") + " " + resultSet.getString("GEOM"));
+ }
+ } finally {
+ file.delete();
+ }
+ }
+
}
diff --git a/baremaps-core/src/main/java/org/apache/baremaps/geocoder/DataRowMapper.java b/baremaps-core/src/main/java/org/apache/baremaps/geocoder/DataRowMapper.java
index df6227a98..2398d1d4a 100644
--- a/baremaps-core/src/main/java/org/apache/baremaps/geocoder/DataRowMapper.java
+++ b/baremaps-core/src/main/java/org/apache/baremaps/geocoder/DataRowMapper.java
@@ -27,6 +27,7 @@ import java.util.List;
import java.util.Map;
import java.util.function.Function;
import org.apache.baremaps.store.DataColumn;
+import org.apache.baremaps.store.DataColumn.ColumnType;
import org.apache.baremaps.store.DataRow;
import org.apache.baremaps.store.DataSchema;
import org.apache.lucene.document.*;
@@ -58,7 +59,7 @@ public class DataRowMapper implements Function<DataRow, Document> {
@SuppressWarnings("squid:S6541")
private void applyValue(DataColumn column, Document doc, Object value) {
String columnName = column.name();
- DataColumn.Type type = column.type();
+ ColumnType type = column.type();
try {
switch (type) {
case BINARY:
diff --git a/baremaps-core/src/main/java/org/apache/baremaps/tasks/ImportGeoPackage.java b/baremaps-core/src/main/java/org/apache/baremaps/tasks/ImportGeoPackage.java
index 93932f02e..47728e60d 100644
--- a/baremaps-core/src/main/java/org/apache/baremaps/tasks/ImportGeoPackage.java
+++ b/baremaps-core/src/main/java/org/apache/baremaps/tasks/ImportGeoPackage.java
@@ -26,7 +26,6 @@ import org.apache.baremaps.store.DataTableGeometryMapper;
import org.apache.baremaps.store.DataTableMapper;
import org.apache.baremaps.workflow.Task;
import org.apache.baremaps.workflow.WorkflowContext;
-import org.apache.baremaps.workflow.WorkflowException;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -71,20 +70,17 @@ public class ImportGeoPackage implements Task {
@Override
public void execute(WorkflowContext context) throws Exception {
var path = file.toAbsolutePath();
- try (var geoPackageDataStore = new GeoPackageDataStore(path)) {
- var dataSource = context.getDataSource(database);
- var postgresDataStore = new PostgresDataStore(dataSource);
- for (var name : geoPackageDataStore.list()) {
- var geoPackageTable = geoPackageDataStore.get(name);
- var projectionTransformer = new ProjectionTransformer(fileSrid, databaseSrid);
- var rowTransformer =
- new DataTableGeometryMapper(geoPackageTable, projectionTransformer);
- var transformedDataTable =
- new DataTableMapper(geoPackageDataStore.get(name), rowTransformer);
- postgresDataStore.add(transformedDataTable);
- }
- } catch (Exception e) {
- throw new WorkflowException(e);
+ var geoPackageDataStore = new GeoPackageDataStore(path);
+ var dataSource = context.getDataSource(database);
+ var postgresDataStore = new PostgresDataStore(dataSource);
+ for (var name : geoPackageDataStore.list()) {
+ var geoPackageTable = geoPackageDataStore.get(name);
+ var projectionTransformer = new ProjectionTransformer(fileSrid, databaseSrid);
+ var rowTransformer =
+ new DataTableGeometryMapper(geoPackageTable, projectionTransformer);
+ var transformedDataTable =
+ new DataTableMapper(geoPackageDataStore.get(name), rowTransformer);
+ postgresDataStore.add(transformedDataTable);
}
}
diff --git a/baremaps-core/src/test/java/org/apache/baremaps/calcite/CalciteTest.java b/baremaps-core/src/test/java/org/apache/baremaps/calcite/CalciteTest.java
index efc13eb4d..a52cfe99c 100644
--- a/baremaps-core/src/test/java/org/apache/baremaps/calcite/CalciteTest.java
+++ b/baremaps-core/src/test/java/org/apache/baremaps/calcite/CalciteTest.java
@@ -18,104 +18,104 @@
package org.apache.baremaps.calcite;
-import com.google.common.collect.ImmutableList;
-import java.sql.*;
-import java.util.List;
-import java.util.Properties;
-import org.apache.baremaps.data.calcite.SqlDataTable;
-import org.apache.baremaps.data.collection.AppendOnlyLog;
-import org.apache.baremaps.data.collection.IndexedDataList;
-import org.apache.baremaps.data.store.BaremapsDataTable;
-import org.apache.baremaps.data.type.RowDataType;
-import org.apache.baremaps.maplibre.vectortile.VectorTileFunctions;
-import org.apache.baremaps.store.*;
-import org.apache.baremaps.store.DataColumn.Cardinality;
-import org.apache.baremaps.store.DataColumn.Type;
-import org.apache.calcite.jdbc.CalciteConnection;
-import org.apache.calcite.model.ModelHandler;
-import org.apache.calcite.runtime.AccumOperation;
-import org.apache.calcite.runtime.CollectOperation;
-import org.apache.calcite.runtime.SpatialTypeFunctions;
-import org.apache.calcite.runtime.UnionOperation;
-import org.apache.calcite.schema.SchemaPlus;
-import org.apache.calcite.schema.impl.AggregateFunctionImpl;
-import org.apache.calcite.sql.fun.SqlSpatialTypeFunctions;
-import org.junit.jupiter.api.Assertions;
-import org.junit.jupiter.api.Test;
-import org.locationtech.jts.geom.*;
+// import com.google.common.collect.ImmutableList;
+// import java.sql.*;
+// import java.util.List;
+// import java.util.Properties;
+// import org.apache.baremaps.data.calcite.BaremapsTable;
+// import org.apache.baremaps.data.collection.AppendOnlyLog;
+// import org.apache.baremaps.data.collection.IndexedDataList;
+// import org.apache.baremaps.data.store.DataTableImpl;
+// import org.apache.baremaps.data.type.DataTypeImpl;
+// import org.apache.baremaps.maplibre.vectortile.VectorTileFunctions;
+// import org.apache.baremaps.store.*;
+// import org.apache.baremaps.store.DataColumn.Cardinality;
+// import org.apache.baremaps.store.DataColumn.Type;
+// import org.apache.calcite.jdbc.CalciteConnection;
+// import org.apache.calcite.model.ModelHandler;
+// import org.apache.calcite.runtime.AccumOperation;
+// import org.apache.calcite.runtime.CollectOperation;
+// import org.apache.calcite.runtime.SpatialTypeFunctions;
+// import org.apache.calcite.runtime.UnionOperation;
+// import org.apache.calcite.schema.SchemaPlus;
+// import org.apache.calcite.schema.impl.AggregateFunctionImpl;
+// import org.apache.calcite.sql.fun.SqlSpatialTypeFunctions;
+// import org.junit.jupiter.api.Assertions;
+// import org.junit.jupiter.api.Test;
+// import org.locationtech.jts.geom.*;
class CalciteTest {
-
- @Test
- void test() throws SQLException {
- GeometryFactory geometryFactory = new GeometryFactory();
-
- Properties info = new Properties();
- info.setProperty("lex", "MYSQL");
-
- try (Connection connection = DriverManager.getConnection("jdbc:calcite:", info)) {
- CalciteConnection calciteConnection = connection.unwrap(CalciteConnection.class);
- SchemaPlus rootSchema = calciteConnection.getRootSchema();
-
- // Add the spatial functions to the root schema
- ImmutableList<String> emptyPath = ImmutableList.of();
- ModelHandler.addFunctions(rootSchema, null, emptyPath,
- SpatialTypeFunctions.class.getName(), "*", true);
- ModelHandler.addFunctions(rootSchema, null, emptyPath,
- SqlSpatialTypeFunctions.class.getName(), "*", true);
-
- rootSchema.add("ST_UNION", AggregateFunctionImpl.create(UnionOperation.class));
- rootSchema.add("ST_ACCUM", AggregateFunctionImpl.create(AccumOperation.class));
- rootSchema.add("ST_COLLECT", AggregateFunctionImpl.create(CollectOperation.class));
-
- ModelHandler.addFunctions(rootSchema, "ST_AsMVTGeom", emptyPath,
- VectorTileFunctions.class.getName(), "asVectorTileGeom", true);
- ModelHandler.addFunctions(rootSchema, "ST_AsMVT", emptyPath,
- VectorTileFunctions.class.getName(), "asVectorTile", true);
-
- // Create the city table
- DataSchema cityRowType = new DataSchemaImpl("city", List.of(
- new DataColumnFixed("id", Cardinality.OPTIONAL, Type.INTEGER),
- new DataColumnFixed("name", Cardinality.OPTIONAL, Type.STRING),
- new DataColumnFixed("geometry", Cardinality.OPTIONAL, Type.GEOMETRY)));
- DataTable cityDataTable = new BaremapsDataTable(
- cityRowType,
- new IndexedDataList<>(new AppendOnlyLog<>(new RowDataType(cityRowType))));
- cityDataTable.add(new DataRowImpl(cityDataTable.schema(),
- List.of(1, "Paris", geometryFactory.createPoint(new Coordinate(2.3522, 48.8566)))));
- cityDataTable.add(new DataRowImpl(cityDataTable.schema(),
- List.of(2, "New York", geometryFactory.createPoint(new Coordinate(-74.0060, 40.7128)))));
- SqlDataTable citySqlDataTable = new SqlDataTable(cityDataTable);
- rootSchema.add("city", citySqlDataTable);
-
- // Create the population table
- DataSchema populationRowType = new DataSchemaImpl("population", List.of(
- new DataColumnFixed("city_id", Cardinality.OPTIONAL, Type.INTEGER),
- new DataColumnFixed("population", Cardinality.OPTIONAL, Type.INTEGER)));
- DataTable populationDataTable = new BaremapsDataTable(
- populationRowType,
- new IndexedDataList<>(new AppendOnlyLog<>(new RowDataType(populationRowType))));
- populationDataTable
- .add(new DataRowImpl(populationDataTable.schema(), List.of(1, 2_161_000)));
- populationDataTable
- .add(new DataRowImpl(populationDataTable.schema(), List.of(2, 8_336_000)));
- SqlDataTable populationSqlDataTable = new SqlDataTable(populationDataTable);
- rootSchema.add("population", populationSqlDataTable);
-
- // Query the database
- String sql = """
- SELECT ST_AsText(ST_AsMVTGeom(
- ST_GeomFromText('POLYGON ((0 0, 10 1, 10 10, 1 10, 0 0))'),
- ST_MakeEnvelope(0, 0, 4096, 4096),
- 4096, 0, true))
- """;
-
- try (Statement statement = connection.createStatement();
- ResultSet resultSet = statement.executeQuery(sql)) {
- Assertions.assertTrue(resultSet.next());
- Assertions.assertEquals("POLYGON ((0 4096, 10 4095, 10 4086, 1 4086, 0 4096))",
- resultSet.getString(1));
- }
- }
- }
+ //
+ // @Test
+ // void test() throws SQLException {
+ // GeometryFactory geometryFactory = new GeometryFactory();
+ //
+ // Properties info = new Properties();
+ // info.setProperty("lex", "MYSQL");
+ //
+ // try (Connection connection = DriverManager.getConnection("jdbc:calcite:", info)) {
+ // CalciteConnection calciteConnection = connection.unwrap(CalciteConnection.class);
+ // SchemaPlus rootSchema = calciteConnection.getRootSchema();
+ //
+ // // Add the spatial functions to the root schema
+ // ImmutableList<String> emptyPath = ImmutableList.of();
+ // ModelHandler.addFunctions(rootSchema, null, emptyPath,
+ // SpatialTypeFunctions.class.getName(), "*", true);
+ // ModelHandler.addFunctions(rootSchema, null, emptyPath,
+ // SqlSpatialTypeFunctions.class.getName(), "*", true);
+ //
+ // rootSchema.add("ST_UNION", AggregateFunctionImpl.create(UnionOperation.class));
+ // rootSchema.add("ST_ACCUM", AggregateFunctionImpl.create(AccumOperation.class));
+ // rootSchema.add("ST_COLLECT", AggregateFunctionImpl.create(CollectOperation.class));
+ //
+ // ModelHandler.addFunctions(rootSchema, "ST_AsMVTGeom", emptyPath,
+ // VectorTileFunctions.class.getName(), "asVectorTileGeom", true);
+ // ModelHandler.addFunctions(rootSchema, "ST_AsMVT", emptyPath,
+ // VectorTileFunctions.class.getName(), "asVectorTile", true);
+ //
+ // // Create the city table
+ // DataSchema cityRowType = new DataSchemaImpl("city", List.of(
+ // new DataColumnFixed("id", Cardinality.OPTIONAL, Type.INTEGER),
+ // new DataColumnFixed("name", Cardinality.OPTIONAL, Type.STRING),
+ // new DataColumnFixed("geometry", Cardinality.OPTIONAL, Type.GEOMETRY)));
+ // DataTable cityDataTable = new DataTableImpl(
+ // cityRowType,
+ // new IndexedDataList<>(new AppendOnlyLog<>(new DataTypeImpl(cityRowType))));
+ // cityDataTable.add(new DataRowImpl(cityDataTable.schema(),
+ // List.of(1, "Paris", geometryFactory.createPoint(new Coordinate(2.3522, 48.8566)))));
+ // cityDataTable.add(new DataRowImpl(cityDataTable.schema(),
+ // List.of(2, "New York", geometryFactory.createPoint(new Coordinate(-74.0060, 40.7128)))));
+ // BaremapsTable cityBaremapsTable = new BaremapsTable(cityDataTable);
+ // rootSchema.add("city", cityBaremapsTable);
+ //
+ // // Create the population table
+ // DataSchema populationRowType = new DataSchemaImpl("population", List.of(
+ // new DataColumnFixed("city_id", Cardinality.OPTIONAL, Type.INTEGER),
+ // new DataColumnFixed("population", Cardinality.OPTIONAL, Type.INTEGER)));
+ // DataTable populationDataTable = new DataTableImpl(
+ // populationRowType,
+ // new IndexedDataList<>(new AppendOnlyLog<>(new DataTypeImpl(populationRowType))));
+ // populationDataTable
+ // .add(new DataRowImpl(populationDataTable.schema(), List.of(1, 2_161_000)));
+ // populationDataTable
+ // .add(new DataRowImpl(populationDataTable.schema(), List.of(2, 8_336_000)));
+ // BaremapsTable populationBaremapsTable = new BaremapsTable(populationDataTable);
+ // rootSchema.add("population", populationBaremapsTable);
+ //
+ // // Query the database
+ // String sql = """
+ // SELECT ST_AsText(ST_AsMVTGeom(
+ // ST_GeomFromText('POLYGON ((0 0, 10 1, 10 10, 1 10, 0 0))'),
+ // ST_MakeEnvelope(0, 0, 4096, 4096),
+ // 4096, 0, true))
+ // """;
+ //
+ // try (Statement statement = connection.createStatement();
+ // ResultSet resultSet = statement.executeQuery(sql)) {
+ // Assertions.assertTrue(resultSet.next());
+ // Assertions.assertEquals("POLYGON ((0 4096, 10 4095, 10 4086, 1 4086, 0 4096))",
+ // resultSet.getString(1));
+ // }
+ // }
+ // }
}
diff --git a/baremaps-csv/src/main/java/org/apache/baremaps/csv/CsvDataStore.java b/baremaps-csv/src/main/java/org/apache/baremaps/csv/CsvDataStore.java
index 1dfc85298..74abb32e3 100644
--- a/baremaps-csv/src/main/java/org/apache/baremaps/csv/CsvDataStore.java
+++ b/baremaps-csv/src/main/java/org/apache/baremaps/csv/CsvDataStore.java
@@ -49,7 +49,7 @@ public class CsvDataStore implements DataStore {
char separator) throws IOException {
this.tableName = tableName;
this.schema = schema;
- this.dataTable = new CsvDataTable(schema, csvFile, hasHeader, separator);
+ this.dataTable = new CsvDataTable(csvFile, hasHeader);
}
/**
diff --git a/baremaps-csv/src/main/java/org/apache/baremaps/csv/CsvDataTable.java b/baremaps-csv/src/main/java/org/apache/baremaps/csv/CsvDataTable.java
index 845e54e93..360debfae 100644
--- a/baremaps-csv/src/main/java/org/apache/baremaps/csv/CsvDataTable.java
+++ b/baremaps-csv/src/main/java/org/apache/baremaps/csv/CsvDataTable.java
@@ -27,6 +27,8 @@ import java.util.*;
import java.util.stream.Stream;
import java.util.stream.StreamSupport;
import org.apache.baremaps.store.*;
+import org.apache.baremaps.store.DataColumn.Cardinality;
+import org.apache.baremaps.store.DataColumn.ColumnType;
import org.locationtech.jts.io.WKTReader;
/**
@@ -34,43 +36,57 @@ import org.locationtech.jts.io.WKTReader;
*/
public class CsvDataTable implements DataTable {
- private final DataSchema schema;
private final File csvFile;
private final CsvSchema csvSchema;
+ private final DataSchema dataSchema;
+
private final long size;
+ private JsonParser parser;
/**
* Constructs a CsvDataTable with the specified schema, CSV file, header presence, and separator.
*
- * @param schema the data schema defining the structure
* @param csvFile the CSV file to read data from
* @param hasHeader whether the CSV file includes a header row
- * @param separator the character used to separate columns in the CSV file
* @throws IOException if an I/O error occurs
*/
- public CsvDataTable(DataSchema schema, File csvFile, boolean hasHeader, char separator)
- throws IOException {
- this.schema = schema;
+ public CsvDataTable(File csvFile, boolean hasHeader) throws IOException {
this.csvFile = csvFile;
- this.csvSchema = buildCsvSchema(schema, hasHeader, separator);
+ this.csvSchema = inferCsvSchema(csvFile);
+ this.dataSchema = createDataSchema(csvFile.getName(), csvSchema);
this.size = calculateSize();
}
- /**
- * Builds the CsvSchema for Jackson based on the provided DataSchema, header presence, and
- * separator.
- *
- * @param dataSchema the data schema
- * @param hasHeader whether the CSV file includes a header row
- * @param separator the character used to separate columns
- * @return the CsvSchema for Jackson
- */
- private CsvSchema buildCsvSchema(DataSchema dataSchema, boolean hasHeader, char separator) {
- CsvSchema.Builder builder = CsvSchema.builder();
- for (DataColumn column : dataSchema.columns()) {
- builder.addColumn(column.name());
+
+ private CsvSchema inferCsvSchema(File csvFile) {
+ CsvSchema csvSchema = CsvSchema.emptySchema().withUseHeader(true).withColumnSeparator(',');
+ try (var ignored = new CsvMapper().readerFor(Map.class)
+ .with(csvSchema)
+ .createParser(csvFile)) {
+ var test = ignored.readValueAsTree();
+ return csvSchema;
+ } catch (IOException e) {
+ throw new DataStoreException("Error reading CSV file", e);
+ }
+ }
+
+ private DataSchema createDataSchema(String name, CsvSchema csvSchema) {
+ List<DataColumn> columns = new ArrayList<>();
+ for (String columnName : csvSchema.getColumnNames()) {
+ switch (csvSchema.column(columnName).getType()) {
+ case STRING -> columns
+ .add(new DataColumnFixed(columnName, Cardinality.REQUIRED, ColumnType.STRING));
+ case NUMBER -> columns
+ .add(new DataColumnFixed(columnName, Cardinality.REQUIRED, ColumnType.DOUBLE));
+ case BOOLEAN -> columns
+ .add(new DataColumnFixed(columnName, Cardinality.REQUIRED, ColumnType.BOOLEAN));
+ case ARRAY -> columns
+ .add(new DataColumnFixed(columnName, Cardinality.REPEATED, ColumnType.STRING));
+ default -> throw new IllegalArgumentException(
+ "Unsupported column type: " + csvSchema.column(columnName).getType());
+ }
}
- return builder.setUseHeader(hasHeader).setColumnSeparator(separator).build();
+ return new DataSchemaImpl(name, columns);
}
/**
@@ -95,7 +111,7 @@ public class CsvDataTable implements DataTable {
@Override
public DataSchema schema() {
- return schema;
+ return dataSchema;
}
@Override
@@ -117,7 +133,7 @@ public class CsvDataTable implements DataTable {
public Iterator<DataRow> iterator() {
try {
CsvMapper csvMapper = new CsvMapper();
- JsonParser parser = csvMapper.readerFor(Map.class)
+ parser = csvMapper.readerFor(Map.class)
.with(csvSchema)
.createParser(csvFile);
@@ -134,10 +150,10 @@ public class CsvDataTable implements DataTable {
@Override
public DataRow next() {
Map<String, String> csvRow = csvIterator.next();
- DataRow dataRow = schema.createRow();
+ DataRow dataRow = dataSchema.createRow();
- for (int i = 0; i < schema.columns().size(); i++) {
- DataColumn column = schema.columns().get(i);
+ for (int i = 0; i < dataSchema.columns().size(); i++) {
+ DataColumn column = dataSchema.columns().get(i);
String columnName = column.name();
String value = csvRow.get(columnName);
@@ -165,7 +181,7 @@ public class CsvDataTable implements DataTable {
* @return the parsed value
*/
private Object parseValue(DataColumn column, String value) {
- DataColumn.Type type = column.type();
+ ColumnType type = column.type();
try {
if (value == null || value.isEmpty()) {
return null;
@@ -198,4 +214,11 @@ public class CsvDataTable implements DataTable {
public Stream<DataRow> stream() {
return StreamSupport.stream(spliterator(), false);
}
+
+ @Override
+ public void close() throws IOException {
+ if (parser != null) {
+ parser.close();
+ }
+ }
}
diff --git a/baremaps-csv/src/test/java/org/apache/baremaps/csv/CsvDataTableGeonamesTest.java b/baremaps-csv/src/test/java/org/apache/baremaps/csv/CsvDataTableGeonamesTest.java
index 8bceebbc2..489634505 100644
--- a/baremaps-csv/src/test/java/org/apache/baremaps/csv/CsvDataTableGeonamesTest.java
+++ b/baremaps-csv/src/test/java/org/apache/baremaps/csv/CsvDataTableGeonamesTest.java
@@ -21,10 +21,7 @@ import static org.apache.baremaps.testing.TestFiles.GEONAMES_CSV;
import static org.junit.jupiter.api.Assertions.*;
import java.io.IOException;
-import java.util.List;
import org.apache.baremaps.store.*;
-import org.apache.baremaps.store.DataColumn.Cardinality;
-import org.apache.baremaps.store.DataColumn.Type;
import org.junit.jupiter.api.*;
import org.locationtech.jts.geom.Point;
@@ -32,31 +29,7 @@ class CsvDataTableGeonamesTest {
@Test
void testGeonamesCsvDataTable() throws IOException {
- List<DataColumn> columns = List.of(
- new DataColumnFixed("id", Cardinality.REQUIRED, Type.INTEGER),
- new DataColumnFixed("name", Cardinality.OPTIONAL, Type.STRING),
- new DataColumnFixed("asciiname", Cardinality.OPTIONAL, Type.STRING),
- new DataColumnFixed("alternatenames", Cardinality.OPTIONAL, Type.STRING),
- new DataColumnFixed("latitude", Cardinality.OPTIONAL, Type.DOUBLE),
- new DataColumnFixed("longitude", Cardinality.OPTIONAL, Type.DOUBLE),
- new DataColumnFixed("feature_class", Cardinality.OPTIONAL, Type.STRING),
- new DataColumnFixed("feature_code", Cardinality.OPTIONAL, Type.STRING),
- new DataColumnFixed("country_code", Cardinality.OPTIONAL, Type.STRING),
- new DataColumnFixed("cc2", Cardinality.OPTIONAL, Type.STRING),
- new DataColumnFixed("admin1_code", Cardinality.OPTIONAL, Type.STRING),
- new DataColumnFixed("admin2_code", Cardinality.OPTIONAL, Type.STRING),
- new DataColumnFixed("admin3_code", Cardinality.OPTIONAL, Type.STRING),
- new DataColumnFixed("admin4_code", Cardinality.OPTIONAL, Type.STRING),
- new DataColumnFixed("population", Cardinality.OPTIONAL, Type.LONG),
- new DataColumnFixed("elevation", Cardinality.OPTIONAL, Type.INTEGER),
- new DataColumnFixed("dem", Cardinality.OPTIONAL, Type.INTEGER),
- new DataColumnFixed("timezone", Cardinality.OPTIONAL, Type.STRING),
- new DataColumnFixed("modification_date", Cardinality.OPTIONAL, Type.STRING));
- DataSchema schema = new DataSchemaImpl("geonames", columns);
-
- boolean hasHeader = false;
- char separator = '\t';
- DataTable dataTable = new CsvDataTable(schema, GEONAMES_CSV.toFile(), hasHeader, separator);
+ DataTable dataTable = new CsvDataTable(GEONAMES_CSV.toFile(), false);
assertEquals(5, dataTable.size(), "DataTable should have 5 rows.");
diff --git a/baremaps-csv/src/test/java/org/apache/baremaps/csv/CsvDataTableTest.java b/baremaps-csv/src/test/java/org/apache/baremaps/csv/CsvDataTableTest.java
index cb38a516d..444b37996 100644
--- a/baremaps-csv/src/test/java/org/apache/baremaps/csv/CsvDataTableTest.java
+++ b/baremaps-csv/src/test/java/org/apache/baremaps/csv/CsvDataTableTest.java
@@ -26,7 +26,7 @@ import java.nio.file.Path;
import java.util.List;
import org.apache.baremaps.store.*;
import org.apache.baremaps.store.DataColumn.Cardinality;
-import org.apache.baremaps.store.DataColumn.Type;
+import org.apache.baremaps.store.DataColumn.ColumnType;
import org.junit.jupiter.api.AfterEach;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
@@ -58,12 +58,7 @@ class CsvDataTableTest {
2,PointB,"POINT(2 2)"
""";
Files.writeString(tempCsvFile.toPath(), csvContent);
- List<DataColumn> columns = List.of(
- new DataColumnFixed("id", Cardinality.REQUIRED, Type.INTEGER),
- new DataColumnFixed("name", Cardinality.OPTIONAL, Type.STRING),
- new DataColumnFixed("geom", Cardinality.OPTIONAL, Type.GEOMETRY));
- DataSchema schema = new DataSchemaImpl("test_table", columns);
- DataTable dataTable = new CsvDataTable(schema, tempCsvFile, true, ',');
+ DataTable dataTable = new CsvDataTable(tempCsvFile, true);
assertEquals(2, dataTable.size());
int rowCount = 0;
for (DataRow row : dataTable) {
@@ -88,11 +83,11 @@ class CsvDataTableTest {
""";
Files.writeString(tempCsvFile.toPath(), csvContent);
List<DataColumn> columns = List.of(
- new DataColumnFixed("column1", Cardinality.REQUIRED, Type.INTEGER),
- new DataColumnFixed("column2", Cardinality.OPTIONAL, Type.STRING),
- new DataColumnFixed("column3", Cardinality.OPTIONAL, Type.GEOMETRY));
+ new DataColumnFixed("column1", Cardinality.REQUIRED, ColumnType.INTEGER),
+ new DataColumnFixed("column2", Cardinality.OPTIONAL, ColumnType.STRING),
+ new DataColumnFixed("column3", Cardinality.OPTIONAL, ColumnType.GEOMETRY));
DataSchema schema = new DataSchemaImpl("test_table", columns);
- DataTable dataTable = new CsvDataTable(schema, tempCsvFile, false, ';');
+ DataTable dataTable = new CsvDataTable(tempCsvFile, false);
assertEquals(2, dataTable.size());
int rowCount = 0;
for (DataRow row : dataTable) {
@@ -121,12 +116,12 @@ class CsvDataTableTest {
""";
Files.writeString(tempCsvFile.toPath(), csvContent);
List<DataColumn> columns = List.of(
- new DataColumnFixed("int_col", Cardinality.REQUIRED, Type.INTEGER),
- new DataColumnFixed("double_col", Cardinality.REQUIRED, Type.DOUBLE),
- new DataColumnFixed("bool_col", Cardinality.REQUIRED, Type.BOOLEAN),
- new DataColumnFixed("string_col", Cardinality.REQUIRED, Type.STRING));
+ new DataColumnFixed("int_col", Cardinality.REQUIRED, ColumnType.INTEGER),
+ new DataColumnFixed("double_col", Cardinality.REQUIRED, ColumnType.DOUBLE),
+ new DataColumnFixed("bool_col", Cardinality.REQUIRED, ColumnType.BOOLEAN),
+ new DataColumnFixed("string_col", Cardinality.REQUIRED, ColumnType.STRING));
DataSchema schema = new DataSchemaImpl("test_table", columns);
- DataTable dataTable = new CsvDataTable(schema, tempCsvFile, true, ',');
+ DataTable dataTable = new CsvDataTable(tempCsvFile, true);
assertEquals(2, dataTable.size());
int rowCount = 0;
for (DataRow row : dataTable) {
@@ -153,10 +148,10 @@ class CsvDataTableTest {
""";
Files.writeString(tempCsvFile.toPath(), csvContent);
List<DataColumn> columns = List.of(
- new DataColumnFixed("id", Cardinality.REQUIRED, Type.INTEGER),
- new DataColumnFixed("name", Cardinality.OPTIONAL, Type.STRING));
+ new DataColumnFixed("id", Cardinality.REQUIRED, ColumnType.INTEGER),
+ new DataColumnFixed("name", Cardinality.OPTIONAL, ColumnType.STRING));
DataSchema schema = new DataSchemaImpl("test_table", columns);
- DataTable dataTable = new CsvDataTable(schema, tempCsvFile, true, ',');
+ DataTable dataTable = new CsvDataTable(tempCsvFile, true);
assertThrows(RuntimeException.class, () -> {
for (DataRow row : dataTable) {
// This line should throw an exception because abc is not a valid integer
@@ -169,11 +164,7 @@ class CsvDataTableTest {
void testAddAndClearUnsupportedOperations() throws IOException {
String csvContent = "";
Files.writeString(tempCsvFile.toPath(), csvContent);
- List<DataColumn> columns = List.of(
- new DataColumnFixed("id", Cardinality.REQUIRED, Type.INTEGER),
- new DataColumnFixed("name", Cardinality.OPTIONAL, Type.STRING));
- DataSchema schema = new DataSchemaImpl("test_table", columns);
- DataTable dataTable = new CsvDataTable(schema, tempCsvFile, true, ',');
+ DataTable dataTable = new CsvDataTable(tempCsvFile, true);
assertThrows(UnsupportedOperationException.class, () -> dataTable.add(null));
assertThrows(UnsupportedOperationException.class, dataTable::clear);
}
@@ -187,11 +178,7 @@ class CsvDataTableTest {
3,Name3
""";
Files.writeString(tempCsvFile.toPath(), csvContent);
- List<DataColumn> columns = List.of(
- new DataColumnFixed("id", Cardinality.REQUIRED, Type.INTEGER),
- new DataColumnFixed("name", Cardinality.OPTIONAL, Type.STRING));
- DataSchema schema = new DataSchemaImpl("test_table", columns);
- DataTable dataTable = new CsvDataTable(schema, tempCsvFile, true, ',');
+ DataTable dataTable = new CsvDataTable(tempCsvFile, true);
assertEquals(3, dataTable.size());
}
}
diff --git a/baremaps-data/src/main/java/org/apache/baremaps/data/calcite/SqlDataSchema.java b/baremaps-data/src/main/java/org/apache/baremaps/data/calcite/SqlDataSchema.java
deleted file mode 100644
index 68651ae7e..000000000
--- a/baremaps-data/src/main/java/org/apache/baremaps/data/calcite/SqlDataSchema.java
+++ /dev/null
@@ -1,38 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to you under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.baremaps.data.calcite;
-
-import com.google.common.collect.ImmutableMap;
-import java.util.Map;
-import org.apache.calcite.schema.Table;
-import org.apache.calcite.schema.impl.AbstractSchema;
-
-public class SqlDataSchema extends AbstractSchema {
-
- private final Map<String, Table> tableMap;
-
- public SqlDataSchema(ImmutableMap<String, Table> tableMap) {
- this.tableMap = ImmutableMap.copyOf(tableMap);
- }
-
- @Override
- protected Map<String, Table> getTableMap() {
- return tableMap;
- }
-
-}
diff --git a/baremaps-data/src/main/java/org/apache/baremaps/data/calcite/SqlDataTable.java b/baremaps-data/src/main/java/org/apache/baremaps/data/calcite/SqlDataTable.java
deleted file mode 100644
index 497ac237a..000000000
--- a/baremaps-data/src/main/java/org/apache/baremaps/data/calcite/SqlDataTable.java
+++ /dev/null
@@ -1,65 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to you under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.baremaps.data.calcite;
-
-import org.apache.baremaps.store.DataColumn;
-import org.apache.baremaps.store.DataTable;
-import org.apache.calcite.DataContext;
-import org.apache.calcite.linq4j.Enumerable;
-import org.apache.calcite.linq4j.Linq4j;
-import org.apache.calcite.rel.type.RelDataType;
-import org.apache.calcite.rel.type.RelDataTypeFactory;
-import org.apache.calcite.schema.ScannableTable;
-import org.apache.calcite.schema.impl.AbstractTable;
-
-/**
- * A simple table based on a list.
- */
-public class SqlDataTable extends AbstractTable implements ScannableTable {
-
- private final DataTable table;
-
- private RelDataType rowType;
-
- public SqlDataTable(DataTable table) {
- this.table = table;
- }
-
- @Override
- public Enumerable<Object[]> scan(final DataContext root) {
- return Linq4j.asEnumerable(() -> table.stream()
- .map(row -> row.values().toArray())
- .iterator());
- }
-
- @Override
- public RelDataType getRowType(final RelDataTypeFactory typeFactory) {
- if (rowType == null) {
- rowType = createRowType(typeFactory);
- }
- return rowType;
- }
-
- private RelDataType createRowType(RelDataTypeFactory typeFactory) {
- var rowTypeBuilder = new RelDataTypeFactory.Builder(typeFactory);
- for (DataColumn column : table.schema().columns()) {
- rowTypeBuilder.add(column.name(), SqlTypeConversion.types.get(column.type()));
- }
- return rowTypeBuilder.build();
- }
-}
diff --git a/baremaps-data/src/main/java/org/apache/baremaps/data/calcite/SqlTypeConversion.java b/baremaps-data/src/main/java/org/apache/baremaps/data/calcite/SqlTypeConversion.java
deleted file mode 100644
index 1c320fa97..000000000
--- a/baremaps-data/src/main/java/org/apache/baremaps/data/calcite/SqlTypeConversion.java
+++ /dev/null
@@ -1,69 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to you under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.baremaps.data.calcite;
-
-import java.util.EnumMap;
-import org.apache.baremaps.store.DataColumn.Type;
-import org.apache.calcite.jdbc.JavaTypeFactoryImpl;
-import org.apache.calcite.rel.type.RelDataType;
-import org.apache.calcite.sql.type.SqlTypeName;
-import org.locationtech.jts.geom.*;
-
-public class SqlTypeConversion {
-
- static final EnumMap<Type, RelDataType> types = new EnumMap<>(Type.class);
-
- static {
- types.put(Type.BYTE, new JavaTypeFactoryImpl()
- .createSqlType(SqlTypeName.TINYINT));
- types.put(Type.BOOLEAN, new JavaTypeFactoryImpl()
- .createSqlType(SqlTypeName.BOOLEAN));
- types.put(Type.SHORT, new JavaTypeFactoryImpl()
- .createSqlType(SqlTypeName.SMALLINT));
- types.put(Type.INTEGER, new JavaTypeFactoryImpl()
- .createSqlType(SqlTypeName.INTEGER));
- types.put(Type.LONG, new JavaTypeFactoryImpl()
- .createSqlType(SqlTypeName.BIGINT));
- types.put(Type.FLOAT, new JavaTypeFactoryImpl()
- .createSqlType(SqlTypeName.FLOAT));
- types.put(Type.DOUBLE, new JavaTypeFactoryImpl()
- .createSqlType(SqlTypeName.DOUBLE));
- types.put(Type.STRING, new JavaTypeFactoryImpl()
- .createSqlType(SqlTypeName.VARCHAR));
- types.put(Type.GEOMETRY, new JavaTypeFactoryImpl()
- .createJavaType(Geometry.class));
- types.put(Type.POINT, new JavaTypeFactoryImpl()
- .createJavaType(Point.class));
- types.put(Type.LINESTRING, new JavaTypeFactoryImpl()
- .createJavaType(LineString.class));
- types.put(Type.POLYGON, new JavaTypeFactoryImpl()
- .createJavaType(Polygon.class));
- types.put(Type.MULTIPOINT, new JavaTypeFactoryImpl()
- .createJavaType(MultiPoint.class));
- types.put(Type.MULTILINESTRING, new JavaTypeFactoryImpl()
- .createJavaType(MultiLineString.class));
- types.put(Type.MULTIPOLYGON, new JavaTypeFactoryImpl()
- .createJavaType(MultiPolygon.class));
- types.put(Type.GEOMETRYCOLLECTION, new JavaTypeFactoryImpl()
- .createJavaType(GeometryCollection.class));
- }
-
- private SqlTypeConversion() {
- // Prevent instantiation
- }
-}
diff --git a/baremaps-data/src/main/java/org/apache/baremaps/data/collection/DataConversions.java b/baremaps-data/src/main/java/org/apache/baremaps/data/collection/DataConversions.java
index 75d5a1a51..10d5414d7 100644
--- a/baremaps-data/src/main/java/org/apache/baremaps/data/collection/DataConversions.java
+++ b/baremaps-data/src/main/java/org/apache/baremaps/data/collection/DataConversions.java
@@ -168,9 +168,9 @@ public class DataConversions {
@Override
public void close() throws IOException {
- if (collection instanceof Closeable closeable) {
- closeable.close();
- }
+ if (collection instanceof Closeable closeable) {
+ closeable.close();
+ }
}
}
@@ -263,9 +263,9 @@ public class DataConversions {
@Override
public void close() throws IOException {
- if (list instanceof Closeable closeable) {
- closeable.close();
- }
+ if (list instanceof Closeable closeable) {
+ closeable.close();
+ }
}
}
diff --git a/baremaps-data/src/main/java/org/apache/baremaps/data/collection/FixedSizeDataList.java b/baremaps-data/src/main/java/org/apache/baremaps/data/collection/FixedSizeDataList.java
index 1d5d01dc9..780462f97 100644
--- a/baremaps-data/src/main/java/org/apache/baremaps/data/collection/FixedSizeDataList.java
+++ b/baremaps-data/src/main/java/org/apache/baremaps/data/collection/FixedSizeDataList.java
@@ -19,6 +19,7 @@ package org.apache.baremaps.data.collection;
+import java.io.IOException;
import java.nio.ByteBuffer;
import java.util.concurrent.atomic.AtomicLong;
import org.apache.baremaps.data.memory.Memory;
@@ -109,4 +110,9 @@ public class FixedSizeDataList<E> implements DataList<E> {
public void clear() {
size.set(0);
}
+
+ @Override
+ public void close() throws IOException {
+ memory.close();
+ }
}
diff --git a/baremaps-data/src/main/java/org/apache/baremaps/data/collection/IndexedDataList.java b/baremaps-data/src/main/java/org/apache/baremaps/data/collection/IndexedDataList.java
index dbe7e555d..600ec0b26 100644
--- a/baremaps-data/src/main/java/org/apache/baremaps/data/collection/IndexedDataList.java
+++ b/baremaps-data/src/main/java/org/apache/baremaps/data/collection/IndexedDataList.java
@@ -19,6 +19,7 @@ package org.apache.baremaps.data.collection;
+import java.io.IOException;
import org.apache.baremaps.data.type.LongDataType;
/**
@@ -86,4 +87,10 @@ public class IndexedDataList<E> implements DataList<E> {
index.clear();
values.clear();
}
+
+ @Override
+ public void close() throws IOException {
+ index.close();
+ values.close();
+ }
}
diff --git a/baremaps-data/src/main/java/org/apache/baremaps/data/collection/MemoryAlignedDataList.java b/baremaps-data/src/main/java/org/apache/baremaps/data/collection/MemoryAlignedDataList.java
index e29f33839..562fdf449 100644
--- a/baremaps-data/src/main/java/org/apache/baremaps/data/collection/MemoryAlignedDataList.java
+++ b/baremaps-data/src/main/java/org/apache/baremaps/data/collection/MemoryAlignedDataList.java
@@ -129,4 +129,8 @@ public class MemoryAlignedDataList<E> implements DataList<E> {
}
}
+ @Override
+ public void close() throws IOException {
+ memory.close();
+ }
}
diff --git a/baremaps-data/src/main/java/org/apache/baremaps/data/store/DataStoreImpl.java b/baremaps-data/src/main/java/org/apache/baremaps/data/store/DataStoreImpl.java
new file mode 100644
index 000000000..c866b1b41
--- /dev/null
+++ b/baremaps-data/src/main/java/org/apache/baremaps/data/store/DataStoreImpl.java
@@ -0,0 +1,59 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to you under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.baremaps.data.store;
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Map;
+import java.util.stream.Collectors;
+import org.apache.baremaps.store.*;
+
+public class DataStoreImpl implements DataStore {
+
+ private Map<String, DataTable> tables;
+
+ public DataStoreImpl(List<DataTable> tables) {
+ this.tables =
+ tables.stream().collect(Collectors.toMap(table -> table.schema().name(), table -> table));
+ }
+
+ @Override
+ public List<String> list() throws DataStoreException {
+ return new ArrayList<>(tables.keySet());
+ }
+
+ @Override
+ public DataTable get(String name) throws DataStoreException {
+ return tables.get(name);
+ }
+
+ @Override
+ public void add(DataTable table) throws DataStoreException {
+ tables.put(table.schema().name(), table);
+ }
+
+ @Override
+ public void add(String name, DataTable table) throws DataStoreException {
+ tables.put(name, table);
+ }
+
+ @Override
+ public void remove(String name) throws DataStoreException {
+ tables.remove(name);
+ }
+}
diff --git a/baremaps-data/src/main/java/org/apache/baremaps/data/store/BaremapsDataTable.java b/baremaps-data/src/main/java/org/apache/baremaps/data/store/DataTableImpl.java
similarity index 76%
rename from baremaps-data/src/main/java/org/apache/baremaps/data/store/BaremapsDataTable.java
rename to baremaps-data/src/main/java/org/apache/baremaps/data/store/DataTableImpl.java
index 89123dab6..59c61d5d7 100644
--- a/baremaps-data/src/main/java/org/apache/baremaps/data/store/BaremapsDataTable.java
+++ b/baremaps-data/src/main/java/org/apache/baremaps/data/store/DataTableImpl.java
@@ -17,8 +17,11 @@
package org.apache.baremaps.data.store;
+import java.io.IOException;
import java.util.Iterator;
+import org.apache.baremaps.data.collection.AppendOnlyLog;
import org.apache.baremaps.data.collection.DataCollection;
+import org.apache.baremaps.data.type.DataTypeImpl;
import org.apache.baremaps.store.DataRow;
import org.apache.baremaps.store.DataSchema;
import org.apache.baremaps.store.DataTable;
@@ -26,7 +29,7 @@ import org.apache.baremaps.store.DataTable;
/**
* A {@link DataTable} is a collection of rows respecting a {@link DataSchema}.
*/
-public class BaremapsDataTable implements DataTable {
+public class DataTableImpl implements DataTable {
private final DataSchema schema;
@@ -38,11 +41,21 @@ public class BaremapsDataTable implements DataTable {
* @param schema the schema of the rows
* @param rows the rows
*/
- public BaremapsDataTable(DataSchema schema, DataCollection<DataRow> rows) {
+ public DataTableImpl(DataSchema schema, DataCollection<DataRow> rows) {
this.schema = schema;
this.rows = rows;
}
+ /**
+ * Constructs a {@link DataTable} with the specified row {@link DataSchema}.
+ *
+ * @param schema the schema of the rows
+ */
+ public DataTableImpl(DataSchema schema) {
+ this.schema = schema;
+ this.rows = new AppendOnlyLog<>(new DataTypeImpl(schema));
+ }
+
/**
* {@inheritDoc}
*/
@@ -75,12 +88,13 @@ public class BaremapsDataTable implements DataTable {
return rows.size();
}
- /**
- * {@inheritDoc}
- */
@Override
public Iterator<DataRow> iterator() {
return rows.iterator();
}
+ @Override
+ public void close() throws IOException {
+ rows.close();
+ }
}
diff --git a/baremaps-data/src/main/java/org/apache/baremaps/data/type/DataType.java b/baremaps-data/src/main/java/org/apache/baremaps/data/type/DataType.java
index 281cc794b..a9fb3b6cd 100644
--- a/baremaps-data/src/main/java/org/apache/baremaps/data/type/DataType.java
+++ b/baremaps-data/src/main/java/org/apache/baremaps/data/type/DataType.java
@@ -63,4 +63,5 @@ public interface DataType<T> {
* @return the object
*/
T read(final ByteBuffer buffer, final int position);
+
}
diff --git a/baremaps-data/src/main/java/org/apache/baremaps/data/type/RowDataType.java b/baremaps-data/src/main/java/org/apache/baremaps/data/type/DataTypeImpl.java
similarity index 67%
rename from baremaps-data/src/main/java/org/apache/baremaps/data/type/RowDataType.java
rename to baremaps-data/src/main/java/org/apache/baremaps/data/type/DataTypeImpl.java
index ba8bede60..d81b7a509 100644
--- a/baremaps-data/src/main/java/org/apache/baremaps/data/type/RowDataType.java
+++ b/baremaps-data/src/main/java/org/apache/baremaps/data/type/DataTypeImpl.java
@@ -21,7 +21,7 @@ import java.nio.ByteBuffer;
import java.util.ArrayList;
import java.util.EnumMap;
import org.apache.baremaps.store.DataColumn;
-import org.apache.baremaps.store.DataColumn.Type;
+import org.apache.baremaps.store.DataColumn.ColumnType;
import org.apache.baremaps.store.DataRow;
import org.apache.baremaps.store.DataRowImpl;
import org.apache.baremaps.store.DataSchema;
@@ -29,33 +29,33 @@ import org.apache.baremaps.store.DataSchema;
/**
* A data type for rows.
*/
-public class RowDataType implements DataType<DataRow> {
+public class DataTypeImpl implements DataType<DataRow> {
- private static final EnumMap<Type, DataType> types = new EnumMap<>(Type.class);
+ private static final EnumMap<ColumnType, DataType> types = new EnumMap<>(ColumnType.class);
static {
- types.put(Type.BYTE, new ByteDataType());
- types.put(Type.BOOLEAN, new BooleanDataType());
- types.put(Type.SHORT, new ShortDataType());
- types.put(Type.INTEGER, new IntegerDataType());
- types.put(Type.LONG, new LongDataType());
- types.put(Type.FLOAT, new FloatDataType());
- types.put(Type.DOUBLE, new DoubleDataType());
- types.put(Type.STRING, new StringDataType());
- types.put(Type.GEOMETRY, new GeometryDataType());
- types.put(Type.POINT, new PointDataType());
- types.put(Type.LINESTRING, new LineStringDataType());
- types.put(Type.POLYGON, new PolygonDataType());
- types.put(Type.MULTIPOINT, new MultiPointDataType());
- types.put(Type.MULTILINESTRING, new MultiLineStringDataType());
- types.put(Type.MULTIPOLYGON, new MultiPolygonDataType());
- types.put(Type.GEOMETRYCOLLECTION, new GeometryCollectionDataType());
- types.put(Type.COORDINATE, new CoordinateDataType());
+ types.put(ColumnType.BYTE, new ByteDataType());
+ types.put(ColumnType.BOOLEAN, new BooleanDataType());
+ types.put(ColumnType.SHORT, new ShortDataType());
+ types.put(ColumnType.INTEGER, new IntegerDataType());
+ types.put(ColumnType.LONG, new LongDataType());
+ types.put(ColumnType.FLOAT, new FloatDataType());
+ types.put(ColumnType.DOUBLE, new DoubleDataType());
+ types.put(ColumnType.STRING, new StringDataType());
+ types.put(ColumnType.GEOMETRY, new GeometryDataType());
+ types.put(ColumnType.POINT, new PointDataType());
+ types.put(ColumnType.LINESTRING, new LineStringDataType());
+ types.put(ColumnType.POLYGON, new PolygonDataType());
+ types.put(ColumnType.MULTIPOINT, new MultiPointDataType());
+ types.put(ColumnType.MULTILINESTRING, new MultiLineStringDataType());
+ types.put(ColumnType.MULTIPOLYGON, new MultiPolygonDataType());
+ types.put(ColumnType.GEOMETRYCOLLECTION, new GeometryCollectionDataType());
+ types.put(ColumnType.COORDINATE, new CoordinateDataType());
}
private final DataSchema rowType;
- public RowDataType(DataSchema rowType) {
+ public DataTypeImpl(DataSchema rowType) {
this.rowType = rowType;
}
diff --git a/baremaps-data/src/test/java/org/apache/baremaps/data/type/DataTypeProvider.java b/baremaps-data/src/test/java/org/apache/baremaps/data/type/DataTypeProvider.java
index 1361a2b0b..955732eaf 100644
--- a/baremaps-data/src/test/java/org/apache/baremaps/data/type/DataTypeProvider.java
+++ b/baremaps-data/src/test/java/org/apache/baremaps/data/type/DataTypeProvider.java
@@ -22,7 +22,7 @@ import java.util.List;
import java.util.Map;
import java.util.stream.Stream;
import org.apache.baremaps.store.DataColumn.Cardinality;
-import org.apache.baremaps.store.DataColumn.Type;
+import org.apache.baremaps.store.DataColumn.ColumnType;
import org.apache.baremaps.store.DataColumnFixed;
import org.apache.baremaps.store.DataRow;
import org.apache.baremaps.store.DataSchema;
@@ -35,23 +35,24 @@ public class DataTypeProvider {
private static final GeometryFactory geometryFactory = new GeometryFactory();
private static final DataSchema DATA_SCHEMA = new DataSchemaImpl("row", List.of(
- new DataColumnFixed("byte", Cardinality.OPTIONAL, Type.BYTE),
- new DataColumnFixed("boolean", Cardinality.OPTIONAL, Type.BOOLEAN),
- new DataColumnFixed("short", Cardinality.OPTIONAL, Type.SHORT),
- new DataColumnFixed("integer", Cardinality.OPTIONAL, Type.INTEGER),
- new DataColumnFixed("long", Cardinality.OPTIONAL, Type.LONG),
- new DataColumnFixed("float", Cardinality.OPTIONAL, Type.FLOAT),
- new DataColumnFixed("double", Cardinality.OPTIONAL, Type.DOUBLE),
- new DataColumnFixed("string", Cardinality.OPTIONAL, Type.STRING),
- new DataColumnFixed("geometry", Cardinality.OPTIONAL, Type.GEOMETRY),
- new DataColumnFixed("point", Cardinality.OPTIONAL, Type.POINT),
- new DataColumnFixed("linestring", Cardinality.OPTIONAL, Type.LINESTRING),
- new DataColumnFixed("polygon", Cardinality.OPTIONAL, Type.POLYGON),
- new DataColumnFixed("multipoint", Cardinality.OPTIONAL, Type.MULTIPOINT),
- new DataColumnFixed("multilinestring", Cardinality.OPTIONAL, Type.MULTILINESTRING),
- new DataColumnFixed("multipolygon", Cardinality.OPTIONAL, Type.MULTIPOLYGON),
- new DataColumnFixed("geometrycollection", Cardinality.OPTIONAL, Type.GEOMETRYCOLLECTION),
- new DataColumnFixed("coordinate", Cardinality.OPTIONAL, Type.COORDINATE)));
+ new DataColumnFixed("byte", Cardinality.OPTIONAL, ColumnType.BYTE),
+ new DataColumnFixed("boolean", Cardinality.OPTIONAL, ColumnType.BOOLEAN),
+ new DataColumnFixed("short", Cardinality.OPTIONAL, ColumnType.SHORT),
+ new DataColumnFixed("integer", Cardinality.OPTIONAL, ColumnType.INTEGER),
+ new DataColumnFixed("long", Cardinality.OPTIONAL, ColumnType.LONG),
+ new DataColumnFixed("float", Cardinality.OPTIONAL, ColumnType.FLOAT),
+ new DataColumnFixed("double", Cardinality.OPTIONAL, ColumnType.DOUBLE),
+ new DataColumnFixed("string", Cardinality.OPTIONAL, ColumnType.STRING),
+ new DataColumnFixed("geometry", Cardinality.OPTIONAL, ColumnType.GEOMETRY),
+ new DataColumnFixed("point", Cardinality.OPTIONAL, ColumnType.POINT),
+ new DataColumnFixed("linestring", Cardinality.OPTIONAL, ColumnType.LINESTRING),
+ new DataColumnFixed("polygon", Cardinality.OPTIONAL, ColumnType.POLYGON),
+ new DataColumnFixed("multipoint", Cardinality.OPTIONAL, ColumnType.MULTIPOINT),
+ new DataColumnFixed("multilinestring", Cardinality.OPTIONAL, ColumnType.MULTILINESTRING),
+ new DataColumnFixed("multipolygon", Cardinality.OPTIONAL, ColumnType.MULTIPOLYGON),
+ new DataColumnFixed("geometrycollection", Cardinality.OPTIONAL,
+ ColumnType.GEOMETRYCOLLECTION),
+ new DataColumnFixed("coordinate", Cardinality.OPTIONAL, ColumnType.COORDINATE)));
private static final DataRow DATA_ROW = DATA_SCHEMA.createRow()
.with("byte", Byte.MAX_VALUE)
@@ -345,7 +346,7 @@ public class DataTypeProvider {
new Coordinate(4, 1), new Coordinate(3, 1)})})})),
// Row
- Arguments.of(new RowDataType(DATA_SCHEMA), DATA_ROW),
+ Arguments.of(new DataTypeImpl(DATA_SCHEMA), DATA_ROW),
// Geometry
Arguments.of(new GeometryDataType(),
diff --git a/baremaps-flatgeobuf/src/main/java/org/apache/baremaps/flatgeobuf/FlatGeoBufDataTable.java b/baremaps-flatgeobuf/src/main/java/org/apache/baremaps/flatgeobuf/FlatGeoBufDataTable.java
index 6e6679231..42d9467d0 100644
--- a/baremaps-flatgeobuf/src/main/java/org/apache/baremaps/flatgeobuf/FlatGeoBufDataTable.java
+++ b/baremaps-flatgeobuf/src/main/java/org/apache/baremaps/flatgeobuf/FlatGeoBufDataTable.java
@@ -27,7 +27,7 @@ import java.util.Iterator;
import java.util.NoSuchElementException;
import java.util.Objects;
import org.apache.baremaps.store.*;
-import org.apache.baremaps.store.DataColumn.Type;
+import org.apache.baremaps.store.DataColumn.ColumnType;
/**
* A {@link DataTable} that stores rows in a flatgeobuf file.
@@ -38,6 +38,8 @@ public class FlatGeoBufDataTable implements DataTable {
private final DataSchema schema;
+ private RowIterator iterator;
+
/**
* Constructs a table from a flatgeobuf file (used for reading).
*
@@ -88,13 +90,7 @@ public class FlatGeoBufDataTable implements DataTable {
@Override
public Iterator<DataRow> iterator() {
try {
- var reader = new FlatGeoBufReader(FileChannel.open(file, StandardOpenOption.READ));
-
- var header = reader.readHeader();
- reader.skipIndex();
-
- // create the feature stream
- return new RowIterator(reader, header, schema);
+ return (iterator = new RowIterator(file, schema));
} catch (IOException e) {
throw new DataStoreException(e);
}
@@ -135,7 +131,7 @@ public class FlatGeoBufDataTable implements DataTable {
schema.columns().stream()
.filter(c -> c.cardinality() == DataColumn.Cardinality.REQUIRED)
.forEach(c -> {
- if (Objects.requireNonNull(c.type()) == Type.BINARY) {
+ if (Objects.requireNonNull(c.type()) == ColumnType.BINARY) {
throw new UnsupportedOperationException();
}
});
@@ -172,10 +168,17 @@ public class FlatGeoBufDataTable implements DataTable {
}
}
+ @Override
+ public void close() throws Exception {
+ if (iterator != null) {
+ iterator.close();
+ }
+ }
+
/**
* An iterator over rows in a flatgeobuf file.
*/
- public static class RowIterator implements Iterator<DataRow> {
+ public static class RowIterator implements Iterator<DataRow>, AutoCloseable {
private final FlatGeoBuf.Header header;
@@ -188,17 +191,16 @@ public class FlatGeoBufDataTable implements DataTable {
/**
* Constructs a row iterator.
*
- * @param reader the channel to read from
- * @param header the header of the file
+ * @param file the path to the flatgeobuf file
* @param schema the schema of the table
*/
public RowIterator(
- FlatGeoBufReader reader,
- FlatGeoBuf.Header header,
- DataSchema schema) {
- this.reader = reader;
- this.header = header;
+ Path file,
+ DataSchema schema) throws IOException {
+ this.reader = new FlatGeoBufReader(FileChannel.open(file, StandardOpenOption.READ));
+ this.header = reader.readHeader();
this.schema = schema;
+ reader.skipIndex();
}
/**
@@ -222,5 +224,10 @@ public class FlatGeoBufDataTable implements DataTable {
throw new NoSuchElementException(e);
}
}
+
+ @Override
+ public void close() throws Exception {
+ reader.close();
+ }
}
}
diff --git a/baremaps-flatgeobuf/src/main/java/org/apache/baremaps/flatgeobuf/FlatGeoBufTypeConversion.java b/baremaps-flatgeobuf/src/main/java/org/apache/baremaps/flatgeobuf/FlatGeoBufTypeConversion.java
index 252d21ca4..db109a5bb 100644
--- a/baremaps-flatgeobuf/src/main/java/org/apache/baremaps/flatgeobuf/FlatGeoBufTypeConversion.java
+++ b/baremaps-flatgeobuf/src/main/java/org/apache/baremaps/flatgeobuf/FlatGeoBufTypeConversion.java
@@ -20,25 +20,24 @@ package org.apache.baremaps.flatgeobuf;
import java.util.*;
import org.apache.baremaps.flatgeobuf.FlatGeoBuf.Feature;
-import org.apache.baremaps.flatgeobuf.generated.ColumnType;
import org.apache.baremaps.store.*;
import org.apache.baremaps.store.DataColumn.Cardinality;
-import org.apache.baremaps.store.DataColumn.Type;
+import org.apache.baremaps.store.DataColumn.ColumnType;
import org.locationtech.jts.geom.Geometry;
public class FlatGeoBufTypeConversion {
- private static final Map<Type, Integer> types = new EnumMap<>(Type.class);
+ private static final Map<ColumnType, Integer> types = new EnumMap<>(ColumnType.class);
static {
- types.put(Type.BYTE, ColumnType.Byte);
- types.put(Type.BOOLEAN, ColumnType.Bool);
- types.put(Type.SHORT, ColumnType.Short);
- types.put(Type.INTEGER, ColumnType.Int);
- types.put(Type.LONG, ColumnType.Long);
- types.put(Type.FLOAT, ColumnType.Float);
- types.put(Type.DOUBLE, ColumnType.Double);
- types.put(Type.STRING, ColumnType.String);
+ types.put(ColumnType.BYTE, org.apache.baremaps.flatgeobuf.generated.ColumnType.Byte);
+ types.put(ColumnType.BOOLEAN, org.apache.baremaps.flatgeobuf.generated.ColumnType.Bool);
+ types.put(ColumnType.SHORT, org.apache.baremaps.flatgeobuf.generated.ColumnType.Short);
+ types.put(ColumnType.INTEGER, org.apache.baremaps.flatgeobuf.generated.ColumnType.Int);
+ types.put(ColumnType.LONG, org.apache.baremaps.flatgeobuf.generated.ColumnType.Long);
+ types.put(ColumnType.FLOAT, org.apache.baremaps.flatgeobuf.generated.ColumnType.Float);
+ types.put(ColumnType.DOUBLE, org.apache.baremaps.flatgeobuf.generated.ColumnType.Double);
+ types.put(ColumnType.STRING, org.apache.baremaps.flatgeobuf.generated.ColumnType.String);
}
private FlatGeoBufTypeConversion() {
@@ -51,7 +50,7 @@ public class FlatGeoBufTypeConversion {
.map(column -> new DataColumnFixed(
column.name(),
column.nullable() ? Cardinality.OPTIONAL : Cardinality.REQUIRED,
- Type.fromBinding(fromColumnType(column.type()))))
+ ColumnType.fromBinding(fromColumnType(column.type()))))
.map(DataColumn.class::cast)
.toList();
return new DataSchemaImpl(name, columns);
diff --git a/baremaps-geopackage/src/main/java/org/apache/baremaps/geopackage/GeoPackageDataTable.java b/baremaps-geopackage/src/main/java/org/apache/baremaps/geopackage/GeoPackageDataTable.java
index 59e91c180..a2e76c183 100644
--- a/baremaps-geopackage/src/main/java/org/apache/baremaps/geopackage/GeoPackageDataTable.java
+++ b/baremaps-geopackage/src/main/java/org/apache/baremaps/geopackage/GeoPackageDataTable.java
@@ -18,6 +18,7 @@
package org.apache.baremaps.geopackage;
+import java.io.IOException;
import java.nio.file.Path;
import java.util.*;
import mil.nga.geopackage.GeoPackage;
@@ -28,7 +29,7 @@ import mil.nga.geopackage.features.user.FeatureResultSet;
import mil.nga.geopackage.geom.GeoPackageGeometryData;
import org.apache.baremaps.store.*;
import org.apache.baremaps.store.DataColumn.Cardinality;
-import org.apache.baremaps.store.DataColumn.Type;
+import org.apache.baremaps.store.DataColumn.ColumnType;
import org.locationtech.jts.geom.*;
/**
@@ -40,7 +41,7 @@ public class GeoPackageDataTable implements DataTable {
private final String name;
- private final GeoPackage geoPackage;
+ private final GeoPackage geoPackage;
private final FeatureDao featureDao;
@@ -71,11 +72,11 @@ public class GeoPackageDataTable implements DataTable {
geometryFactory = new GeometryFactory(new PrecisionModel(), (int) featureDao.getSrs().getId());
}
- private Type classType(FeatureColumn column) {
+ private ColumnType classType(FeatureColumn column) {
if (column.isGeometry()) {
- return Type.fromBinding(Geometry.class);
+ return ColumnType.fromBinding(Geometry.class);
} else {
- return Type.fromBinding(column.getDataType().getClassType());
+ return ColumnType.fromBinding(column.getDataType().getClassType());
}
}
@@ -222,6 +223,11 @@ public class GeoPackageDataTable implements DataTable {
return geometryFactory.createPoint(new Coordinate(point.getX(), point.getY()));
}
+ @Override
+ public void close() throws IOException {
+ geoPackage.close();
+ }
+
/**
* An iterator over the rows of a GeoPackage data table.
*/
diff --git a/baremaps-geoparquet/src/main/java/org/apache/baremaps/geoparquet/GeoParquetDataTable.java b/baremaps-geoparquet/src/main/java/org/apache/baremaps/geoparquet/GeoParquetDataTable.java
index de1f8b32e..db2c2f729 100644
--- a/baremaps-geoparquet/src/main/java/org/apache/baremaps/geoparquet/GeoParquetDataTable.java
+++ b/baremaps-geoparquet/src/main/java/org/apache/baremaps/geoparquet/GeoParquetDataTable.java
@@ -17,6 +17,7 @@
package org.apache.baremaps.geoparquet;
+import java.io.IOException;
import java.net.URI;
import java.util.Iterator;
import java.util.Spliterator;
@@ -95,4 +96,11 @@ public class GeoParquetDataTable implements DataTable {
throw new GeoParquetException("Fail to read the SRID from the GeoParquet metadata", e);
}
}
+
+ @Override
+ public void close() throws IOException {
+ if (reader != null) {
+ reader.close();
+ }
+ }
}
diff --git a/baremaps-geoparquet/src/main/java/org/apache/baremaps/geoparquet/GeoParquetReader.java b/baremaps-geoparquet/src/main/java/org/apache/baremaps/geoparquet/GeoParquetReader.java
index e965d250a..6a4bd4050 100644
--- a/baremaps-geoparquet/src/main/java/org/apache/baremaps/geoparquet/GeoParquetReader.java
+++ b/baremaps-geoparquet/src/main/java/org/apache/baremaps/geoparquet/GeoParquetReader.java
@@ -19,6 +19,7 @@ package org.apache.baremaps.geoparquet;
import com.fasterxml.jackson.databind.DeserializationFeature;
import com.fasterxml.jackson.databind.ObjectMapper;
+import java.io.Closeable;
import java.io.IOException;
import java.util.*;
import java.util.concurrent.atomic.AtomicLong;
@@ -42,7 +43,7 @@ import org.locationtech.jts.geom.Envelope;
* automatically inferred from the files. The reader can be used to read the records in a sequential
* or parallel manner. It is also capable of filtering records based on an envelope.
*/
-public class GeoParquetReader {
+public class GeoParquetReader implements Closeable {
protected final Configuration configuration;
protected final List<FileStatus> files;
@@ -194,6 +195,11 @@ public class GeoParquetReader {
return streamGeoParquetGroups(true);
}
+ @Override
+ public void close() throws IOException {
+ // TODO: Implement close
+ }
+
private record FileInfo(
FileStatus file,
long recordCount,
diff --git a/baremaps-geoparquet/src/main/java/org/apache/baremaps/geoparquet/GeoParquetSpliterator.java b/baremaps-geoparquet/src/main/java/org/apache/baremaps/geoparquet/GeoParquetSpliterator.java
index 12d4a2662..8ab7581ba 100644
--- a/baremaps-geoparquet/src/main/java/org/apache/baremaps/geoparquet/GeoParquetSpliterator.java
+++ b/baremaps-geoparquet/src/main/java/org/apache/baremaps/geoparquet/GeoParquetSpliterator.java
@@ -284,4 +284,5 @@ class GeoParquetSpliterator implements Spliterator<GeoParquetGroup> {
public int characteristics() {
return NONNULL | IMMUTABLE;
}
+
}
diff --git a/baremaps-geoparquet/src/main/java/org/apache/baremaps/geoparquet/GeoParquetTypeConversion.java b/baremaps-geoparquet/src/main/java/org/apache/baremaps/geoparquet/GeoParquetTypeConversion.java
index f35e6c010..bbf25dfcb 100644
--- a/baremaps-geoparquet/src/main/java/org/apache/baremaps/geoparquet/GeoParquetTypeConversion.java
+++ b/baremaps-geoparquet/src/main/java/org/apache/baremaps/geoparquet/GeoParquetTypeConversion.java
@@ -25,7 +25,7 @@ import org.apache.baremaps.geoparquet.GeoParquetSchema.Field;
import org.apache.baremaps.geoparquet.GeoParquetSchema.GroupField;
import org.apache.baremaps.store.*;
import org.apache.baremaps.store.DataColumn.Cardinality;
-import org.apache.baremaps.store.DataColumn.Type;
+import org.apache.baremaps.store.DataColumn.ColumnType;
import org.apache.parquet.io.api.Binary;
public class GeoParquetTypeConversion {
@@ -50,15 +50,15 @@ public class GeoParquetTypeConversion {
case REPEATED -> Cardinality.REPEATED;
};
return switch (field.type()) {
- case BINARY -> new DataColumnFixed(field.name(), cardinality, Type.BINARY);
- case BOOLEAN -> new DataColumnFixed(field.name(), cardinality, Type.BOOLEAN);
- case INTEGER -> new DataColumnFixed(field.name(), cardinality, Type.INTEGER);
- case INT96, LONG -> new DataColumnFixed(field.name(), cardinality, Type.LONG);
- case FLOAT -> new DataColumnFixed(field.name(), cardinality, Type.FLOAT);
- case DOUBLE -> new DataColumnFixed(field.name(), cardinality, Type.DOUBLE);
- case STRING -> new DataColumnFixed(field.name(), cardinality, Type.STRING);
- case GEOMETRY -> new DataColumnFixed(field.name(), cardinality, Type.GEOMETRY);
- case ENVELOPE -> new DataColumnFixed(field.name(), cardinality, Type.ENVELOPE);
+ case BINARY -> new DataColumnFixed(field.name(), cardinality, ColumnType.BINARY);
+ case BOOLEAN -> new DataColumnFixed(field.name(), cardinality, ColumnType.BOOLEAN);
+ case INTEGER -> new DataColumnFixed(field.name(), cardinality, ColumnType.INTEGER);
+ case INT96, LONG -> new DataColumnFixed(field.name(), cardinality, ColumnType.LONG);
+ case FLOAT -> new DataColumnFixed(field.name(), cardinality, ColumnType.FLOAT);
+ case DOUBLE -> new DataColumnFixed(field.name(), cardinality, ColumnType.DOUBLE);
+ case STRING -> new DataColumnFixed(field.name(), cardinality, ColumnType.STRING);
+ case GEOMETRY -> new DataColumnFixed(field.name(), cardinality, ColumnType.GEOMETRY);
+ case ENVELOPE -> new DataColumnFixed(field.name(), cardinality, ColumnType.ENVELOPE);
case GROUP -> new DataColumnNested(field.name(), cardinality,
asDataColumns(((GroupField) field).schema()));
};
diff --git a/baremaps-openstreetmap/pom.xml b/baremaps-openstreetmap/pom.xml
index 7fd49a360..42d9c747f 100644
--- a/baremaps-openstreetmap/pom.xml
+++ b/baremaps-openstreetmap/pom.xml
@@ -42,9 +42,14 @@ limitations under the License.
<groupId>org.locationtech.jts</groupId>
<artifactId>jts-core</artifactId>
</dependency>
+ <dependency>
+ <groupId>org.locationtech.jts.io</groupId>
+ <artifactId>jts-io-common</artifactId>
+ </dependency>
<dependency>
<groupId>org.locationtech.proj4j</groupId>
<artifactId>proj4j</artifactId>
+ <version>${version.lib.proj4j}</version>
</dependency>
</dependencies>
diff --git a/baremaps-openstreetmap/src/main/java/org/apache/baremaps/openstreetmap/OpenStreetMapDataTable.java b/baremaps-openstreetmap/src/main/java/org/apache/baremaps/openstreetmap/OpenStreetMapDataTable.java
index 1d1f5bb24..6101b69c1 100644
--- a/baremaps-openstreetmap/src/main/java/org/apache/baremaps/openstreetmap/OpenStreetMapDataTable.java
+++ b/baremaps-openstreetmap/src/main/java/org/apache/baremaps/openstreetmap/OpenStreetMapDataTable.java
@@ -23,6 +23,7 @@ import java.util.stream.Stream;
import org.apache.baremaps.openstreetmap.OpenStreetMapFormat.EntityReader;
import org.apache.baremaps.openstreetmap.model.*;
import org.apache.baremaps.store.*;
+import org.apache.baremaps.store.DataColumn.ColumnType;
/**
* A DataTable implementation for OpenStreetMap data.
@@ -52,17 +53,17 @@ public class OpenStreetMapDataTable implements DataTable {
*/
private DataSchema createSchema() {
List<DataColumn> columns = List.of(
- new DataColumnFixed("id", DataColumn.Cardinality.REQUIRED, DataColumn.Type.LONG),
- new DataColumnFixed("type", DataColumn.Cardinality.REQUIRED, DataColumn.Type.STRING),
- new DataColumnFixed("version", DataColumn.Cardinality.OPTIONAL, DataColumn.Type.INTEGER),
+ new DataColumnFixed("id", DataColumn.Cardinality.REQUIRED, ColumnType.LONG),
+ new DataColumnFixed("type", DataColumn.Cardinality.REQUIRED, ColumnType.STRING),
+ new DataColumnFixed("version", DataColumn.Cardinality.OPTIONAL, ColumnType.INTEGER),
new DataColumnFixed("timestamp", DataColumn.Cardinality.OPTIONAL,
- DataColumn.Type.LOCAL_DATE_TIME),
- new DataColumnFixed("uid", DataColumn.Cardinality.OPTIONAL, DataColumn.Type.INTEGER),
- new DataColumnFixed("user", DataColumn.Cardinality.OPTIONAL, DataColumn.Type.STRING),
- new DataColumnFixed("changeset", DataColumn.Cardinality.OPTIONAL, DataColumn.Type.LONG),
- new DataColumnFixed("tags", DataColumn.Cardinality.OPTIONAL, DataColumn.Type.NESTED),
+ ColumnType.LOCAL_DATE_TIME),
+ new DataColumnFixed("uid", DataColumn.Cardinality.OPTIONAL, ColumnType.INTEGER),
+ new DataColumnFixed("user", DataColumn.Cardinality.OPTIONAL, ColumnType.STRING),
+ new DataColumnFixed("changeset", DataColumn.Cardinality.OPTIONAL, ColumnType.LONG),
+ new DataColumnFixed("tags", DataColumn.Cardinality.OPTIONAL, ColumnType.NESTED),
new DataColumnFixed("geometry", DataColumn.Cardinality.OPTIONAL,
- DataColumn.Type.GEOMETRY));
+ ColumnType.GEOMETRY));
return new DataSchemaImpl("osm_data", columns);
}
@@ -132,4 +133,9 @@ public class OpenStreetMapDataTable implements DataTable {
else
return "unknown";
}
+
+ @Override
+ public void close() throws Exception {
+ inputStream.close();
+ }
}
diff --git a/baremaps-openstreetmap/src/test/java/org/apache/baremaps/openstreetmap/geometry/EntityDataTypeGeometryBuilderTest.java b/baremaps-openstreetmap/src/test/java/org/apache/baremaps/openstreetmap/geometry/EntityDataColumnTypeGeometryBuilderTest.java
similarity index 99%
rename from baremaps-openstreetmap/src/test/java/org/apache/baremaps/openstreetmap/geometry/EntityDataTypeGeometryBuilderTest.java
rename to baremaps-openstreetmap/src/test/java/org/apache/baremaps/openstreetmap/geometry/EntityDataColumnTypeGeometryBuilderTest.java
index 5e15daebe..19a7bd7dd 100644
--- a/baremaps-openstreetmap/src/test/java/org/apache/baremaps/openstreetmap/geometry/EntityDataTypeGeometryBuilderTest.java
+++ b/baremaps-openstreetmap/src/test/java/org/apache/baremaps/openstreetmap/geometry/EntityDataColumnTypeGeometryBuilderTest.java
@@ -49,7 +49,7 @@ import org.locationtech.proj4j.CoordinateTransform;
import org.locationtech.proj4j.Proj4jException;
import org.locationtech.proj4j.ProjCoordinate;
-class EntityDataTypeGeometryBuilderTest {
+class EntityDataColumnTypeGeometryBuilderTest {
static final CRSFactory CRS_FACTORY = new CRSFactory();
diff --git a/baremaps-postgres/src/main/java/org/apache/baremaps/postgres/store/PostgresDataStore.java b/baremaps-postgres/src/main/java/org/apache/baremaps/postgres/store/PostgresDataStore.java
index 3e1170eaf..512b0b837 100644
--- a/baremaps-postgres/src/main/java/org/apache/baremaps/postgres/store/PostgresDataStore.java
+++ b/baremaps-postgres/src/main/java/org/apache/baremaps/postgres/store/PostgresDataStore.java
@@ -31,7 +31,7 @@ import org.apache.baremaps.postgres.copy.GeometryValueHandler;
import org.apache.baremaps.postgres.metadata.DatabaseMetadata;
import org.apache.baremaps.postgres.metadata.TableMetadata;
import org.apache.baremaps.store.*;
-import org.apache.baremaps.store.DataColumn.Type;
+import org.apache.baremaps.store.DataColumn.ColumnType;
import org.postgresql.PGConnection;
import org.postgresql.copy.PGCopyOutputStream;
import org.slf4j.Logger;
@@ -291,7 +291,7 @@ public class PostgresDataStore implements DataStore {
* @param type the type
* @return the handler
*/
- protected BaseValueHandler getHandler(Type type) {
+ protected BaseValueHandler getHandler(ColumnType type) {
return switch (type) {
case STRING -> new StringValueHandler();
case SHORT -> new ShortValueHandler<Short>();
diff --git a/baremaps-postgres/src/main/java/org/apache/baremaps/postgres/store/PostgresDataTable.java b/baremaps-postgres/src/main/java/org/apache/baremaps/postgres/store/PostgresDataTable.java
index 7df19761b..cad98ce33 100644
--- a/baremaps-postgres/src/main/java/org/apache/baremaps/postgres/store/PostgresDataTable.java
+++ b/baremaps-postgres/src/main/java/org/apache/baremaps/postgres/store/PostgresDataTable.java
@@ -242,6 +242,11 @@ public class PostgresDataTable implements DataTable {
return String.format("TRUNCATE TABLE \"%s\"", schema.name());
}
+ @Override
+ public void close() throws Exception {
+ // Do nothing
+ }
+
/**
* An iterator that iterates over the rows of a table.
*/
diff --git a/baremaps-postgres/src/main/java/org/apache/baremaps/postgres/store/PostgresTypeConversion.java b/baremaps-postgres/src/main/java/org/apache/baremaps/postgres/store/PostgresTypeConversion.java
index 6d03d9392..eae52d969 100644
--- a/baremaps-postgres/src/main/java/org/apache/baremaps/postgres/store/PostgresTypeConversion.java
+++ b/baremaps-postgres/src/main/java/org/apache/baremaps/postgres/store/PostgresTypeConversion.java
@@ -19,7 +19,7 @@ package org.apache.baremaps.postgres.store;
import java.util.EnumMap;
import java.util.Map;
-import org.apache.baremaps.store.DataColumn.Type;
+import org.apache.baremaps.store.DataColumn.ColumnType;
@SuppressWarnings("squid:S1192")
public class PostgresTypeConversion {
@@ -28,45 +28,45 @@ public class PostgresTypeConversion {
// Prevent instantiation
}
- protected static final Map<Type, String> typeToName = new EnumMap<>(Type.class);
+ protected static final Map<ColumnType, String> typeToName = new EnumMap<>(ColumnType.class);
static {
- typeToName.put(Type.STRING, "varchar");
- typeToName.put(Type.SHORT, "int2");
- typeToName.put(Type.INTEGER, "int4");
- typeToName.put(Type.LONG, "int8");
- typeToName.put(Type.FLOAT, "float4");
- typeToName.put(Type.DOUBLE, "float8");
- typeToName.put(Type.GEOMETRY, "geometry");
- typeToName.put(Type.POINT, "geometry");
- typeToName.put(Type.MULTIPOINT, "geometry");
- typeToName.put(Type.LINESTRING, "geometry");
- typeToName.put(Type.MULTILINESTRING, "geometry");
- typeToName.put(Type.POLYGON, "geometry");
- typeToName.put(Type.MULTIPOLYGON, "geometry");
- typeToName.put(Type.GEOMETRYCOLLECTION, "geometry");
- typeToName.put(Type.ENVELOPE, "geometry");
- typeToName.put(Type.INET_ADDRESS, "inet");
- typeToName.put(Type.INET4_ADDRESS, "inet");
- typeToName.put(Type.INET6_ADDRESS, "inet");
- typeToName.put(Type.LOCAL_DATE, "date");
- typeToName.put(Type.LOCAL_TIME, "time");
- typeToName.put(Type.LOCAL_DATE_TIME, "timestamp");
- typeToName.put(Type.NESTED, "jsonb");
+ typeToName.put(ColumnType.STRING, "varchar");
+ typeToName.put(ColumnType.SHORT, "int2");
+ typeToName.put(ColumnType.INTEGER, "int4");
+ typeToName.put(ColumnType.LONG, "int8");
+ typeToName.put(ColumnType.FLOAT, "float4");
+ typeToName.put(ColumnType.DOUBLE, "float8");
+ typeToName.put(ColumnType.GEOMETRY, "geometry");
+ typeToName.put(ColumnType.POINT, "geometry");
+ typeToName.put(ColumnType.MULTIPOINT, "geometry");
+ typeToName.put(ColumnType.LINESTRING, "geometry");
+ typeToName.put(ColumnType.MULTILINESTRING, "geometry");
+ typeToName.put(ColumnType.POLYGON, "geometry");
+ typeToName.put(ColumnType.MULTIPOLYGON, "geometry");
+ typeToName.put(ColumnType.GEOMETRYCOLLECTION, "geometry");
+ typeToName.put(ColumnType.ENVELOPE, "geometry");
+ typeToName.put(ColumnType.INET_ADDRESS, "inet");
+ typeToName.put(ColumnType.INET4_ADDRESS, "inet");
+ typeToName.put(ColumnType.INET6_ADDRESS, "inet");
+ typeToName.put(ColumnType.LOCAL_DATE, "date");
+ typeToName.put(ColumnType.LOCAL_TIME, "time");
+ typeToName.put(ColumnType.LOCAL_DATE_TIME, "timestamp");
+ typeToName.put(ColumnType.NESTED, "jsonb");
}
- protected static final Map<String, Type> nameToType = Map.ofEntries(
- Map.entry("varchar", Type.STRING),
- Map.entry("int2", Type.SHORT),
- Map.entry("int4", Type.INTEGER),
- Map.entry("int8", Type.LONG),
- Map.entry("float4", Type.FLOAT),
- Map.entry("float8", Type.DOUBLE),
- Map.entry("geometry", Type.GEOMETRY),
- Map.entry("inet", Type.INET6_ADDRESS),
- Map.entry("date", Type.LOCAL_DATE),
- Map.entry("time", Type.LOCAL_TIME),
- Map.entry("timestamp", Type.LOCAL_DATE_TIME),
- Map.entry("jsonb", Type.NESTED));
+ protected static final Map<String, ColumnType> nameToType = Map.ofEntries(
+ Map.entry("varchar", ColumnType.STRING),
+ Map.entry("int2", ColumnType.SHORT),
+ Map.entry("int4", ColumnType.INTEGER),
+ Map.entry("int8", ColumnType.LONG),
+ Map.entry("float4", ColumnType.FLOAT),
+ Map.entry("float8", ColumnType.DOUBLE),
+ Map.entry("geometry", ColumnType.GEOMETRY),
+ Map.entry("inet", ColumnType.INET6_ADDRESS),
+ Map.entry("date", ColumnType.LOCAL_DATE),
+ Map.entry("time", ColumnType.LOCAL_TIME),
+ Map.entry("timestamp", ColumnType.LOCAL_DATE_TIME),
+ Map.entry("jsonb", ColumnType.NESTED));
}
diff --git a/baremaps-postgres/src/test/java/org/apache/baremaps/postgres/MockDataTable.java b/baremaps-postgres/src/test/java/org/apache/baremaps/postgres/MockDataTable.java
index 797f5608e..668aaa819 100644
--- a/baremaps-postgres/src/test/java/org/apache/baremaps/postgres/MockDataTable.java
+++ b/baremaps-postgres/src/test/java/org/apache/baremaps/postgres/MockDataTable.java
@@ -22,7 +22,7 @@ import java.util.Iterator;
import java.util.List;
import org.apache.baremaps.store.*;
import org.apache.baremaps.store.DataColumn.Cardinality;
-import org.apache.baremaps.store.DataColumn.Type;
+import org.apache.baremaps.store.DataColumn.ColumnType;
import org.locationtech.jts.geom.Coordinate;
public class MockDataTable implements DataTable {
@@ -33,11 +33,11 @@ public class MockDataTable implements DataTable {
public MockDataTable() {
this.rowType = new DataSchemaImpl("mock", List.of(
- new DataColumnFixed("string", Cardinality.OPTIONAL, Type.STRING),
- new DataColumnFixed("integer", Cardinality.OPTIONAL, Type.INTEGER),
- new DataColumnFixed("double", Cardinality.OPTIONAL, Type.DOUBLE),
- new DataColumnFixed("float", Cardinality.OPTIONAL, Type.FLOAT),
- new DataColumnFixed("geometry", Cardinality.OPTIONAL, Type.GEOMETRY)));
+ new DataColumnFixed("string", Cardinality.OPTIONAL, ColumnType.STRING),
+ new DataColumnFixed("integer", Cardinality.OPTIONAL, ColumnType.INTEGER),
+ new DataColumnFixed("double", Cardinality.OPTIONAL, ColumnType.DOUBLE),
+ new DataColumnFixed("float", Cardinality.OPTIONAL, ColumnType.FLOAT),
+ new DataColumnFixed("geometry", Cardinality.OPTIONAL, ColumnType.GEOMETRY)));
this.rows = List.of(
new DataRowImpl(rowType,
List.of("string", 1, 1.0, 1.0f,
@@ -75,4 +75,9 @@ public class MockDataTable implements DataTable {
public DataSchema schema() {
return rowType;
}
+
+ @Override
+ public void close() throws Exception {
+ // Do nothing
+ }
}
diff --git a/baremaps-rpsl/src/main/java/org/apache/baremaps/rpsl/RpslDataTable.java b/baremaps-rpsl/src/main/java/org/apache/baremaps/rpsl/RpslDataTable.java
index 9bbfeaae7..b4f1b3a1e 100644
--- a/baremaps-rpsl/src/main/java/org/apache/baremaps/rpsl/RpslDataTable.java
+++ b/baremaps-rpsl/src/main/java/org/apache/baremaps/rpsl/RpslDataTable.java
@@ -22,7 +22,7 @@ import java.io.InputStream;
import java.util.*;
import org.apache.baremaps.store.*;
import org.apache.baremaps.store.DataColumn.Cardinality;
-import org.apache.baremaps.store.DataColumn.Type;
+import org.apache.baremaps.store.DataColumn.ColumnType;
/**
* A DataTable implementation that reads RPSL data using RpslReader.
@@ -49,20 +49,20 @@ public class RpslDataTable implements DataTable {
*/
private DataSchema createSchema() {
List<DataColumn> columns = new ArrayList<>();
- columns.add(new DataColumnFixed("type", Cardinality.REQUIRED, Type.STRING));
- columns.add(new DataColumnFixed("id", Cardinality.REQUIRED, Type.STRING));
- columns.add(new DataColumnFixed("inetnum", Cardinality.OPTIONAL, Type.STRING));
- columns.add(new DataColumnFixed("inet6num", Cardinality.OPTIONAL, Type.STRING));
- columns.add(new DataColumnFixed("netname", Cardinality.OPTIONAL, Type.STRING));
- columns.add(new DataColumnFixed("descr", Cardinality.REPEATED, Type.STRING));
- columns.add(new DataColumnFixed("country", Cardinality.OPTIONAL, Type.STRING));
- columns.add(new DataColumnFixed("admin-c", Cardinality.OPTIONAL, Type.STRING));
- columns.add(new DataColumnFixed("tech-c", Cardinality.OPTIONAL, Type.STRING));
- columns.add(new DataColumnFixed("status", Cardinality.OPTIONAL, Type.STRING));
- columns.add(new DataColumnFixed("mnt-by", Cardinality.OPTIONAL, Type.STRING));
- columns.add(new DataColumnFixed("created", Cardinality.OPTIONAL, Type.STRING));
- columns.add(new DataColumnFixed("last-modified", Cardinality.OPTIONAL, Type.STRING));
- columns.add(new DataColumnFixed("changed", Cardinality.REPEATED, Type.STRING));
+ columns.add(new DataColumnFixed("type", Cardinality.REQUIRED, ColumnType.STRING));
+ columns.add(new DataColumnFixed("id", Cardinality.REQUIRED, ColumnType.STRING));
+ columns.add(new DataColumnFixed("inetnum", Cardinality.OPTIONAL, ColumnType.STRING));
+ columns.add(new DataColumnFixed("inet6num", Cardinality.OPTIONAL, ColumnType.STRING));
+ columns.add(new DataColumnFixed("netname", Cardinality.OPTIONAL, ColumnType.STRING));
+ columns.add(new DataColumnFixed("descr", Cardinality.REPEATED, ColumnType.STRING));
+ columns.add(new DataColumnFixed("country", Cardinality.OPTIONAL, ColumnType.STRING));
+ columns.add(new DataColumnFixed("admin-c", Cardinality.OPTIONAL, ColumnType.STRING));
+ columns.add(new DataColumnFixed("tech-c", Cardinality.OPTIONAL, ColumnType.STRING));
+ columns.add(new DataColumnFixed("status", Cardinality.OPTIONAL, ColumnType.STRING));
+ columns.add(new DataColumnFixed("mnt-by", Cardinality.OPTIONAL, ColumnType.STRING));
+ columns.add(new DataColumnFixed("created", Cardinality.OPTIONAL, ColumnType.STRING));
+ columns.add(new DataColumnFixed("last-modified", Cardinality.OPTIONAL, ColumnType.STRING));
+ columns.add(new DataColumnFixed("changed", Cardinality.REPEATED, ColumnType.STRING));
return new DataSchemaImpl("RpslObject", columns);
}
@@ -86,4 +86,9 @@ public class RpslDataTable implements DataTable {
public void clear() {
throw new UnsupportedOperationException("Clear operation is not supported.");
}
+
+ @Override
+ public void close() throws Exception {
+ inputStream.close();
+ }
}
diff --git a/baremaps-shapefile/src/main/java/org/apache/baremaps/shapefile/ShapefileByteReader.java b/baremaps-shapefile/src/main/java/org/apache/baremaps/shapefile/ShapefileByteReader.java
index 68b017a28..ac6de721a 100644
--- a/baremaps-shapefile/src/main/java/org/apache/baremaps/shapefile/ShapefileByteReader.java
+++ b/baremaps-shapefile/src/main/java/org/apache/baremaps/shapefile/ShapefileByteReader.java
@@ -26,7 +26,7 @@ import java.nio.channels.FileChannel;
import java.util.*;
import org.apache.baremaps.store.*;
import org.apache.baremaps.store.DataColumn.Cardinality;
-import org.apache.baremaps.store.DataColumn.Type;
+import org.apache.baremaps.store.DataColumn.ColumnType;
import org.locationtech.jts.algorithm.Orientation;
import org.locationtech.jts.geom.Coordinate;
import org.locationtech.jts.geom.CoordinateList;
@@ -131,29 +131,29 @@ public class ShapefileByteReader extends CommonByteReader {
var fieldDescriptor = this.databaseFieldsDescriptors.get(i);
var columnName = fieldDescriptor.getName();
var columnType = switch (fieldDescriptor.getType()) {
- case CHARACTER -> Type.STRING;
- case NUMBER -> fieldDescriptor.getDecimalCount() == 0 ? Type.LONG : Type.DOUBLE;
- case CURRENCY -> Type.DOUBLE;
- case DOUBLE -> Type.DOUBLE;
- case INTEGER -> Type.INTEGER;
- case AUTO_INCREMENT -> Type.INTEGER;
+ case CHARACTER -> ColumnType.STRING;
+ case NUMBER -> fieldDescriptor.getDecimalCount() == 0 ? ColumnType.LONG : ColumnType.DOUBLE;
+ case CURRENCY -> ColumnType.DOUBLE;
+ case DOUBLE -> ColumnType.DOUBLE;
+ case INTEGER -> ColumnType.INTEGER;
+ case AUTO_INCREMENT -> ColumnType.INTEGER;
// TODO: Implement the following types
- case LOGICAL -> Type.STRING;
- case DATE -> Type.STRING;
- case MEMO -> Type.STRING;
- case FLOATING_POINT -> Type.STRING;
- case PICTURE -> Type.STRING;
- case VARI_FIELD -> Type.STRING;
- case VARIANT -> Type.STRING;
- case TIMESTAMP -> Type.STRING;
- case DATE_TIME -> Type.STRING;
+ case LOGICAL -> ColumnType.STRING;
+ case DATE -> ColumnType.STRING;
+ case MEMO -> ColumnType.STRING;
+ case FLOATING_POINT -> ColumnType.STRING;
+ case PICTURE -> ColumnType.STRING;
+ case VARI_FIELD -> ColumnType.STRING;
+ case VARIANT -> ColumnType.STRING;
+ case TIMESTAMP -> ColumnType.STRING;
+ case DATE_TIME -> ColumnType.STRING;
};
columns.add(new DataColumnFixed(columnName, Cardinality.OPTIONAL, columnType));
}
// Add geometry column.
- columns.add(new DataColumnFixed(GEOMETRY_NAME, Cardinality.OPTIONAL, Type.GEOMETRY));
+ columns.add(new DataColumnFixed(GEOMETRY_NAME, Cardinality.OPTIONAL, ColumnType.GEOMETRY));
return new DataSchemaImpl(name, columns);
}
diff --git a/baremaps-shapefile/src/main/java/org/apache/baremaps/shapefile/ShapefileDataTable.java b/baremaps-shapefile/src/main/java/org/apache/baremaps/shapefile/ShapefileDataTable.java
index f62258dfe..8822bb921 100644
--- a/baremaps-shapefile/src/main/java/org/apache/baremaps/shapefile/ShapefileDataTable.java
+++ b/baremaps-shapefile/src/main/java/org/apache/baremaps/shapefile/ShapefileDataTable.java
@@ -38,6 +38,8 @@ public class ShapefileDataTable implements DataTable {
private final ShapefileReader shapeFile;
+ private ShapefileIterator iterator;
+
/**
* Constructs a table from a shapefile.
*
@@ -73,7 +75,7 @@ public class ShapefileDataTable implements DataTable {
@Override
public Iterator<DataRow> iterator() {
try {
- return new ShapefileIterator(shapeFile.read());
+ return (iterator = new ShapefileIterator(shapeFile));
} catch (IOException e) {
throw new DataStoreException(e);
}
@@ -87,10 +89,17 @@ public class ShapefileDataTable implements DataTable {
throw new UnsupportedOperationException();
}
+ @Override
+ public void close() throws Exception {
+ if (iterator != null) {
+ iterator.close();
+ }
+ }
+
/**
* An iterator over the rows of a shapefile.
*/
- public static class ShapefileIterator implements Iterator<DataRow> {
+ public static class ShapefileIterator implements Iterator<DataRow>, AutoCloseable {
private final ShapefileInputStream shapefileInputStream;
@@ -99,10 +108,10 @@ public class ShapefileDataTable implements DataTable {
/**
* Constructs an iterator from a shapefile input stream.
*
- * @param shapefileInputStream the shapefile input stream
+ * @param shapefileReader the shapefile input stream
*/
- public ShapefileIterator(ShapefileInputStream shapefileInputStream) {
- this.shapefileInputStream = shapefileInputStream;
+ public ShapefileIterator(ShapefileReader shapefileReader) throws IOException {
+ this.shapefileInputStream = shapefileReader.read();
}
/**
@@ -147,5 +156,10 @@ public class ShapefileDataTable implements DataTable {
throw new NoSuchElementException();
}
}
+
+ @Override
+ public void close() throws Exception {
+ shapefileInputStream.close();
+ }
}
}
diff --git a/baremaps-shapefile/src/main/java/org/apache/baremaps/shapefile/ShapefileReader.java b/baremaps-shapefile/src/main/java/org/apache/baremaps/shapefile/ShapefileReader.java
index 3a796befa..777411241 100644
--- a/baremaps-shapefile/src/main/java/org/apache/baremaps/shapefile/ShapefileReader.java
+++ b/baremaps-shapefile/src/main/java/org/apache/baremaps/shapefile/ShapefileReader.java
@@ -190,7 +190,6 @@ public class ShapefileReader {
* this is also automatically done when executing a query on it, by findAll.
*/
public void loadDescriptors() throws IOException {
-
try (ShapefileInputStream is = read()) {
// Doing a read is sufficient to initialize the internal descriptors.
}
diff --git a/baremaps-store/pom.xml b/baremaps-store/pom.xml
index 4135f4376..bce86c497 100644
--- a/baremaps-store/pom.xml
+++ b/baremaps-store/pom.xml
@@ -24,6 +24,16 @@ limitations under the License.
</parent>
<artifactId>baremaps-store</artifactId>
<dependencies>
+ <dependency>
+ <groupId>com.fasterxml.jackson.core</groupId>
+ <artifactId>jackson-core</artifactId>
+ <version>${version.lib.jackson}</version>
+ </dependency>
+ <dependency>
+ <groupId>com.fasterxml.jackson.core</groupId>
+ <artifactId>jackson-databind</artifactId>
+ <version>${version.lib.jackson}</version>
+ </dependency>
<dependency>
<groupId>org.locationtech.jts</groupId>
<artifactId>jts-core</artifactId>
diff --git a/baremaps-store/src/main/java/org/apache/baremaps/store/DataColumn.java b/baremaps-store/src/main/java/org/apache/baremaps/store/DataColumn.java
index 11c5d3567..609674eb1 100644
--- a/baremaps-store/src/main/java/org/apache/baremaps/store/DataColumn.java
+++ b/baremaps-store/src/main/java/org/apache/baremaps/store/DataColumn.java
@@ -17,6 +17,7 @@
package org.apache.baremaps.store;
+import java.io.Serializable;
import java.net.Inet4Address;
import java.net.Inet6Address;
import java.net.InetAddress;
@@ -29,7 +30,7 @@ import org.locationtech.jts.geom.*;
/**
* A column in a table.
*/
-public interface DataColumn {
+public interface DataColumn extends Serializable {
/**
* Returns the name of the column.
@@ -38,7 +39,6 @@ public interface DataColumn {
*/
String name();
-
Cardinality cardinality();
enum Cardinality {
@@ -52,12 +52,12 @@ public interface DataColumn {
*
* @return the type of the column
*/
- Type type();
+ ColumnType type();
/**
* An enumeration of the supported data column types.
*/
- enum Type {
+ enum ColumnType {
BINARY(byte[].class),
BYTE(Byte.class),
BOOLEAN(Boolean.class),
@@ -87,7 +87,7 @@ public interface DataColumn {
private final Class<?> binding;
- Type(Class<?> binding) {
+ ColumnType(Class<?> binding) {
this.binding = binding;
}
@@ -95,8 +95,8 @@ public interface DataColumn {
return binding;
}
- public static Type fromBinding(Class<?> binding) {
- for (Type type : Type.values()) {
+ public static ColumnType fromBinding(Class<?> binding) {
+ for (ColumnType type : ColumnType.values()) {
if (type.binding().equals(binding)) {
return type;
}
diff --git a/baremaps-store/src/main/java/org/apache/baremaps/store/DataColumnFixed.java b/baremaps-store/src/main/java/org/apache/baremaps/store/DataColumnFixed.java
index e8e6cdd88..2f8990e9c 100644
--- a/baremaps-store/src/main/java/org/apache/baremaps/store/DataColumnFixed.java
+++ b/baremaps-store/src/main/java/org/apache/baremaps/store/DataColumnFixed.java
@@ -17,12 +17,13 @@
package org.apache.baremaps.store;
+
/**
* A column in a table.
*/
public record DataColumnFixed(
String name,
Cardinality cardinality,
- Type type) implements DataColumn {
+ ColumnType type) implements DataColumn {
}
diff --git a/baremaps-store/src/main/java/org/apache/baremaps/store/DataColumnNested.java b/baremaps-store/src/main/java/org/apache/baremaps/store/DataColumnNested.java
index 176f0c626..bcf3458c1 100644
--- a/baremaps-store/src/main/java/org/apache/baremaps/store/DataColumnNested.java
+++ b/baremaps-store/src/main/java/org/apache/baremaps/store/DataColumnNested.java
@@ -25,7 +25,7 @@ public record DataColumnNested(
List<DataColumn> columns) implements DataColumn {
@Override
- public Type type() {
- return Type.NESTED;
+ public ColumnType type() {
+ return ColumnType.NESTED;
}
}
diff --git a/baremaps-store/src/main/java/org/apache/baremaps/store/DataSchema.java b/baremaps-store/src/main/java/org/apache/baremaps/store/DataSchema.java
index 48382aeb8..aef70f1e7 100644
--- a/baremaps-store/src/main/java/org/apache/baremaps/store/DataSchema.java
+++ b/baremaps-store/src/main/java/org/apache/baremaps/store/DataSchema.java
@@ -17,32 +17,97 @@
package org.apache.baremaps.store;
+import com.fasterxml.jackson.core.JsonParser;
+import com.fasterxml.jackson.databind.DeserializationContext;
+import com.fasterxml.jackson.databind.JsonDeserializer;
+import com.fasterxml.jackson.databind.JsonNode;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.fasterxml.jackson.databind.jsontype.NamedType;
+import com.fasterxml.jackson.databind.module.SimpleModule;
+import com.fasterxml.jackson.databind.node.ObjectNode;
+import java.io.IOException;
+import java.io.Serializable;
+import java.nio.file.Path;
+import java.util.ArrayList;
import java.util.List;
+import org.apache.baremaps.store.DataColumn.Cardinality;
+import org.apache.baremaps.store.DataColumn.ColumnType;
/**
* A {@link DataSchema} is a description of the structure of a row in a {@link DataTable}.
*/
-public interface DataSchema {
+public interface DataSchema extends Serializable {
/**
* Returns the name of the schema.
- *
+ *
* @return the name of the schema
*/
String name();
/**
* Returns the columns of the schema.
- *
+ *
* @return the columns of the schema
*/
List<DataColumn> columns();
/**
* Creates a new row of the schema.
- *
+ *
* @return a new row of the schema
*/
DataRow createRow();
+ class DataSchemaDeserializer extends JsonDeserializer<DataSchema> {
+
+ @Override
+ public DataSchema deserialize(JsonParser parser, DeserializationContext ctxt)
+ throws IOException {
+ ObjectNode node = parser.getCodec().readTree(parser);
+ String name = node.get("name").asText();
+ List<DataColumn> columns = new ArrayList<>();
+ node.get("columns").elements().forEachRemaining(column -> {
+ columns.add(deserialize(column));
+ });
+ return new DataSchemaImpl(name, columns);
+ }
+
+ DataColumn deserialize(JsonNode node) {
+ String columnName = node.get("name").asText();
+ Cardinality cardinality = Cardinality.valueOf(node.get("cardinality").asText());
+ ColumnType type = ColumnType.valueOf(node.get("type").asText());
+ if (type == ColumnType.NESTED) {
+ List<DataColumn> columns = new ArrayList<>();
+ node.get("columns").elements().forEachRemaining(column -> {
+ columns.add(deserialize(column));
+ });
+ return new DataColumnNested(columnName, cardinality, columns);
+ } else {
+ return new DataColumnFixed(columnName, cardinality, type);
+ }
+ }
+ }
+
+ private static ObjectMapper configureObjectMapper() {
+ var mapper = new ObjectMapper();
+ mapper.registerSubtypes(
+ new NamedType(DataColumnFixed.class, "FIXED"),
+ new NamedType(DataColumnNested.class, "NESTED"));
+ var module = new SimpleModule();
+ module.addDeserializer(DataSchema.class, new DataSchemaDeserializer());
+ mapper.registerModule(module);
+ return mapper;
+ }
+
+ static DataSchema read(Path path) throws IOException {
+ var mapper = configureObjectMapper();
+ return mapper.readValue(path.toFile(), DataSchema.class);
+ }
+
+ static void write(Path path, DataSchema schema) throws IOException {
+ var mapper = configureObjectMapper();
+ mapper.writerWithDefaultPrettyPrinter().writeValue(path.toFile(), schema);
+ }
+
}
diff --git a/baremaps-store/src/main/java/org/apache/baremaps/store/DataSchemaImpl.java b/baremaps-store/src/main/java/org/apache/baremaps/store/DataSchemaImpl.java
index 7c5ad1612..302dc7c00 100644
--- a/baremaps-store/src/main/java/org/apache/baremaps/store/DataSchemaImpl.java
+++ b/baremaps-store/src/main/java/org/apache/baremaps/store/DataSchemaImpl.java
@@ -17,13 +17,15 @@
package org.apache.baremaps.store;
+import java.io.Serializable;
import java.util.ArrayList;
import java.util.List;
/**
* A {@link DataSchema} defines the structure of a table.
*/
-public record DataSchemaImpl(String name, List<DataColumn> columns) implements DataSchema {
+public record DataSchemaImpl(String name,
+ List<DataColumn> columns) implements DataSchema, Serializable {
/**
* {@inheritDoc}
diff --git a/baremaps-store/src/main/java/org/apache/baremaps/store/DataTable.java b/baremaps-store/src/main/java/org/apache/baremaps/store/DataTable.java
index 11749f00c..a270f4d6b 100644
--- a/baremaps-store/src/main/java/org/apache/baremaps/store/DataTable.java
+++ b/baremaps-store/src/main/java/org/apache/baremaps/store/DataTable.java
@@ -26,7 +26,7 @@ import java.util.stream.StreamSupport;
/**
* A {@link DataTable} is a collection of rows respecting a {@link DataSchema} .
*/
-public interface DataTable extends Iterable<DataRow> {
+public interface DataTable extends Iterable<DataRow>, AutoCloseable {
/**
* Returns the schema of the row.
diff --git a/baremaps-store/src/main/java/org/apache/baremaps/store/DataTableMapper.java b/baremaps-store/src/main/java/org/apache/baremaps/store/DataTableMapper.java
index 429ff18c4..a9e2ea151 100644
--- a/baremaps-store/src/main/java/org/apache/baremaps/store/DataTableMapper.java
+++ b/baremaps-store/src/main/java/org/apache/baremaps/store/DataTableMapper.java
@@ -68,4 +68,8 @@ public class DataTableMapper implements DataTable {
table.clear();
}
+ @Override
+ public void close() throws Exception {
+ table.close();
+ }
}
diff --git a/pom.xml b/pom.xml
index edb9af60a..65856f741 100644
--- a/pom.xml
+++ b/pom.xml
@@ -48,6 +48,7 @@ limitations under the License.
<modules>
<module>baremaps-benchmarking</module>
+ <module>baremaps-calcite</module>
<module>baremaps-cli</module>
<module>baremaps-core</module>
<module>baremaps-csv</module>
(incubator-baremaps) 04/07: Improve GeoPackage store
Posted by bc...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
bchapuis pushed a commit to branch calcite-framework
in repository https://gitbox.apache.org/repos/asf/incubator-baremaps.git
commit fa97d3cb045bfa9d3b16d2d13f9e484b56a43220
Author: Bertil Chapuis <bc...@gmail.com>
AuthorDate: Sat Jan 18 14:09:51 2025 +0100
Improve GeoPackage store
---
.../baremaps/geopackage/GeoPackageDataStore.java | 22 +++++++++-------------
.../baremaps/geopackage/GeoPackageDataTable.java | 20 ++++++++++++++++----
2 files changed, 25 insertions(+), 17 deletions(-)
diff --git a/baremaps-geopackage/src/main/java/org/apache/baremaps/geopackage/GeoPackageDataStore.java b/baremaps-geopackage/src/main/java/org/apache/baremaps/geopackage/GeoPackageDataStore.java
index 74290917a..71ddb3fb2 100644
--- a/baremaps-geopackage/src/main/java/org/apache/baremaps/geopackage/GeoPackageDataStore.java
+++ b/baremaps-geopackage/src/main/java/org/apache/baremaps/geopackage/GeoPackageDataStore.java
@@ -29,9 +29,9 @@ import org.apache.baremaps.store.DataTable;
/**
* A {@link DataStore} corresponding to a GeoPackage file.
*/
-public class GeoPackageDataStore implements DataStore, AutoCloseable {
+public class GeoPackageDataStore implements DataStore {
- private final GeoPackage geoPackage;
+ private final Path file;
/**
* Constructs a {@link GeoPackageDataStore} from a GeoPackage file.
@@ -39,15 +39,7 @@ public class GeoPackageDataStore implements DataStore, AutoCloseable {
* @param file the path to the GeoPackage file
*/
public GeoPackageDataStore(Path file) {
- this.geoPackage = GeoPackageManager.open(file.toFile());
- }
-
- /**
- * {@inheritDoc}
- */
- @Override
- public void close() throws Exception {
- geoPackage.close();
+ this.file = file;
}
/**
@@ -55,7 +47,11 @@ public class GeoPackageDataStore implements DataStore, AutoCloseable {
*/
@Override
public List<String> list() throws DataStoreException {
- return geoPackage.getFeatureTables();
+ try (GeoPackage geoPackage = GeoPackageManager.open(file.toFile())) {
+ return geoPackage.getFeatureTables();
+ } catch (Exception e) {
+ throw new DataStoreException(e);
+ }
}
/**
@@ -63,7 +59,7 @@ public class GeoPackageDataStore implements DataStore, AutoCloseable {
*/
@Override
public DataTable get(String name) throws DataStoreException {
- return new GeoPackageDataTable(geoPackage.getFeatureDao(name));
+ return new GeoPackageDataTable(file, name);
}
/**
diff --git a/baremaps-geopackage/src/main/java/org/apache/baremaps/geopackage/GeoPackageDataTable.java b/baremaps-geopackage/src/main/java/org/apache/baremaps/geopackage/GeoPackageDataTable.java
index 8ea223665..59e91c180 100644
--- a/baremaps-geopackage/src/main/java/org/apache/baremaps/geopackage/GeoPackageDataTable.java
+++ b/baremaps-geopackage/src/main/java/org/apache/baremaps/geopackage/GeoPackageDataTable.java
@@ -18,7 +18,10 @@
package org.apache.baremaps.geopackage;
+import java.nio.file.Path;
import java.util.*;
+import mil.nga.geopackage.GeoPackage;
+import mil.nga.geopackage.GeoPackageManager;
import mil.nga.geopackage.features.user.FeatureColumn;
import mil.nga.geopackage.features.user.FeatureDao;
import mil.nga.geopackage.features.user.FeatureResultSet;
@@ -33,6 +36,12 @@ import org.locationtech.jts.geom.*;
*/
public class GeoPackageDataTable implements DataTable {
+ private final Path file;
+
+ private final String name;
+
+ private final GeoPackage geoPackage;
+
private final FeatureDao featureDao;
private final DataSchema schema;
@@ -42,11 +51,14 @@ public class GeoPackageDataTable implements DataTable {
/**
* Constructs a {@link DataTable} from a feature DAO.
*
- * @param featureDao the feature DAO
+ * @param file the path to the GeoPackage file
+ * @param name the name of the data table
*/
- public GeoPackageDataTable(FeatureDao featureDao) {
- this.featureDao = featureDao;
- var name = featureDao.getTableName();
+ public GeoPackageDataTable(Path file, String name) {
+ this.file = file;
+ this.name = name;
+ this.geoPackage = GeoPackageManager.open(file.toFile());
+ this.featureDao = geoPackage.getFeatureDao(name);
var columns = new ArrayList<DataColumn>();
for (FeatureColumn column : featureDao.getColumns()) {
var propertyName = column.getName();
(incubator-baremaps) 05/07: Make data collection closeable
Posted by bc...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
bchapuis pushed a commit to branch calcite-framework
in repository https://gitbox.apache.org/repos/asf/incubator-baremaps.git
commit d6b161c898121b301f9b526b7121b721a04862d2
Author: Bertil Chapuis <bc...@gmail.com>
AuthorDate: Sat Jan 18 14:27:09 2025 +0100
Make data collection closeable
---
.../apache/baremaps/data/collection/AppendOnlyLog.java | 5 +++++
.../apache/baremaps/data/collection/DataCollection.java | 3 ++-
.../baremaps/data/collection/DataCollectionMapper.java | 6 ++++++
.../apache/baremaps/data/collection/DataConversions.java | 16 ++++++++++++++++
4 files changed, 29 insertions(+), 1 deletion(-)
diff --git a/baremaps-data/src/main/java/org/apache/baremaps/data/collection/AppendOnlyLog.java b/baremaps-data/src/main/java/org/apache/baremaps/data/collection/AppendOnlyLog.java
index fbb549988..1a9830b1b 100644
--- a/baremaps-data/src/main/java/org/apache/baremaps/data/collection/AppendOnlyLog.java
+++ b/baremaps-data/src/main/java/org/apache/baremaps/data/collection/AppendOnlyLog.java
@@ -146,6 +146,11 @@ public class AppendOnlyLog<E> implements DataCollection<E> {
return new AppendOnlyLogIterator(size);
}
+ @Override
+ public void close() throws IOException {
+ memory.close();
+ }
+
/**
* An iterator over the values of the log that can be used to iterate over the values of the log
* and to get the current position in the memory.
diff --git a/baremaps-data/src/main/java/org/apache/baremaps/data/collection/DataCollection.java b/baremaps-data/src/main/java/org/apache/baremaps/data/collection/DataCollection.java
index 1189cd846..3d5924058 100644
--- a/baremaps-data/src/main/java/org/apache/baremaps/data/collection/DataCollection.java
+++ b/baremaps-data/src/main/java/org/apache/baremaps/data/collection/DataCollection.java
@@ -17,6 +17,7 @@
package org.apache.baremaps.data.collection;
+import java.io.Closeable;
import java.util.*;
import java.util.stream.Stream;
import java.util.stream.StreamSupport;
@@ -25,7 +26,7 @@ import java.util.stream.StreamSupport;
* A {@code DataCollection<E>} is a group of elements that can be iterated over. It is similar to a
* {@link java.util.Collection<E> Collection}, but can hold up to {@link Long#MAX_VALUE} elements.
*/
-public interface DataCollection<E> extends Iterable<E> {
+public interface DataCollection<E> extends Iterable<E>, Closeable {
/**
* Returns the number of values stored in the data collection.
diff --git a/baremaps-data/src/main/java/org/apache/baremaps/data/collection/DataCollectionMapper.java b/baremaps-data/src/main/java/org/apache/baremaps/data/collection/DataCollectionMapper.java
index 970407919..355a76eb2 100644
--- a/baremaps-data/src/main/java/org/apache/baremaps/data/collection/DataCollectionMapper.java
+++ b/baremaps-data/src/main/java/org/apache/baremaps/data/collection/DataCollectionMapper.java
@@ -18,6 +18,7 @@
package org.apache.baremaps.data.collection;
+import java.io.IOException;
import java.util.Iterator;
import java.util.function.Function;
@@ -58,4 +59,9 @@ public class DataCollectionMapper<S, T> implements DataCollection<T> {
public void clear() {
collection.clear();
}
+
+ @Override
+ public void close() throws IOException {
+ collection.close();
+ }
}
diff --git a/baremaps-data/src/main/java/org/apache/baremaps/data/collection/DataConversions.java b/baremaps-data/src/main/java/org/apache/baremaps/data/collection/DataConversions.java
index 593f6acd4..75d5a1a51 100644
--- a/baremaps-data/src/main/java/org/apache/baremaps/data/collection/DataConversions.java
+++ b/baremaps-data/src/main/java/org/apache/baremaps/data/collection/DataConversions.java
@@ -17,6 +17,8 @@
package org.apache.baremaps.data.collection;
+import java.io.Closeable;
+import java.io.IOException;
import java.util.*;
import java.util.Map.Entry;
@@ -163,6 +165,13 @@ public class DataConversions {
public Iterator<E> iterator() {
return collection.iterator();
}
+
+ @Override
+ public void close() throws IOException {
+ if (collection instanceof Closeable closeable) {
+ closeable.close();
+ }
+ }
}
private static class ListAdapter<E> extends AbstractList<E> {
@@ -251,6 +260,13 @@ public class DataConversions {
public int hashCode() {
return list.hashCode();
}
+
+ @Override
+ public void close() throws IOException {
+ if (list instanceof Closeable closeable) {
+ closeable.close();
+ }
+ }
}
private static class MapAdapter<K, V> extends AbstractMap<K, V> {
(incubator-baremaps) 02/07: Add list
Posted by bc...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
bchapuis pushed a commit to branch calcite-framework
in repository https://gitbox.apache.org/repos/asf/incubator-baremaps.git
commit 14d2eb9ddeae793655cecf7368c760f027b0ac66
Author: Bertil Chapuis <bc...@gmail.com>
AuthorDate: Tue Jan 14 21:08:02 2025 +0100
Add list
---
.../apache/baremaps/data/calcite/CalciteTest.java | 157 +++++++++++++++++----
1 file changed, 130 insertions(+), 27 deletions(-)
diff --git a/baremaps-data/src/test/java/org/apache/baremaps/data/calcite/CalciteTest.java b/baremaps-data/src/test/java/org/apache/baremaps/data/calcite/CalciteTest.java
index 52768a70d..9e7fc96d6 100644
--- a/baremaps-data/src/test/java/org/apache/baremaps/data/calcite/CalciteTest.java
+++ b/baremaps-data/src/test/java/org/apache/baremaps/data/calcite/CalciteTest.java
@@ -24,28 +24,56 @@ import org.apache.baremaps.data.type.RowDataType;
import org.apache.baremaps.store.DataColumn.Cardinality;
import org.apache.baremaps.store.DataColumn.Type;
import org.apache.baremaps.store.*;
+import org.apache.calcite.DataContext;
+import org.apache.calcite.DataContexts;
+import org.apache.calcite.adapter.enumerable.EnumerableConvention;
+import org.apache.calcite.adapter.enumerable.EnumerableInterpreter;
import org.apache.calcite.config.Lex;
+import org.apache.calcite.interpreter.Interpreter;
import org.apache.calcite.jdbc.CalciteConnection;
+import org.apache.calcite.linq4j.Enumerable;
+import org.apache.calcite.linq4j.Linq4j;
import org.apache.calcite.model.ModelHandler;
+import org.apache.calcite.plan.RelOptCluster;
+import org.apache.calcite.plan.RelOptPlanner;
+import org.apache.calcite.plan.RelTraitSet;
+import org.apache.calcite.plan.volcano.VolcanoPlanner;
+import org.apache.calcite.rel.RelNode;
+import org.apache.calcite.rel.type.RelDataType;
+import org.apache.calcite.rel.type.RelDataTypeFactory;
+import org.apache.calcite.rex.RexBuilder;
import org.apache.calcite.runtime.SpatialTypeFunctions;
+import org.apache.calcite.schema.ScannableTable;
import org.apache.calcite.schema.SchemaPlus;
-import org.apache.calcite.schema.impl.ViewTable;
-import org.apache.calcite.schema.impl.ViewTableMacro;
+import org.apache.calcite.schema.Table;
+import org.apache.calcite.schema.impl.*;
import org.apache.calcite.sql.SqlNode;
import org.apache.calcite.sql.SqlNodeList;
import org.apache.calcite.sql.fun.SqlSpatialTypeFunctions;
import org.apache.calcite.sql.parser.SqlParseException;
import org.apache.calcite.sql.parser.SqlParser;
import org.apache.calcite.sql.parser.ddl.SqlDdlParserImpl;
+import org.apache.calcite.sql.type.SqlTypeName;
import org.apache.calcite.sql.validate.SqlConformanceEnum;
+import org.apache.calcite.tools.FrameworkConfig;
+import org.apache.calcite.tools.Frameworks;
+import org.apache.calcite.tools.Planner;
import org.junit.jupiter.api.Test;
import org.locationtech.jts.geom.Coordinate;
+import org.locationtech.jts.geom.Geometry;
import org.locationtech.jts.geom.GeometryFactory;
+import org.apache.calcite.adapter.enumerable.EnumerableRel;
+import org.apache.calcite.adapter.enumerable.EnumerableInterpreter;
+import org.apache.calcite.adapter.enumerable.EnumerableRelImplementor;
+import org.apache.calcite.linq4j.Enumerator;
+import org.apache.calcite.plan.*;
+import org.apache.calcite.rel.RelNode;
+import org.apache.calcite.rel.type.RelDataType;
+import org.apache.calcite.schema.SchemaPlus;
+import org.apache.calcite.tools.*;
import java.sql.*;
-import java.util.Collections;
-import java.util.List;
-import java.util.Properties;
+import java.util.*;
public class CalciteTest {
@@ -102,13 +130,14 @@ public class CalciteTest {
"FROM city c " + // lowercase and unquoted
"JOIN population p ON c.id = p.city_id";
- ViewTableMacro materializedView = ViewTable.viewMacro(
+ ViewTableMacro materializedView = MaterializedViewTable.viewMacro(
rootSchema,
mvSql,
Collections.emptyList(), // Schema path
List.of("city_population"), // Name parts
false); // Not a materialized view
+
rootSchema.add("city_population", materializedView);
// Debug: List all tables in the root schema
@@ -121,7 +150,7 @@ public class CalciteTest {
try (Statement statement = connection.createStatement();
ResultSet resultSet = statement.executeQuery(sql)) {
while (resultSet.next()) {
- System.out.println(resultSet.getString("id") + " " + resultSet.getString("name"));
+ System.out.println(resultSet.getString("id") + " " + resultSet.getString("geometry"));
}
}
@@ -139,32 +168,106 @@ public class CalciteTest {
}
+ public class ListTable extends AbstractTable implements ScannableTable {
+ private final List<Integer> data;
+
+ public ListTable(List<Integer> data) {
+ this.data = data;
+ }
+
+ @Override
+ public RelDataType getRowType(RelDataTypeFactory typeFactory) {
+ // Define a single column named "value" of type INTEGER
+ return typeFactory.builder()
+ .add("V", SqlTypeName.INTEGER)
+ .build();
+ }
+
+ @Override
+ public Enumerable<Object[]> scan(DataContext root) {
+ // Convert the List<Integer> to Enumerable<Object[]>
+ return Linq4j.asEnumerable(data)
+ .select(i -> new Object[]{i});
+ }
+ }
+
+ public class ListSchema extends AbstractSchema {
+ private final List<Integer> listA;
+ private final List<Integer> listB;
+
+ public ListSchema(List<Integer> listA, List<Integer> listB) {
+ this.listA = listA;
+ this.listB = listB;
+ }
+
+ @Override
+ protected Map<String, Table> getTableMap() {
+ Map<String, Table> tables = new HashMap<>();
+ tables.put("LIST_A", new ListTable(listA));
+ tables.put("LIST_B", new ListTable(listB)); // Initially empty
+ return tables;
+ }
+ }
+
@Test
- void ddl() throws SqlParseException {
-
- // Example SQL script with multiple DDL statements
- String sqlScript = """
- CREATE MATERIALIZED VIEW IF NOT EXISTS my_view AS
- SELECT * FROM my_table;
- """;
-
- // Build a parser config that supports DDL
- SqlParser.Config config = SqlParser.configBuilder()
- .setParserFactory(SqlDdlParserImpl.FACTORY)
- .setConformance(SqlConformanceEnum.BABEL)
- .setLex(Lex.MYSQL)
+ void list() throws Exception {
+ // Initialize your Java lists
+ List<Integer> listA = List.of(1, 2, 3, 4, 5);
+ List<Integer> listB = new ArrayList<>();
+
+ // Set up Calcite schema
+ SchemaPlus rootSchema = Frameworks.createRootSchema(true);
+ rootSchema.add("MY_SCHEMA", new ListSchema(listA, listB));
+
+ // Create and add 'city' table
+ DataSchema cityRowType = new DataSchemaImpl("city", List.of(
+ new DataColumnFixed("id", Cardinality.OPTIONAL, Type.INTEGER),
+ new DataColumnFixed("name", Cardinality.OPTIONAL, Type.STRING),
+ new DataColumnFixed("geometry", Cardinality.OPTIONAL, Type.GEOMETRY)));
+
+ DataTable cityDataTable = new BaremapsDataTable(
+ cityRowType,
+ new IndexedDataList<>(new AppendOnlyLog<>(new RowDataType(cityRowType))));
+
+ GeometryFactory geometryFactory = new GeometryFactory();
+ cityDataTable.add(new DataRowImpl(cityDataTable.schema(),
+ List.of(1, "Paris", geometryFactory.createPoint(new Coordinate(2.3522, 48.8566)))));
+ cityDataTable.add(new DataRowImpl(cityDataTable.schema(),
+ List.of(2, "New York", geometryFactory.createPoint(new Coordinate(-74.0060, 40.7128)))));
+
+ SqlDataTable citySqlDataTable = new SqlDataTable(cityDataTable);
+ rootSchema.add("CITY", citySqlDataTable);
+
+ // Configure the framework
+ FrameworkConfig config = Frameworks.newConfigBuilder()
+ .defaultSchema(rootSchema.getSubSchema("MY_SCHEMA"))
.build();
- // Create the parser from the config
- SqlParser parser = SqlParser.create(sqlScript, config);
+ // Create a planner
+ Planner planner = Frameworks.getPlanner(config);
+
+ // Define the SQL query to populate list_b from list_a
+ String sql = "SELECT V * 2 AS V FROM LIST_A";
+
+ // Parse the SQL query
+ org.apache.calcite.sql.SqlNode parsed = planner.parse(sql);
- // Parse the script as a list of statements
- SqlNodeList sqlNodeList = parser.parseStmtList();
+ // Validate the SQL query
+ org.apache.calcite.sql.SqlNode validated = planner.validate(parsed);
- // Iterate and print each parsed statement
- for (SqlNode sqlNode : sqlNodeList) {
- System.out.println("Parsed statement: " + sqlNode.toString());
+ // Convert the SQL query to a relational expression
+ RelNode rel = planner.rel(validated).rel;
+
+ Interpreter interpreter = new Interpreter(DataContexts.EMPTY, rel);
+
+ // Create an interpreter to execute the RelNode
+ for (Object[] row : interpreter.asEnumerable()) {
+ listB.add((Integer) row[0]);
}
+
+ // Display the results
+ System.out.println("List A: " + listA);
+ System.out.println("List B (after SQL): " + listB);
}
}
(incubator-baremaps) 03/07: Format code
Posted by bc...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
bchapuis pushed a commit to branch calcite-framework
in repository https://gitbox.apache.org/repos/asf/incubator-baremaps.git
commit 97799717bab8f23348c6886003c943bb712aa925
Author: Bertil Chapuis <bc...@gmail.com>
AuthorDate: Fri Jan 17 18:39:12 2025 +0100
Format code
---
.../org/apache/baremaps/calcite/CalciteTest.java | 2 +-
.../data/calcite/CalciteMultipleDDLParser.java | 14 -
.../data/{calcite => store}/BaremapsDataTable.java | 2 +-
.../apache/baremaps/data/calcite/CalciteTest.java | 365 ++++++++++-----------
4 files changed, 170 insertions(+), 213 deletions(-)
diff --git a/baremaps-core/src/test/java/org/apache/baremaps/calcite/CalciteTest.java b/baremaps-core/src/test/java/org/apache/baremaps/calcite/CalciteTest.java
index 1b5e8b3b2..efc13eb4d 100644
--- a/baremaps-core/src/test/java/org/apache/baremaps/calcite/CalciteTest.java
+++ b/baremaps-core/src/test/java/org/apache/baremaps/calcite/CalciteTest.java
@@ -22,10 +22,10 @@ import com.google.common.collect.ImmutableList;
import java.sql.*;
import java.util.List;
import java.util.Properties;
-import org.apache.baremaps.data.calcite.BaremapsDataTable;
import org.apache.baremaps.data.calcite.SqlDataTable;
import org.apache.baremaps.data.collection.AppendOnlyLog;
import org.apache.baremaps.data.collection.IndexedDataList;
+import org.apache.baremaps.data.store.BaremapsDataTable;
import org.apache.baremaps.data.type.RowDataType;
import org.apache.baremaps.maplibre.vectortile.VectorTileFunctions;
import org.apache.baremaps.store.*;
diff --git a/baremaps-data/src/main/java/org/apache/baremaps/data/calcite/CalciteMultipleDDLParser.java b/baremaps-data/src/main/java/org/apache/baremaps/data/calcite/CalciteMultipleDDLParser.java
deleted file mode 100644
index 04984fd3e..000000000
--- a/baremaps-data/src/main/java/org/apache/baremaps/data/calcite/CalciteMultipleDDLParser.java
+++ /dev/null
@@ -1,14 +0,0 @@
-package org.apache.baremaps.data.calcite;
-
-import org.apache.calcite.config.Lex;
-import org.apache.calcite.sql.SqlNode;
-import org.apache.calcite.sql.SqlNodeList;
-import org.apache.calcite.sql.parser.SqlParser;
-import org.apache.calcite.sql.parser.ddl.SqlDdlParserImpl;
-import org.apache.calcite.sql.validate.SqlConformanceEnum;
-
-public class CalciteMultipleDDLParser {
- public static void main(String[] args) throws Exception {
-
- }
-}
diff --git a/baremaps-data/src/main/java/org/apache/baremaps/data/calcite/BaremapsDataTable.java b/baremaps-data/src/main/java/org/apache/baremaps/data/store/BaremapsDataTable.java
similarity index 97%
rename from baremaps-data/src/main/java/org/apache/baremaps/data/calcite/BaremapsDataTable.java
rename to baremaps-data/src/main/java/org/apache/baremaps/data/store/BaremapsDataTable.java
index 55ddee115..89123dab6 100644
--- a/baremaps-data/src/main/java/org/apache/baremaps/data/calcite/BaremapsDataTable.java
+++ b/baremaps-data/src/main/java/org/apache/baremaps/data/store/BaremapsDataTable.java
@@ -15,7 +15,7 @@
* limitations under the License.
*/
-package org.apache.baremaps.data.calcite;
+package org.apache.baremaps.data.store;
import java.util.Iterator;
import org.apache.baremaps.data.collection.DataCollection;
diff --git a/baremaps-data/src/test/java/org/apache/baremaps/data/calcite/CalciteTest.java b/baremaps-data/src/test/java/org/apache/baremaps/data/calcite/CalciteTest.java
index 9e7fc96d6..fd645d163 100644
--- a/baremaps-data/src/test/java/org/apache/baremaps/data/calcite/CalciteTest.java
+++ b/baremaps-data/src/test/java/org/apache/baremaps/data/calcite/CalciteTest.java
@@ -17,257 +17,228 @@
package org.apache.baremaps.data.calcite;
-import com.google.common.collect.ImmutableList;
+import java.sql.*;
+import java.util.*;
import org.apache.baremaps.data.collection.AppendOnlyLog;
import org.apache.baremaps.data.collection.IndexedDataList;
+import org.apache.baremaps.data.store.BaremapsDataTable;
import org.apache.baremaps.data.type.RowDataType;
+import org.apache.baremaps.store.*;
import org.apache.baremaps.store.DataColumn.Cardinality;
import org.apache.baremaps.store.DataColumn.Type;
-import org.apache.baremaps.store.*;
import org.apache.calcite.DataContext;
import org.apache.calcite.DataContexts;
-import org.apache.calcite.adapter.enumerable.EnumerableConvention;
-import org.apache.calcite.adapter.enumerable.EnumerableInterpreter;
-import org.apache.calcite.config.Lex;
import org.apache.calcite.interpreter.Interpreter;
import org.apache.calcite.jdbc.CalciteConnection;
import org.apache.calcite.linq4j.Enumerable;
import org.apache.calcite.linq4j.Linq4j;
-import org.apache.calcite.model.ModelHandler;
-import org.apache.calcite.plan.RelOptCluster;
-import org.apache.calcite.plan.RelOptPlanner;
-import org.apache.calcite.plan.RelTraitSet;
-import org.apache.calcite.plan.volcano.VolcanoPlanner;
import org.apache.calcite.rel.RelNode;
import org.apache.calcite.rel.type.RelDataType;
import org.apache.calcite.rel.type.RelDataTypeFactory;
-import org.apache.calcite.rex.RexBuilder;
-import org.apache.calcite.runtime.SpatialTypeFunctions;
import org.apache.calcite.schema.ScannableTable;
import org.apache.calcite.schema.SchemaPlus;
import org.apache.calcite.schema.Table;
import org.apache.calcite.schema.impl.*;
-import org.apache.calcite.sql.SqlNode;
-import org.apache.calcite.sql.SqlNodeList;
-import org.apache.calcite.sql.fun.SqlSpatialTypeFunctions;
-import org.apache.calcite.sql.parser.SqlParseException;
-import org.apache.calcite.sql.parser.SqlParser;
-import org.apache.calcite.sql.parser.ddl.SqlDdlParserImpl;
import org.apache.calcite.sql.type.SqlTypeName;
-import org.apache.calcite.sql.validate.SqlConformanceEnum;
import org.apache.calcite.tools.FrameworkConfig;
import org.apache.calcite.tools.Frameworks;
import org.apache.calcite.tools.Planner;
import org.junit.jupiter.api.Test;
import org.locationtech.jts.geom.Coordinate;
-import org.locationtech.jts.geom.Geometry;
import org.locationtech.jts.geom.GeometryFactory;
-import org.apache.calcite.adapter.enumerable.EnumerableRel;
-import org.apache.calcite.adapter.enumerable.EnumerableInterpreter;
-import org.apache.calcite.adapter.enumerable.EnumerableRelImplementor;
-import org.apache.calcite.linq4j.Enumerator;
-import org.apache.calcite.plan.*;
-import org.apache.calcite.rel.RelNode;
-import org.apache.calcite.rel.type.RelDataType;
-import org.apache.calcite.schema.SchemaPlus;
-import org.apache.calcite.tools.*;
-
-import java.sql.*;
-import java.util.*;
public class CalciteTest {
- @Test
- void sql() throws SQLException {
- GeometryFactory geometryFactory = new GeometryFactory();
-
- // Configure Calcite connection properties
- Properties info = new Properties();
- info.setProperty("lex", "MYSQL"); // Use MySQL dialect
- info.setProperty("caseSensitive", "false"); // Disable case sensitivity
- info.setProperty("unquotedCasing", "TO_LOWER"); // Convert unquoted identifiers to lowercase
- info.setProperty("quotedCasing", "TO_LOWER");
-
- try (Connection connection = DriverManager.getConnection("jdbc:calcite:", info)) {
- CalciteConnection calciteConnection = connection.unwrap(CalciteConnection.class);
- SchemaPlus rootSchema = calciteConnection.getRootSchema();
-
- // Create and add 'city' table
- DataSchema cityRowType = new DataSchemaImpl("city", List.of(
- new DataColumnFixed("id", Cardinality.OPTIONAL, Type.INTEGER),
- new DataColumnFixed("name", Cardinality.OPTIONAL, Type.STRING),
- new DataColumnFixed("geometry", Cardinality.OPTIONAL, Type.GEOMETRY)));
-
- DataTable cityDataTable = new BaremapsDataTable(
- cityRowType,
- new IndexedDataList<>(new AppendOnlyLog<>(new RowDataType(cityRowType))));
-
- cityDataTable.add(new DataRowImpl(cityDataTable.schema(),
- List.of(1, "Paris", geometryFactory.createPoint(new Coordinate(2.3522, 48.8566)))));
- cityDataTable.add(new DataRowImpl(cityDataTable.schema(),
- List.of(2, "New York", geometryFactory.createPoint(new Coordinate(-74.0060, 40.7128)))));
-
- SqlDataTable citySqlDataTable = new SqlDataTable(cityDataTable);
- rootSchema.add("city", citySqlDataTable);
-
- // Create and add 'population' table
- DataSchema populationRowType = new DataSchemaImpl("population", List.of(
- new DataColumnFixed("city_id", Cardinality.OPTIONAL, Type.INTEGER),
- new DataColumnFixed("population", Cardinality.OPTIONAL, Type.INTEGER)));
-
- DataTable populationDataTable = new BaremapsDataTable(
- populationRowType,
- new IndexedDataList<>(new AppendOnlyLog<>(new RowDataType(populationRowType))));
-
- populationDataTable.add(new DataRowImpl(populationDataTable.schema(), List.of(1, 2_161_000)));
- populationDataTable.add(new DataRowImpl(populationDataTable.schema(), List.of(2, 8_336_000)));
-
- SqlDataTable populationSqlDataTable = new SqlDataTable(populationDataTable);
- rootSchema.add("population", populationSqlDataTable);
-
- // Create view 'city_population'
- String mvSql = "SELECT c.id, c.name, c.geometry, p.population " +
- "FROM city c " + // lowercase and unquoted
- "JOIN population p ON c.id = p.city_id";
-
- ViewTableMacro materializedView = MaterializedViewTable.viewMacro(
- rootSchema,
- mvSql,
- Collections.emptyList(), // Schema path
- List.of("city_population"), // Name parts
- false); // Not a materialized view
-
-
- rootSchema.add("city_population", materializedView);
-
- // Debug: List all tables in the root schema
- System.out.println("Available tables in the root schema:");
- for (String tableName : rootSchema.getTableNames()) {
- System.out.println(" - " + tableName);
- }
-
- String sql = "SELECT * FROM city";
- try (Statement statement = connection.createStatement();
- ResultSet resultSet = statement.executeQuery(sql)) {
- while (resultSet.next()) {
- System.out.println(resultSet.getString("id") + " " + resultSet.getString("geometry"));
- }
- }
-
- // Query the view
- sql = "SELECT * FROM city_population";
- try (Statement statement = connection.createStatement();
- ResultSet resultSet = statement.executeQuery(sql)) {
- while (resultSet.next()) {
- System.out.println(
- resultSet.getString("id") + " " + resultSet.getString("name")
- );
- }
- }
+ @Test
+ void sql() throws SQLException {
+ GeometryFactory geometryFactory = new GeometryFactory();
+
+ // Configure Calcite connection properties
+ Properties info = new Properties();
+ info.setProperty("lex", "MYSQL"); // Use MySQL dialect
+ info.setProperty("caseSensitive", "false"); // Disable case sensitivity
+ info.setProperty("unquotedCasing", "TO_LOWER"); // Convert unquoted identifiers to lowercase
+ info.setProperty("quotedCasing", "TO_LOWER");
+
+ try (Connection connection = DriverManager.getConnection("jdbc:calcite:", info)) {
+ CalciteConnection calciteConnection = connection.unwrap(CalciteConnection.class);
+ SchemaPlus rootSchema = calciteConnection.getRootSchema();
+
+ // Create and add 'city' table
+ DataSchema cityRowType = new DataSchemaImpl("city", List.of(
+ new DataColumnFixed("id", Cardinality.OPTIONAL, Type.INTEGER),
+ new DataColumnFixed("name", Cardinality.OPTIONAL, Type.STRING),
+ new DataColumnFixed("geometry", Cardinality.OPTIONAL, Type.GEOMETRY)));
+
+ DataTable cityDataTable = new BaremapsDataTable(
+ cityRowType,
+ new IndexedDataList<>(new AppendOnlyLog<>(new RowDataType(cityRowType))));
+
+ cityDataTable.add(new DataRowImpl(cityDataTable.schema(),
+ List.of(1, "Paris", geometryFactory.createPoint(new Coordinate(2.3522, 48.8566)))));
+ cityDataTable.add(new DataRowImpl(cityDataTable.schema(),
+ List.of(2, "New York", geometryFactory.createPoint(new Coordinate(-74.0060, 40.7128)))));
+
+ SqlDataTable citySqlDataTable = new SqlDataTable(cityDataTable);
+ rootSchema.add("city", citySqlDataTable);
+
+ // Create and add 'population' table
+ DataSchema populationRowType = new DataSchemaImpl("population", List.of(
+ new DataColumnFixed("city_id", Cardinality.OPTIONAL, Type.INTEGER),
+ new DataColumnFixed("population", Cardinality.OPTIONAL, Type.INTEGER)));
+
+ DataTable populationDataTable = new BaremapsDataTable(
+ populationRowType,
+ new IndexedDataList<>(new AppendOnlyLog<>(new RowDataType(populationRowType))));
+
+ populationDataTable.add(new DataRowImpl(populationDataTable.schema(), List.of(1, 2_161_000)));
+ populationDataTable.add(new DataRowImpl(populationDataTable.schema(), List.of(2, 8_336_000)));
+
+ SqlDataTable populationSqlDataTable = new SqlDataTable(populationDataTable);
+ rootSchema.add("population", populationSqlDataTable);
+
+ // Create view 'city_population'
+ String mvSql = "SELECT c.id, c.name, c.geometry, p.population " +
+ "FROM city c " + // lowercase and unquoted
+ "JOIN population p ON c.id = p.city_id";
+
+ ViewTableMacro materializedView = MaterializedViewTable.viewMacro(
+ rootSchema,
+ mvSql,
+ Collections.emptyList(), // Schema path
+ List.of("city_population"), // Name parts
+ false); // Not a materialized view
+
+
+ rootSchema.add("city_population", materializedView);
+
+ // Debug: List all tables in the root schema
+ System.out.println("Available tables in the root schema:");
+ for (String tableName : rootSchema.getTableNames()) {
+ System.out.println(" - " + tableName);
+ }
+
+ String sql = "SELECT * FROM city";
+ try (Statement statement = connection.createStatement();
+ ResultSet resultSet = statement.executeQuery(sql)) {
+ while (resultSet.next()) {
+ System.out.println(resultSet.getString("id") + " " + resultSet.getString("geometry"));
}
-
+ }
+
+ // Query the view
+ sql = "SELECT * FROM city_population";
+ try (Statement statement = connection.createStatement();
+ ResultSet resultSet = statement.executeQuery(sql)) {
+ while (resultSet.next()) {
+ System.out.println(
+ resultSet.getString("id") + " " + resultSet.getString("name"));
+ }
+ }
}
- public class ListTable extends AbstractTable implements ScannableTable {
- private final List<Integer> data;
+ }
- public ListTable(List<Integer> data) {
- this.data = data;
- }
+ public class ListTable extends AbstractTable implements ScannableTable {
+ private final List<Integer> data;
- @Override
- public RelDataType getRowType(RelDataTypeFactory typeFactory) {
- // Define a single column named "value" of type INTEGER
- return typeFactory.builder()
- .add("V", SqlTypeName.INTEGER)
- .build();
- }
+ public ListTable(List<Integer> data) {
+ this.data = data;
+ }
- @Override
- public Enumerable<Object[]> scan(DataContext root) {
- // Convert the List<Integer> to Enumerable<Object[]>
- return Linq4j.asEnumerable(data)
- .select(i -> new Object[]{i});
- }
+ @Override
+ public RelDataType getRowType(RelDataTypeFactory typeFactory) {
+ // Define a single column named "value" of type INTEGER
+ return typeFactory.builder()
+ .add("V", SqlTypeName.INTEGER)
+ .build();
}
- public class ListSchema extends AbstractSchema {
- private final List<Integer> listA;
- private final List<Integer> listB;
+ @Override
+ public Enumerable<Object[]> scan(DataContext root) {
+ // Convert the List<Integer> to Enumerable<Object[]>
+ return Linq4j.asEnumerable(data)
+ .select(i -> new Object[] {i});
+ }
+ }
- public ListSchema(List<Integer> listA, List<Integer> listB) {
- this.listA = listA;
- this.listB = listB;
- }
+ public class ListSchema extends AbstractSchema {
+ private final List<Integer> listA;
+ private final List<Integer> listB;
- @Override
- protected Map<String, Table> getTableMap() {
- Map<String, Table> tables = new HashMap<>();
- tables.put("LIST_A", new ListTable(listA));
- tables.put("LIST_B", new ListTable(listB)); // Initially empty
- return tables;
- }
+ public ListSchema(List<Integer> listA, List<Integer> listB) {
+ this.listA = listA;
+ this.listB = listB;
}
- @Test
- void list() throws Exception {
- // Initialize your Java lists
- List<Integer> listA = List.of(1, 2, 3, 4, 5);
- List<Integer> listB = new ArrayList<>();
+ @Override
+ protected Map<String, Table> getTableMap() {
+ Map<String, Table> tables = new HashMap<>();
+ tables.put("LIST_A", new ListTable(listA));
+ tables.put("LIST_B", new ListTable(listB)); // Initially empty
+ return tables;
+ }
+ }
- // Set up Calcite schema
- SchemaPlus rootSchema = Frameworks.createRootSchema(true);
- rootSchema.add("MY_SCHEMA", new ListSchema(listA, listB));
+ @Test
+ void list() throws Exception {
+ // Initialize your Java lists
+ List<Integer> listA = List.of(1, 2, 3, 4, 5);
+ List<Integer> listB = new ArrayList<>();
- // Create and add 'city' table
- DataSchema cityRowType = new DataSchemaImpl("city", List.of(
- new DataColumnFixed("id", Cardinality.OPTIONAL, Type.INTEGER),
- new DataColumnFixed("name", Cardinality.OPTIONAL, Type.STRING),
- new DataColumnFixed("geometry", Cardinality.OPTIONAL, Type.GEOMETRY)));
+ // Set up Calcite schema
+ SchemaPlus rootSchema = Frameworks.createRootSchema(true);
+ rootSchema.add("MY_SCHEMA", new ListSchema(listA, listB));
- DataTable cityDataTable = new BaremapsDataTable(
- cityRowType,
- new IndexedDataList<>(new AppendOnlyLog<>(new RowDataType(cityRowType))));
+ // Create and add 'city' table
+ DataSchema cityRowType = new DataSchemaImpl("city", List.of(
+ new DataColumnFixed("id", Cardinality.OPTIONAL, Type.INTEGER),
+ new DataColumnFixed("name", Cardinality.OPTIONAL, Type.STRING),
+ new DataColumnFixed("geometry", Cardinality.OPTIONAL, Type.GEOMETRY)));
- GeometryFactory geometryFactory = new GeometryFactory();
- cityDataTable.add(new DataRowImpl(cityDataTable.schema(),
- List.of(1, "Paris", geometryFactory.createPoint(new Coordinate(2.3522, 48.8566)))));
- cityDataTable.add(new DataRowImpl(cityDataTable.schema(),
- List.of(2, "New York", geometryFactory.createPoint(new Coordinate(-74.0060, 40.7128)))));
+ DataTable cityDataTable = new BaremapsDataTable(
+ cityRowType,
+ new IndexedDataList<>(new AppendOnlyLog<>(new RowDataType(cityRowType))));
- SqlDataTable citySqlDataTable = new SqlDataTable(cityDataTable);
- rootSchema.add("CITY", citySqlDataTable);
+ GeometryFactory geometryFactory = new GeometryFactory();
+ cityDataTable.add(new DataRowImpl(cityDataTable.schema(),
+ List.of(1, "Paris", geometryFactory.createPoint(new Coordinate(2.3522, 48.8566)))));
+ cityDataTable.add(new DataRowImpl(cityDataTable.schema(),
+ List.of(2, "New York", geometryFactory.createPoint(new Coordinate(-74.0060, 40.7128)))));
- // Configure the framework
- FrameworkConfig config = Frameworks.newConfigBuilder()
- .defaultSchema(rootSchema.getSubSchema("MY_SCHEMA"))
- .build();
+ SqlDataTable citySqlDataTable = new SqlDataTable(cityDataTable);
+ rootSchema.add("CITY", citySqlDataTable);
- // Create a planner
- Planner planner = Frameworks.getPlanner(config);
+ // Configure the framework
+ FrameworkConfig config = Frameworks.newConfigBuilder()
+ .defaultSchema(rootSchema.getSubSchema("MY_SCHEMA"))
+ .build();
- // Define the SQL query to populate list_b from list_a
- String sql = "SELECT V * 2 AS V FROM LIST_A";
+ // Create a planner
+ Planner planner = Frameworks.getPlanner(config);
- // Parse the SQL query
- org.apache.calcite.sql.SqlNode parsed = planner.parse(sql);
+ // Define the SQL query to populate list_b from list_a
+ String sql = "SELECT V * 2 AS V FROM LIST_A";
- // Validate the SQL query
- org.apache.calcite.sql.SqlNode validated = planner.validate(parsed);
+ // Parse the SQL query
+ org.apache.calcite.sql.SqlNode parsed = planner.parse(sql);
- // Convert the SQL query to a relational expression
- RelNode rel = planner.rel(validated).rel;
+ // Validate the SQL query
+ org.apache.calcite.sql.SqlNode validated = planner.validate(parsed);
- Interpreter interpreter = new Interpreter(DataContexts.EMPTY, rel);
+ // Convert the SQL query to a relational expression
+ RelNode rel = planner.rel(validated).rel;
- // Create an interpreter to execute the RelNode
- for (Object[] row : interpreter.asEnumerable()) {
- listB.add((Integer) row[0]);
- }
+ Interpreter interpreter = new Interpreter(DataContexts.EMPTY, rel);
- // Display the results
- System.out.println("List A: " + listA);
- System.out.println("List B (after SQL): " + listB);
+ // Create an interpreter to execute the RelNode
+ for (Object[] row : interpreter.asEnumerable()) {
+ listB.add((Integer) row[0]);
}
+ // Display the results
+ System.out.println("List A: " + listA);
+ System.out.println("List B (after SQL): " + listB);
+ }
+
}