You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@baremaps.apache.org by bc...@apache.org on 2023/09/12 19:21:23 UTC
[incubator-baremaps] branch main updated: Add a datasource object to the tileset (#777)
This is an automated email from the ASF dual-hosted git repository.
bchapuis pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/incubator-baremaps.git
The following commit(s) were added to refs/heads/main by this push:
new dfa1e949 Add a datasource object to the tileset (#777)
dfa1e949 is described below
commit dfa1e949059561b2d0523b5e298a80b5f7f48341
Author: Bertil Chapuis <bc...@gmail.com>
AuthorDate: Tue Sep 12 21:21:19 2023 +0200
Add a datasource object to the tileset (#777)
* Add a database object to the tileset
* Allow the database to be either a jdbc url or an object describing a datasource
---
.../main/java/org/apache/baremaps/cli/map/Dev.java | 2 +-
.../java/org/apache/baremaps/cli/map/Serve.java | 4 +-
.../org/apache/baremaps/utils/PostgresUtils.java | 104 +++++++++--
.../baremaps/vectortile/tileset/Database.java | 194 +++++++++++++++++++++
.../baremaps/vectortile/tileset/Tileset.java | 6 +-
.../apache/baremaps/workflow/WorkflowContext.java | 6 +-
.../apache/baremaps/workflow/tasks/ExecuteSql.java | 2 +-
.../baremaps/workflow/tasks/ExecuteSqlScript.java | 2 +-
.../baremaps/workflow/tasks/ExportVectorTiles.java | 1 +
.../baremaps/workflow/tasks/ImportGeoPackage.java | 2 +-
.../workflow/tasks/ImportOpenStreetMap.java | 2 +-
.../baremaps/workflow/tasks/ImportShapefile.java | 2 +-
.../workflow/tasks/UpdateOpenStreetMap.java | 2 +-
13 files changed, 303 insertions(+), 26 deletions(-)
diff --git a/baremaps-cli/src/main/java/org/apache/baremaps/cli/map/Dev.java b/baremaps-cli/src/main/java/org/apache/baremaps/cli/map/Dev.java
index 1813823b..82aa6b77 100644
--- a/baremaps-cli/src/main/java/org/apache/baremaps/cli/map/Dev.java
+++ b/baremaps-cli/src/main/java/org/apache/baremaps/cli/map/Dev.java
@@ -69,7 +69,7 @@ public class Dev implements Callable<Integer> {
var configReader = new ConfigReader();
var objectMapper = objectMapper();
var tileset = objectMapper.readValue(configReader.read(this.tilesetPath), Tileset.class);
- var datasource = PostgresUtils.createDataSource(tileset.getDatabase());
+ var datasource = PostgresUtils.createDataSourceFromObject(tileset.getDatabase());
var tileStoreType = new TypeLiteral<Supplier<TileStore>>() {};
var tileStoreSupplier = (Supplier<TileStore>) () -> {
diff --git a/baremaps-cli/src/main/java/org/apache/baremaps/cli/map/Serve.java b/baremaps-cli/src/main/java/org/apache/baremaps/cli/map/Serve.java
index f4a47619..ae0137fe 100644
--- a/baremaps-cli/src/main/java/org/apache/baremaps/cli/map/Serve.java
+++ b/baremaps-cli/src/main/java/org/apache/baremaps/cli/map/Serve.java
@@ -71,9 +71,9 @@ public class Serve implements Callable<Integer> {
var objectMapper = objectMapper();
var configReader = new ConfigReader();
var caffeineSpec = CaffeineSpec.parse(cache);
-
var tileset = objectMapper.readValue(configReader.read(tilesetPath), Tileset.class);
- var datasource = PostgresUtils.createDataSource(tileset.getDatabase());
+ var datasource = PostgresUtils.createDataSourceFromObject(tileset.getDatabase());
+
var tileStoreSupplierType = new TypeLiteral<Supplier<TileStore>>() {};
var tileStore = new PostgresTileStore(datasource, tileset);
var tileCache = new TileCache(tileStore, caffeineSpec);
diff --git a/baremaps-core/src/main/java/org/apache/baremaps/utils/PostgresUtils.java b/baremaps-core/src/main/java/org/apache/baremaps/utils/PostgresUtils.java
index e9679483..438f614b 100644
--- a/baremaps-core/src/main/java/org/apache/baremaps/utils/PostgresUtils.java
+++ b/baremaps-core/src/main/java/org/apache/baremaps/utils/PostgresUtils.java
@@ -13,7 +13,7 @@
package org.apache.baremaps.utils;
-
+import com.fasterxml.jackson.databind.ObjectMapper;
import com.google.common.io.Resources;
import com.zaxxer.hikari.HikariConfig;
import com.zaxxer.hikari.HikariDataSource;
@@ -24,32 +24,64 @@ import java.sql.Connection;
import java.sql.SQLException;
import java.sql.Statement;
import javax.sql.DataSource;
+import org.apache.baremaps.vectortile.tileset.Database;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-/** A helper class for creating data sources and executing queries. */
+/**
+ * A helper class for creating data sources and executing queries.
+ */
public final class PostgresUtils {
private static final Logger logger = LoggerFactory.getLogger(PostgresUtils.class);
private PostgresUtils() {}
- public static DataSource createDataSource(String host, Integer port, String database,
- String username, String password) {
+
+ /**
+ * Creates a data source from an object, either a JDBC url or a json representation of a database.
+ *
+ * @param database the database object, either a JDBC url or a json representation of a database
+ * @return the data source
+ */
+ public static DataSource createDataSourceFromObject(Object database) {
+ if (database instanceof String url) {
+ return createDataSource(url);
+ } else {
+ var json = new ObjectMapper().convertValue(database, Database.class);
+ return createDataSource(json);
+ }
+ }
+
+ /**
+ * Creates a data source from parameters.
+ *
+ * @param host the host
+ * @param port the port
+ * @param database the database
+ * @param username the username
+ * @param password the password
+ * @return the data source
+ */
+ public static DataSource createDataSource(
+ String host,
+ Integer port,
+ String database,
+ String username,
+ String password) {
return createDataSource(
String.format("jdbc:postgresql://%s:%s/%s?&user=%s&password=%s", host, port,
database, username, password));
}
/**
- * Creates a data source from a JDBC url with a pool size corresponding to the number of available
- * processors.
+ * Creates a data source from a JDBC url.
*
- * @param url the JDBC url
+ * @param database the JDBC url
* @return the data source
*/
- public static DataSource createDataSource(String url) {
- return createDataSource(url, Runtime.getRuntime().availableProcessors() * 2);
+ public static DataSource createDataSource(String database) {
+ return createDataSource(database, Runtime.getRuntime().availableProcessors() * 2);
}
/**
@@ -70,6 +102,56 @@ public final class PostgresUtils {
return new HikariDataSource(config);
}
+ /**
+ * Creates a data source from a json representation of a database.
+ *
+ * @param datasource the object representation of a database
+ * @return the data source
+ */
+ public static DataSource createDataSource(Database datasource) {
+ var config = new HikariConfig();
+ if (datasource.getDataSourceClassName() != null) {
+ config.setDataSourceClassName(datasource.getDataSourceClassName());
+ }
+ if (datasource.getJdbcUrl() != null) {
+ config.setJdbcUrl(datasource.getJdbcUrl());
+ }
+ if (datasource.getUsername() != null) {
+ config.setUsername(datasource.getUsername());
+ }
+ if (datasource.getPassword() != null) {
+ config.setPassword(datasource.getPassword());
+ }
+ if (datasource.getAutoCommit() != null) {
+ config.setAutoCommit(datasource.getAutoCommit());
+ }
+ if (datasource.getConnectionTimeout() != null) {
+ config.setConnectionTimeout(datasource.getConnectionTimeout());
+ }
+ if (datasource.getIdleTimeout() != null) {
+ config.setInitializationFailTimeout(datasource.getIdleTimeout());
+ }
+ if (datasource.getKeepAliveTime() != null) {
+ config.setKeepaliveTime(datasource.getKeepAliveTime());
+ }
+ if (datasource.getMaxLifetime() != null) {
+ config.setMaxLifetime(datasource.getMaxLifetime());
+ }
+ if (datasource.getMinimumIdle() != null) {
+ config.setMinimumIdle(datasource.getMinimumIdle());
+ }
+ if (datasource.getMaximumPoolSize() != null) {
+ config.setMaximumPoolSize(datasource.getMaximumPoolSize());
+ }
+ if (datasource.getPoolName() != null) {
+ config.setPoolName(datasource.getPoolName());
+ }
+ if (datasource.getReadOnly() != null) {
+ config.setReadOnly(datasource.getReadOnly());
+ }
+ return new HikariDataSource(config);
+ }
+
/**
* Appends the multi-queries parameter to the given JDBC URL.
*
@@ -92,8 +174,8 @@ public final class PostgresUtils {
*
* @param connection the JDBC connection
* @param resource the path of the resource file
- * @throws IOException
- * @throws SQLException
+ * @throws IOException if an I/O error occurs
+ * @throws SQLException if a database access error occurs
*/
public static void executeResource(Connection connection, String resource)
throws IOException, SQLException {
diff --git a/baremaps-core/src/main/java/org/apache/baremaps/vectortile/tileset/Database.java b/baremaps-core/src/main/java/org/apache/baremaps/vectortile/tileset/Database.java
new file mode 100644
index 00000000..06247077
--- /dev/null
+++ b/baremaps-core/src/main/java/org/apache/baremaps/vectortile/tileset/Database.java
@@ -0,0 +1,194 @@
+/*
+ * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except
+ * in compliance with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software distributed under the License
+ * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
+ * or implied. See the License for the specific language governing permissions and limitations under
+ * the License.
+ */
+
+package org.apache.baremaps.vectortile.tileset;
+
+import com.fasterxml.jackson.annotation.JsonProperty;
+import java.util.Objects;
+
+public class Database {
+
+ @JsonProperty("dataSourceClassName")
+ private String dataSourceClassName;
+
+ @JsonProperty("jdbcUrl")
+ private String jdbcUrl;
+
+ @JsonProperty("username")
+ private String username;
+
+ @JsonProperty("password")
+ private String password;
+
+ @JsonProperty("autoCommit")
+ private Boolean autoCommit;
+
+ @JsonProperty("connectionTimeout")
+ private Integer connectionTimeout;
+
+ @JsonProperty("idleTimeout")
+ private Integer idleTimeout;
+
+ @JsonProperty("keepAliveTime")
+ private Integer keepAliveTime;
+
+ @JsonProperty("maxLifetime")
+ private Integer maxLifetime;
+
+ @JsonProperty("minimumIdle")
+ private Integer minimumIdle;
+
+ @JsonProperty("maximumPoolSize")
+ private Integer maximumPoolSize;
+
+ @JsonProperty("poolName")
+ private String poolName;
+
+ @JsonProperty("readOnly")
+ private Boolean readOnly;
+
+ public Database() {}
+
+ public String getDataSourceClassName() {
+ return dataSourceClassName;
+ }
+
+ public void setDataSourceClassName(String dataSourceClassName) {
+ this.dataSourceClassName = dataSourceClassName;
+ }
+
+ public String getJdbcUrl() {
+ return jdbcUrl;
+ }
+
+ public void setJdbcUrl(String jdbcUrl) {
+ this.jdbcUrl = jdbcUrl;
+ }
+
+ public String getUsername() {
+ return username;
+ }
+
+ public void setUsername(String username) {
+ this.username = username;
+ }
+
+ public String getPassword() {
+ return password;
+ }
+
+ public void setPassword(String password) {
+ this.password = password;
+ }
+
+ public Boolean getAutoCommit() {
+ return autoCommit;
+ }
+
+ public void setAutoCommit(Boolean autoCommit) {
+ this.autoCommit = autoCommit;
+ }
+
+ public Integer getConnectionTimeout() {
+ return connectionTimeout;
+ }
+
+ public void setConnectionTimeout(Integer connectionTimeout) {
+ this.connectionTimeout = connectionTimeout;
+ }
+
+ public Integer getIdleTimeout() {
+ return idleTimeout;
+ }
+
+ public void setIdleTimeout(Integer idleTimeout) {
+ this.idleTimeout = idleTimeout;
+ }
+
+ public Integer getKeepAliveTime() {
+ return keepAliveTime;
+ }
+
+ public void setKeepAliveTime(Integer keepAliveTime) {
+ this.keepAliveTime = keepAliveTime;
+ }
+
+ public Integer getMaxLifetime() {
+ return maxLifetime;
+ }
+
+ public void setMaxLifetime(Integer maxLifetime) {
+ this.maxLifetime = maxLifetime;
+ }
+
+ public Integer getMinimumIdle() {
+ return minimumIdle;
+ }
+
+ public void setMinimumIdle(Integer minimumIdle) {
+ this.minimumIdle = minimumIdle;
+ }
+
+ public Integer getMaximumPoolSize() {
+ return maximumPoolSize;
+ }
+
+ public void setMaximumPoolSize(Integer maximumPoolSize) {
+ this.maximumPoolSize = maximumPoolSize;
+ }
+
+ public String getPoolName() {
+ return poolName;
+ }
+
+ public void setPoolName(String poolName) {
+ this.poolName = poolName;
+ }
+
+ public Boolean getReadOnly() {
+ return readOnly;
+ }
+
+ public void setReadOnly(Boolean readOnly) {
+ this.readOnly = readOnly;
+ }
+
+ @Override
+ public boolean equals(Object o) {
+ if (this == o) {
+ return true;
+ }
+ if (o == null || getClass() != o.getClass()) {
+ return false;
+ }
+ Database database = (Database) o;
+ return Objects.equals(dataSourceClassName, database.dataSourceClassName)
+ && Objects.equals(jdbcUrl, database.jdbcUrl) && Objects.equals(username, database.username)
+ && Objects.equals(password, database.password)
+ && Objects.equals(autoCommit, database.autoCommit)
+ && Objects.equals(connectionTimeout, database.connectionTimeout)
+ && Objects.equals(idleTimeout, database.idleTimeout)
+ && Objects.equals(keepAliveTime, database.keepAliveTime)
+ && Objects.equals(maxLifetime, database.maxLifetime)
+ && Objects.equals(minimumIdle, database.minimumIdle)
+ && Objects.equals(maximumPoolSize, database.maximumPoolSize)
+ && Objects.equals(poolName, database.poolName)
+ && Objects.equals(readOnly, database.readOnly);
+ }
+
+ @Override
+ public int hashCode() {
+ return Objects.hash(dataSourceClassName, jdbcUrl, username, password, autoCommit,
+ connectionTimeout, idleTimeout, keepAliveTime, maxLifetime, minimumIdle, maximumPoolSize,
+ poolName, readOnly);
+ }
+}
diff --git a/baremaps-core/src/main/java/org/apache/baremaps/vectortile/tileset/Tileset.java b/baremaps-core/src/main/java/org/apache/baremaps/vectortile/tileset/Tileset.java
index 7338435c..14822c7d 100644
--- a/baremaps-core/src/main/java/org/apache/baremaps/vectortile/tileset/Tileset.java
+++ b/baremaps-core/src/main/java/org/apache/baremaps/vectortile/tileset/Tileset.java
@@ -13,7 +13,6 @@
package org.apache.baremaps.vectortile.tileset;
-
import com.fasterxml.jackson.annotation.JsonProperty;
import java.util.ArrayList;
import java.util.List;
@@ -72,8 +71,9 @@ public class Tileset {
@JsonProperty("center")
private List<Double> center = new ArrayList<>();
+ @Deprecated
@JsonProperty("database")
- private String database;
+ private Object database;
@JsonProperty("vector_layers")
private List<TilesetLayer> vectorLayers = new ArrayList<>();
@@ -240,7 +240,7 @@ public class Tileset {
return this;
}
- public String getDatabase() {
+ public Object getDatabase() {
return database;
}
diff --git a/baremaps-core/src/main/java/org/apache/baremaps/workflow/WorkflowContext.java b/baremaps-core/src/main/java/org/apache/baremaps/workflow/WorkflowContext.java
index b03060ca..4ba340a1 100644
--- a/baremaps-core/src/main/java/org/apache/baremaps/workflow/WorkflowContext.java
+++ b/baremaps-core/src/main/java/org/apache/baremaps/workflow/WorkflowContext.java
@@ -24,7 +24,7 @@ import org.apache.baremaps.utils.PostgresUtils;
*/
public class WorkflowContext {
- private Map<String, DataSource> dataSources = new ConcurrentHashMap<>() {};
+ private Map<Object, DataSource> dataSources = new ConcurrentHashMap<>() {};
/**
* Returns the data source associated with the specified database.
@@ -32,8 +32,8 @@ public class WorkflowContext {
* @param database the JDBC connection string to the database
* @return the data source
*/
- public DataSource getDataSource(String database) {
- return dataSources.computeIfAbsent(database, PostgresUtils::createDataSource);
+ public DataSource getDataSource(Object database) {
+ return dataSources.computeIfAbsent(database, PostgresUtils::createDataSourceFromObject);
}
}
diff --git a/baremaps-core/src/main/java/org/apache/baremaps/workflow/tasks/ExecuteSql.java b/baremaps-core/src/main/java/org/apache/baremaps/workflow/tasks/ExecuteSql.java
index 95af43e4..f40842bb 100644
--- a/baremaps-core/src/main/java/org/apache/baremaps/workflow/tasks/ExecuteSql.java
+++ b/baremaps-core/src/main/java/org/apache/baremaps/workflow/tasks/ExecuteSql.java
@@ -22,7 +22,7 @@ import org.apache.baremaps.workflow.Task;
import org.apache.baremaps.workflow.WorkflowContext;
import org.apache.baremaps.workflow.WorkflowException;
-public record ExecuteSql(String database, Path file, boolean parallel) implements Task {
+public record ExecuteSql(Object database, Path file, boolean parallel) implements Task {
@Override
public void execute(WorkflowContext context) throws Exception {
diff --git a/baremaps-core/src/main/java/org/apache/baremaps/workflow/tasks/ExecuteSqlScript.java b/baremaps-core/src/main/java/org/apache/baremaps/workflow/tasks/ExecuteSqlScript.java
index 9e641590..8855bf99 100644
--- a/baremaps-core/src/main/java/org/apache/baremaps/workflow/tasks/ExecuteSqlScript.java
+++ b/baremaps-core/src/main/java/org/apache/baremaps/workflow/tasks/ExecuteSqlScript.java
@@ -19,7 +19,7 @@ import org.apache.baremaps.workflow.Task;
import org.apache.baremaps.workflow.WorkflowContext;
import org.apache.baremaps.workflow.WorkflowException;
-public record ExecuteSqlScript(String database, Path file) implements Task {
+public record ExecuteSqlScript(Object database, Path file) implements Task {
@Override
public void execute(WorkflowContext context) throws Exception {
diff --git a/baremaps-core/src/main/java/org/apache/baremaps/workflow/tasks/ExportVectorTiles.java b/baremaps-core/src/main/java/org/apache/baremaps/workflow/tasks/ExportVectorTiles.java
index 5bced5a8..5b9f672e 100644
--- a/baremaps-core/src/main/java/org/apache/baremaps/workflow/tasks/ExportVectorTiles.java
+++ b/baremaps-core/src/main/java/org/apache/baremaps/workflow/tasks/ExportVectorTiles.java
@@ -59,6 +59,7 @@ public record ExportVectorTiles(
var objectMapper = objectMapper();
var tileset = objectMapper.readValue(configReader.read(this.tileset), Tileset.class);
var datasource = context.getDataSource(tileset.getDatabase());
+
var sourceTileStore = sourceTileStore(tileset, datasource);
var targetTileStore = targetTileStore(tileset);
diff --git a/baremaps-core/src/main/java/org/apache/baremaps/workflow/tasks/ImportGeoPackage.java b/baremaps-core/src/main/java/org/apache/baremaps/workflow/tasks/ImportGeoPackage.java
index 1592619d..5490c7de 100644
--- a/baremaps-core/src/main/java/org/apache/baremaps/workflow/tasks/ImportGeoPackage.java
+++ b/baremaps-core/src/main/java/org/apache/baremaps/workflow/tasks/ImportGeoPackage.java
@@ -24,7 +24,7 @@ import org.apache.baremaps.workflow.WorkflowException;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-public record ImportGeoPackage(Path file, String database, Integer sourceSRID, Integer targetSRID)
+public record ImportGeoPackage(Path file, Object database, Integer sourceSRID, Integer targetSRID)
implements
Task {
diff --git a/baremaps-core/src/main/java/org/apache/baremaps/workflow/tasks/ImportOpenStreetMap.java b/baremaps-core/src/main/java/org/apache/baremaps/workflow/tasks/ImportOpenStreetMap.java
index b6486b9c..c387fe0c 100644
--- a/baremaps-core/src/main/java/org/apache/baremaps/workflow/tasks/ImportOpenStreetMap.java
+++ b/baremaps-core/src/main/java/org/apache/baremaps/workflow/tasks/ImportOpenStreetMap.java
@@ -41,7 +41,7 @@ import org.locationtech.jts.geom.Coordinate;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-public record ImportOpenStreetMap(Path file, String database, Integer databaseSrid)
+public record ImportOpenStreetMap(Path file, Object database, Integer databaseSrid)
implements
Task {
diff --git a/baremaps-core/src/main/java/org/apache/baremaps/workflow/tasks/ImportShapefile.java b/baremaps-core/src/main/java/org/apache/baremaps/workflow/tasks/ImportShapefile.java
index a32fab78..70597fe6 100644
--- a/baremaps-core/src/main/java/org/apache/baremaps/workflow/tasks/ImportShapefile.java
+++ b/baremaps-core/src/main/java/org/apache/baremaps/workflow/tasks/ImportShapefile.java
@@ -24,7 +24,7 @@ import org.apache.baremaps.workflow.WorkflowException;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-public record ImportShapefile(Path file, String database, Integer sourceSRID, Integer targetSRID)
+public record ImportShapefile(Path file, Object database, Integer sourceSRID, Integer targetSRID)
implements
Task {
diff --git a/baremaps-core/src/main/java/org/apache/baremaps/workflow/tasks/UpdateOpenStreetMap.java b/baremaps-core/src/main/java/org/apache/baremaps/workflow/tasks/UpdateOpenStreetMap.java
index 6b7babb5..4d0a7092 100644
--- a/baremaps-core/src/main/java/org/apache/baremaps/workflow/tasks/UpdateOpenStreetMap.java
+++ b/baremaps-core/src/main/java/org/apache/baremaps/workflow/tasks/UpdateOpenStreetMap.java
@@ -44,7 +44,7 @@ import org.locationtech.jts.geom.Coordinate;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-public record UpdateOpenStreetMap(String database, Integer databaseSrid) implements Task {
+public record UpdateOpenStreetMap(Object database, Integer databaseSrid) implements Task {
private static final Logger logger = LoggerFactory.getLogger(UpdateOpenStreetMap.class);