You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@baremaps.apache.org by ad...@apache.org on 2024/06/02 09:49:35 UTC
(incubator-baremaps) branch 849-overture-example created (now 61b615c9)
This is an automated email from the ASF dual-hosted git repository.
adrabble pushed a change to branch 849-overture-example
in repository https://gitbox.apache.org/repos/asf/incubator-baremaps.git
at 61b615c9 Create a simple overture example
This branch includes the following new commits:
new cda5ec81 Merge remote-tracking branch 'origin/849-nested-type-and-jsonb' into 849-default-crs
new 6cf69a22 Merge remote-tracking branch 'origin/849-nested-type-and-jsonb' into 849-default-crs
new 61b615c9 Create a simple overture example
The 3 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
(incubator-baremaps) 03/03: Create a simple overture example
Posted by ad...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
adrabble pushed a commit to branch 849-overture-example
in repository https://gitbox.apache.org/repos/asf/incubator-baremaps.git
commit 61b615c98ed623a76313340f43a2a8f66bdb0c1c
Author: drabble <an...@gmail.com>
AuthorDate: Sun Jun 2 11:49:16 2024 +0200
Create a simple overture example
---
.run/overture-serve.run.xml | 11 +++
.run/overture-workflow.run.xml | 11 +++
baremaps-cli/src/main/resources/log4j.properties | 8 ++
.../storage/geoparquet/GeoParquetDataStore.java | 10 ++-
.../storage/geoparquet/GeoParquetDataTable.java | 19 ++++-
.../geoparquet/GeoParquetTypeConversion.java | 12 ++-
.../java/org/apache/baremaps/workflow/Task.java | 1 +
.../baremaps/workflow/tasks/ImportGeoParquet.java | 98 ++++++++++++++++++++++
.../baremaps/geoparquet/GeoParquetReader.java | 67 +++++++--------
.../baremaps/geoparquet/data/GeoParquetGroup.java | 2 +
.../geoparquet/data/GeoParquetGroupImpl.java | 10 +--
.../geoparquet/data/GeoParquetMetadata.java | 2 +
examples/overture/indexes.sql | 15 ++++
examples/overture/style.json | 24 ++++++
examples/overture/tiles.json | 24 ++++++
examples/overture/tileset.json | 24 ++++++
examples/overture/views.sql | 31 +++++++
examples/overture/workflow.json | 27 ++++++
18 files changed, 346 insertions(+), 50 deletions(-)
diff --git a/.run/overture-serve.run.xml b/.run/overture-serve.run.xml
new file mode 100644
index 00000000..374bdab4
--- /dev/null
+++ b/.run/overture-serve.run.xml
@@ -0,0 +1,11 @@
+<component name="ProjectRunConfigurationManager">
+ <configuration default="false" name="overture-serve" type="Application" factoryName="Application">
+ <option name="MAIN_CLASS_NAME" value="org.apache.baremaps.cli.Baremaps" />
+ <module name="baremaps-cli" />
+ <option name="PROGRAM_PARAMETERS" value="map serve --tileset tileset.json --style style.json" />
+ <option name="WORKING_DIRECTORY" value="$PROJECT_DIR$/examples/overture" />
+ <method v="2">
+ <option name="Make" enabled="true" />
+ </method>
+ </configuration>
+</component>
\ No newline at end of file
diff --git a/.run/overture-workflow.run.xml b/.run/overture-workflow.run.xml
new file mode 100644
index 00000000..80cccb25
--- /dev/null
+++ b/.run/overture-workflow.run.xml
@@ -0,0 +1,11 @@
+<component name="ProjectRunConfigurationManager">
+ <configuration default="false" name="overture-workflow" type="Application" factoryName="Application">
+ <option name="MAIN_CLASS_NAME" value="org.apache.baremaps.cli.Baremaps" />
+ <module name="baremaps-cli" />
+ <option name="PROGRAM_PARAMETERS" value="workflow execute --file workflow.json" />
+ <option name="WORKING_DIRECTORY" value="$PROJECT_DIR$/examples/overture" />
+ <method v="2">
+ <option name="Make" enabled="true" />
+ </method>
+ </configuration>
+</component>
\ No newline at end of file
diff --git a/baremaps-cli/src/main/resources/log4j.properties b/baremaps-cli/src/main/resources/log4j.properties
new file mode 100644
index 00000000..9109d40f
--- /dev/null
+++ b/baremaps-cli/src/main/resources/log4j.properties
@@ -0,0 +1,8 @@
+# Root logger option
+log4j.rootLogger=INFO, stdout
+
+# Direct log messages to console
+log4j.appender.stdout=org.apache.log4j.ConsoleAppender
+log4j.appender.stdout.Target=System.out
+log4j.appender.stdout.layout=org.apache.log4j.PatternLayout
+log4j.appender.stdout.layout.ConversionPattern=%d{yyyy-MM-dd HH:mm:ss} %-5p %c{1}:%L - %m%n
\ No newline at end of file
diff --git a/baremaps-core/src/main/java/org/apache/baremaps/storage/geoparquet/GeoParquetDataStore.java b/baremaps-core/src/main/java/org/apache/baremaps/storage/geoparquet/GeoParquetDataStore.java
index 05874cf9..46283391 100644
--- a/baremaps-core/src/main/java/org/apache/baremaps/storage/geoparquet/GeoParquetDataStore.java
+++ b/baremaps-core/src/main/java/org/apache/baremaps/storage/geoparquet/GeoParquetDataStore.java
@@ -30,22 +30,24 @@ import org.apache.baremaps.data.storage.DataTable;
public class GeoParquetDataStore implements DataStore {
private final URI uri;
+ private final String tableName;
- public GeoParquetDataStore(URI uri) {
+ public GeoParquetDataStore(URI uri, String tableName) {
this.uri = uri;
+ this.tableName = tableName;
}
@Override
public List<String> list() throws DataStoreException {
- return List.of(uri.toString());
+ return List.of(tableName);
}
@Override
public DataTable get(String name) throws DataStoreException {
- if (!uri.toString().equals(name)) {
+ if (!tableName.equals(name)) {
throw new DataStoreException("Table not found");
}
- return new GeoParquetDataTable(uri);
+ return new GeoParquetDataTable(uri, tableName);
}
@Override
diff --git a/baremaps-core/src/main/java/org/apache/baremaps/storage/geoparquet/GeoParquetDataTable.java b/baremaps-core/src/main/java/org/apache/baremaps/storage/geoparquet/GeoParquetDataTable.java
index 0d05f680..dcfa4383 100644
--- a/baremaps-core/src/main/java/org/apache/baremaps/storage/geoparquet/GeoParquetDataTable.java
+++ b/baremaps-core/src/main/java/org/apache/baremaps/storage/geoparquet/GeoParquetDataTable.java
@@ -32,12 +32,15 @@ public class GeoParquetDataTable implements DataTable {
private final URI path;
+ private final String name;
+
private DataSchema schema;
private GeoParquetReader reader;
- public GeoParquetDataTable(URI path) {
+ public GeoParquetDataTable(URI path, String name) {
this.path = path;
+ this.name = name;
}
private GeoParquetReader reader() {
@@ -75,7 +78,7 @@ public class GeoParquetDataTable implements DataTable {
public Stream<DataRow> parallelStream() {
try {
return reader().read().map(group -> new DataRowImpl(
- GeoParquetTypeConversion.asSchema(path.toString(), group.getSchema()),
+ GeoParquetTypeConversion.asSchema(name, group.getSchema()),
GeoParquetTypeConversion.asRowValues(group)));
} catch (IOException | URISyntaxException e) {
throw new GeoParquetException("Fail to read() the reader", e);
@@ -98,12 +101,20 @@ public class GeoParquetDataTable implements DataTable {
if (schema == null) {
try {
Schema schema = reader().getGeoParquetSchema();
- this.schema = GeoParquetTypeConversion.asSchema(path.toString(), schema);
+ this.schema = GeoParquetTypeConversion.asSchema(name, schema);
return this.schema;
} catch (URISyntaxException e) {
- throw new GeoParquetException("Fail toe get the schema.", e);
+ throw new GeoParquetException("Failed to get the schema.", e);
}
}
return schema;
}
+
+ public int srid(String column) {
+ try {
+ return reader().getGeoParquetMetadata().getSrid(column);
+ } catch (Exception e) {
+ throw new RuntimeException(e);
+ }
+ }
}
diff --git a/baremaps-core/src/main/java/org/apache/baremaps/storage/geoparquet/GeoParquetTypeConversion.java b/baremaps-core/src/main/java/org/apache/baremaps/storage/geoparquet/GeoParquetTypeConversion.java
index 78be7062..9ff30a62 100644
--- a/baremaps-core/src/main/java/org/apache/baremaps/storage/geoparquet/GeoParquetTypeConversion.java
+++ b/baremaps-core/src/main/java/org/apache/baremaps/storage/geoparquet/GeoParquetTypeConversion.java
@@ -46,9 +46,9 @@ public class GeoParquetTypeConversion {
private static DataColumn asDataColumn(Field field) {
Cardinality cardinality = switch (field.cardinality()) {
- case REQUIRED -> Cardinality.REQUIRED;
+ case REQUIRED -> Cardinality.OPTIONAL;
case OPTIONAL -> Cardinality.OPTIONAL;
- case REPEATED -> Cardinality.REPEATED;
+ case REPEATED -> Cardinality.OPTIONAL;
};
return switch (field.type()) {
case BINARY -> new DataColumnFixed(field.name(), cardinality, Type.BINARY);
@@ -71,6 +71,10 @@ public class GeoParquetTypeConversion {
List<Field> fields = schema.fields();
for (int i = 0; i < fields.size(); i++) {
Field field = fields.get(i);
+ if(group.getValues(i).isEmpty()){
+ values.add(null);
+ continue;
+ }
switch (field.type()) {
case BINARY -> values.add(group.getBinaryValue(i).getBytes());
case BOOLEAN -> values.add(group.getBooleanValue(i));
@@ -93,6 +97,10 @@ public class GeoParquetTypeConversion {
List<Field> fields = schema.fields();
for (int i = 0; i < fields.size(); i++) {
Field field = fields.get(i);
+ if(group.getValues(i).isEmpty()){
+ nested.put(field.name(), null);
+ continue;
+ }
nested.put(field.name(), switch (field.type()) {
case BINARY -> group.getBinaryValue(i).getBytes();
case BOOLEAN -> group.getBooleanValue(i);
diff --git a/baremaps-core/src/main/java/org/apache/baremaps/workflow/Task.java b/baremaps-core/src/main/java/org/apache/baremaps/workflow/Task.java
index 1baedefe..0a1f156f 100644
--- a/baremaps-core/src/main/java/org/apache/baremaps/workflow/Task.java
+++ b/baremaps-core/src/main/java/org/apache/baremaps/workflow/Task.java
@@ -47,6 +47,7 @@ import org.apache.baremaps.workflow.tasks.*;
@Type(value = ImportDaylightFeatures.class, name = "ImportDaylightFeatures"),
@Type(value = ImportDaylightTranslations.class, name = "ImportDaylightTranslations"),
@Type(value = ImportGeoPackage.class, name = "ImportGeoPackage"),
+ @Type(value = ImportGeoParquet.class, name = "ImportGeoParquet"),
@Type(value = ImportOsmOsc.class, name = "ImportOsmOsc"),
@Type(value = ImportOsmPbf.class, name = "ImportOsmPbf"),
@Type(value = ImportShapefile.class, name = "ImportShapefile"),
diff --git a/baremaps-core/src/main/java/org/apache/baremaps/workflow/tasks/ImportGeoParquet.java b/baremaps-core/src/main/java/org/apache/baremaps/workflow/tasks/ImportGeoParquet.java
new file mode 100644
index 00000000..bbee5784
--- /dev/null
+++ b/baremaps-core/src/main/java/org/apache/baremaps/workflow/tasks/ImportGeoParquet.java
@@ -0,0 +1,98 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to you under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.baremaps.workflow.tasks;
+
+import org.apache.baremaps.data.storage.DataTableGeometryMapper;
+import org.apache.baremaps.data.storage.DataTableMapper;
+import org.apache.baremaps.openstreetmap.function.ProjectionTransformer;
+import org.apache.baremaps.storage.geoparquet.GeoParquetDataStore;
+import org.apache.baremaps.storage.geoparquet.GeoParquetDataTable;
+import org.apache.baremaps.storage.postgres.PostgresDataStore;
+import org.apache.baremaps.workflow.Task;
+import org.apache.baremaps.workflow.WorkflowContext;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.net.URI;
+import java.util.StringJoiner;
+
+/**
+ * Import a GeoParquet into a database.
+ */
+public class ImportGeoParquet implements Task {
+
+ private static final Logger logger = LoggerFactory.getLogger(ImportGeoParquet.class);
+
+ private URI uri;
+ private String tableName;
+ private Object database;
+ private Integer databaseSrid;
+
+ /**
+ * Constructs a {@code ImportGeoParquet}.
+ */
+ public ImportGeoParquet() {
+
+ }
+
+ /**
+ * Constructs an {@code ImportGeoParquet}.
+ *
+ * @param uri the GeoParquet uri
+ * @param database the database
+ * @param databaseSrid the target SRID
+ */
+ public ImportGeoParquet(URI uri, String tableName, Object database, Integer databaseSrid) {
+ this.uri = uri;
+ this.tableName = tableName;
+ this.database = database;
+ this.databaseSrid = databaseSrid;
+ }
+
+ /**
+ * {@inheritDoc}
+ */
+ @Override
+ public void execute(WorkflowContext context) throws Exception {
+ var geoParquetDataStore = new GeoParquetDataStore(uri, tableName);
+ var dataSource = context.getDataSource(database);
+ var postgresDataStore = new PostgresDataStore(dataSource);
+ for (var name : geoParquetDataStore.list()) {
+ var geoParquetTable = (GeoParquetDataTable)geoParquetDataStore.get(name);
+ // TODO : How can we apply a different SRID for each column based on the geometry
+ var projectionTransformer = new ProjectionTransformer(geoParquetTable.srid("geometry"), databaseSrid);
+ var rowTransformer =
+ new DataTableGeometryMapper(geoParquetTable, projectionTransformer);
+ var transformedDataTable =
+ new DataTableMapper(geoParquetDataStore.get(name), rowTransformer);
+ postgresDataStore.add(transformedDataTable);
+ }
+ }
+
+ /**
+ * {@inheritDoc}
+ */
+ @Override
+ public String toString() {
+ return new StringJoiner(", ", ImportGeoParquet.class.getSimpleName() + "[", "]")
+ .add("uri=" + uri)
+ .add("database=" + database)
+ .add("databaseSrid=" + databaseSrid)
+ .toString();
+ }
+}
diff --git a/baremaps-geoparquet/src/main/java/org/apache/baremaps/geoparquet/GeoParquetReader.java b/baremaps-geoparquet/src/main/java/org/apache/baremaps/geoparquet/GeoParquetReader.java
index 4e760d8b..a66734bf 100644
--- a/baremaps-geoparquet/src/main/java/org/apache/baremaps/geoparquet/GeoParquetReader.java
+++ b/baremaps-geoparquet/src/main/java/org/apache/baremaps/geoparquet/GeoParquetReader.java
@@ -36,6 +36,9 @@ import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.s3a.AnonymousAWSCredentialsProvider;
+import org.apache.hadoop.fs.s3a.S3AFileSystem;
+import org.apache.hadoop.fs.s3a.SimpleAWSCredentialsProvider;
import org.apache.parquet.hadoop.ParquetFileReader;
import org.apache.parquet.hadoop.ParquetReader;
import org.apache.parquet.schema.MessageType;
@@ -95,13 +98,27 @@ public class GeoParquetReader {
try {
if (files == null) {
files = new HashMap<>();
- Path globPath = new Path(uri.getPath());
- URI rootUri = getRootUri(uri);
- FileSystem fileSystem = FileSystem.get(rootUri, configuration);
-
- // Iterate over all the files in the path
- for (FileStatus file : fileSystem.globStatus(globPath)) {
- files.put(file, buildFileInfo(file));
+ FileSystem fs = FileSystem.get(uri, configuration);
+ FileStatus[] fileStatuses = fs.globStatus(new Path(uri));
+
+ for (FileStatus fileStatus : fileStatuses) {
+ Path filePath = fileStatus.getPath();
+ ParquetFileReader reader = ParquetFileReader.open(configuration, filePath);
+ Long recordCount = reader.getRecordCount();
+ MessageType messageType = reader.getFileMetaData().getSchema();
+ Map<String, String> keyValueMetadata = reader.getFileMetaData().getKeyValueMetaData();
+ GeoParquetMetadata geoParquetMetadata = null;
+ GeoParquetGroup.Schema geoParquetSchema = null;
+ if (keyValueMetadata.containsKey("geo")) {
+ ObjectMapper objectMapper = new ObjectMapper();
+ objectMapper.configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false);
+ geoParquetMetadata =
+ objectMapper.readValue(keyValueMetadata.get("geo"), GeoParquetMetadata.class);
+ geoParquetSchema =
+ GeoParquetGroupFactory.createGeoParquetSchema(messageType, geoParquetMetadata);
+ }
+ files.put(fileStatus, new FileInfo(fileStatus, recordCount, keyValueMetadata, messageType,
+ geoParquetMetadata, geoParquetSchema));
}
// Verify that the files all have the same schema
@@ -110,12 +127,12 @@ public class GeoParquetReader {
if (commonMessageType == null) {
commonMessageType = entry.messageType;
} else if (!commonMessageType.equals(entry.messageType)) {
- throw new GeoParquetException("The files do not have the same schema");
+ throw new RuntimeException("The files do not have the same schema");
}
}
}
} catch (IOException e) {
- throw new GeoParquetException("IOException while attempting to list files.", e);
+ throw new RuntimeException("IOException while attempting to list files.", e);
}
return files;
}
@@ -254,31 +271,11 @@ public class GeoParquetReader {
}
private static Configuration createConfiguration() {
- Configuration configuration = new Configuration();
- configuration.set("fs.s3a.aws.credentials.provider",
- "org.apache.hadoop.fs.s3a.AnonymousAWSCredentialsProvider");
- configuration.setBoolean("fs.s3a.path.style.access", true);
- return configuration;
- }
-
- private static URI getRootUri(URI uri) throws URISyntaxException {
- // TODO:
- // This is a quick and dirty way to get the root uri of the path.
- // We take everything before the first wildcard in the path.
- // This is not a perfect solution, and we should probably look for a better way to do this.
- String path = uri.getPath();
- int index = path.indexOf("*");
- if (index != -1) {
- path = path.substring(0, path.lastIndexOf("/", index) + 1);
- }
- return new URI(
- uri.getScheme(),
- uri.getUserInfo(),
- uri.getHost(),
- uri.getPort(),
- path,
- null,
- null);
+ Configuration conf = new Configuration();
+ conf.set("fs.s3a.endpoint", "s3.us-west-2.amazonaws.com");
+ conf.set("fs.s3a.aws.credentials.provider", AnonymousAWSCredentialsProvider.class.getName());
+ conf.set("fs.s3a.impl", S3AFileSystem.class.getName());
+ conf.set("fs.s3a.path.style.access", "true");
+ return conf;
}
-
}
diff --git a/baremaps-geoparquet/src/main/java/org/apache/baremaps/geoparquet/data/GeoParquetGroup.java b/baremaps-geoparquet/src/main/java/org/apache/baremaps/geoparquet/data/GeoParquetGroup.java
index 5a3b3709..46e57bb3 100644
--- a/baremaps-geoparquet/src/main/java/org/apache/baremaps/geoparquet/data/GeoParquetGroup.java
+++ b/baremaps-geoparquet/src/main/java/org/apache/baremaps/geoparquet/data/GeoParquetGroup.java
@@ -59,6 +59,8 @@ public interface GeoParquetGroup {
*/
GeoParquetGroup createGroup(int fieldIndex);
+ List<Primitive> getValues(int fieldIndex);
+
Binary getBinaryValue(int fieldIndex);
List<Binary> getBinaryValues(int fieldIndex);
diff --git a/baremaps-geoparquet/src/main/java/org/apache/baremaps/geoparquet/data/GeoParquetGroupImpl.java b/baremaps-geoparquet/src/main/java/org/apache/baremaps/geoparquet/data/GeoParquetGroupImpl.java
index a9c4ffa0..352fd8ef 100644
--- a/baremaps-geoparquet/src/main/java/org/apache/baremaps/geoparquet/data/GeoParquetGroupImpl.java
+++ b/baremaps-geoparquet/src/main/java/org/apache/baremaps/geoparquet/data/GeoParquetGroupImpl.java
@@ -278,7 +278,7 @@ public class GeoParquetGroupImpl implements GeoParquetGroup {
}
}
- private List<Primitive> getValues(int fieldIndex) {
+ public List<Primitive> getValues(int fieldIndex) {
return (List<Primitive>) data[fieldIndex];
}
@@ -435,10 +435,10 @@ public class GeoParquetGroupImpl implements GeoParquetGroup {
@Override
public List<Envelope> getEnvelopeValues(int fieldIndex) {
return getGroupValues(fieldIndex).stream().map(group -> {
- var xMin = group.getDoubleValue(0);
- var yMin = group.getDoubleValue(1);
- var xMax = group.getDoubleValue(2);
- var yMax = group.getDoubleValue(3);
+ var xMin = group.getFloatValue(0);
+ var yMin = group.getFloatValue(1);
+ var xMax = group.getFloatValue(2);
+ var yMax = group.getFloatValue(3);
return new Envelope(xMin, xMax, yMin, yMax);
}).toList();
}
diff --git a/baremaps-geoparquet/src/main/java/org/apache/baremaps/geoparquet/data/GeoParquetMetadata.java b/baremaps-geoparquet/src/main/java/org/apache/baremaps/geoparquet/data/GeoParquetMetadata.java
index 4b32299e..39731407 100644
--- a/baremaps-geoparquet/src/main/java/org/apache/baremaps/geoparquet/data/GeoParquetMetadata.java
+++ b/baremaps-geoparquet/src/main/java/org/apache/baremaps/geoparquet/data/GeoParquetMetadata.java
@@ -58,6 +58,7 @@ public class GeoParquetMetadata {
this.columns = columns;
}
+
public int getSrid(String column) {
return Optional.ofNullable(getColumns().get(column).getCrs()).map(crs -> {
JsonNode id = crs.get("id");
@@ -72,6 +73,7 @@ public class GeoParquetMetadata {
}).orElse(4326);
}
+
public boolean isGeometryColumn(String column) {
return columns.containsKey(column);
}
diff --git a/examples/overture/indexes.sql b/examples/overture/indexes.sql
new file mode 100644
index 00000000..b7b7e5db
--- /dev/null
+++ b/examples/overture/indexes.sql
@@ -0,0 +1,15 @@
+-- Licensed to the Apache Software Foundation (ASF) under one or more
+-- contributor license agreements. See the NOTICE file distributed with
+-- this work for additional information regarding copyright ownership.
+-- The ASF licenses this file to you under the Apache License, Version 2.0
+-- (the "License"); you may not use this file except in compliance with
+-- the License. You may obtain a copy of the License at
+--
+-- http://www.apache.org/licenses/LICENSE-2.0
+--
+-- Unless required by applicable law or agreed to in writing, software
+-- distributed under the License is distributed on an "AS IS" BASIS,
+-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+-- See the License for the specific language governing permissions and
+-- limitations under the License.
+CREATE INDEX IF NOT EXISTS overture_admins_administrative_boundary_materialized_view_gist ON overture_admins_administrative_boundary_materialized_view USING GIST(geom);
\ No newline at end of file
diff --git a/examples/overture/style.json b/examples/overture/style.json
new file mode 100644
index 00000000..400a7f52
--- /dev/null
+++ b/examples/overture/style.json
@@ -0,0 +1,24 @@
+{
+ "version" : 8,
+ "sources" : {
+ "baremaps" : {
+ "type" : "vector",
+ "url" : "http://localhost:9000/tiles.json"
+ }
+ },
+ "layers" : [ {
+ "id" : "administrative_boundary",
+ "type" : "line",
+ "source" : "baremaps",
+ "source-layer" : "administrative_boundary",
+ "layout" : {
+ "visibility" : "visible"
+ },
+ "paint" : {
+ "line-color": "black",
+ "line-width": 1
+ }
+ }],
+ "center" : [ 0, 0 ],
+ "zoom" : 2
+}
\ No newline at end of file
diff --git a/examples/overture/tiles.json b/examples/overture/tiles.json
new file mode 100644
index 00000000..2d6da885
--- /dev/null
+++ b/examples/overture/tiles.json
@@ -0,0 +1,24 @@
+{
+ "tilejson": "2.2.0",
+ "tiles": [
+ "http://localhost:9000/tiles/{z}/{x}/{y}.mvt"
+ ],
+ "minzoom": 0.0,
+ "maxzoom": 14.0,
+ "center": [0, 0],
+ "bounds": [-180, -85, 180, 85],
+ "zoom": 2,
+ "database": "jdbc:postgresql://localhost:5432/baremaps?&user=baremaps&password=baremaps",
+ "vector_layers": [
+ {
+ "id": "administrative_boundary",
+ "queries": [
+ {
+ "minzoom": 0,
+ "maxzoom": 14,
+ "sql": "SELECT id, tags, geom FROM overture_admins_administrative_boundary_materialized_view"
+ }
+ ]
+ }
+ ]
+}
\ No newline at end of file
diff --git a/examples/overture/tileset.json b/examples/overture/tileset.json
new file mode 100644
index 00000000..fb1dccc0
--- /dev/null
+++ b/examples/overture/tileset.json
@@ -0,0 +1,24 @@
+{
+ "tilejson": "2.2.0",
+ "minzoom": 0,
+ "maxzoom": 14,
+ "center": [0, 0],
+ "bounds": [-180, -85, 180, 85],
+ "zoom": 2,
+ "tiles": [
+ "http://localhost:9000/tiles/{z}/{x}/{y}.mvt"
+ ],
+ "database": "jdbc:postgresql://localhost:5432/baremaps?&user=baremaps&password=baremaps",
+ "vector_layers": [
+ {
+ "id": "administrative_boundary",
+ "queries": [
+ {
+ "minzoom": 0,
+ "maxzoom": 14,
+ "sql": "SELECT id, tags, geom FROM overture_admins_administrative_boundary_materialized_view"
+ }
+ ]
+ }
+ ]
+}
diff --git a/examples/overture/views.sql b/examples/overture/views.sql
new file mode 100644
index 00000000..20258530
--- /dev/null
+++ b/examples/overture/views.sql
@@ -0,0 +1,31 @@
+-- Licensed to the Apache Software Foundation (ASF) under one or more
+-- contributor license agreements. See the NOTICE file distributed with
+-- this work for additional information regarding copyright ownership.
+-- The ASF licenses this file to you under the Apache License, Version 2.0
+-- (the "License"); you may not use this file except in compliance with
+-- the License. You may obtain a copy of the License at
+--
+-- http://www.apache.org/licenses/LICENSE-2.0
+--
+-- Unless required by applicable law or agreed to in writing, software
+-- distributed under the License is distributed on an "AS IS" BASIS,
+-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+-- See the License for the specific language governing permissions and
+-- limitations under the License.
+CREATE MATERIALIZED VIEW IF NOT EXISTS overture_admins_administrative_boundary_materialized_view AS
+SELECT
+ -- Generate a unique id for each row
+ row_number() OVER () AS id,
+
+ -- Rename the geometry column
+ st_simplifypreservetopology(geometry, 78270 / power(2, 2)) AS geom,
+
+ -- Aggregate other fields into a jsonb tags field
+ jsonb_build_object(
+ 'admin_level', admin_level,
+ 'version', version,
+ 'sources', sources,
+ 'population', population,
+ 'names', names
+ ) AS tags
+FROM overture_admins_administrative_boundary;
\ No newline at end of file
diff --git a/examples/overture/workflow.json b/examples/overture/workflow.json
new file mode 100644
index 00000000..3aaa3c1d
--- /dev/null
+++ b/examples/overture/workflow.json
@@ -0,0 +1,27 @@
+{
+ "steps": [
+ {
+ "id": "overture",
+ "needs": [],
+ "tasks": [
+ {
+ "type": "ImportGeoParquet",
+ "uri": "s3a://overturemaps-us-west-2/release/2024-05-16-beta.0/theme=admins/type=administrative_boundary/*",
+ "tableName": "overture_admins_administrative_boundary",
+ "database": "jdbc:postgresql://localhost:5432/baremaps?&user=baremaps&password=baremaps",
+ "databaseSrid": 3857
+ },
+ {
+ "type": "ExecuteSql",
+ "file": "views.sql",
+ "database": "jdbc:postgresql://localhost:5432/baremaps?&user=baremaps&password=baremaps"
+ },
+ {
+ "type": "ExecuteSql",
+ "file": "indexes.sql",
+ "database": "jdbc:postgresql://localhost:5432/baremaps?&user=baremaps&password=baremaps"
+ }
+ ]
+ }
+ ]
+}
\ No newline at end of file
(incubator-baremaps) 01/03: Merge remote-tracking branch 'origin/849-nested-type-and-jsonb' into 849-default-crs
Posted by ad...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
adrabble pushed a commit to branch 849-overture-example
in repository https://gitbox.apache.org/repos/asf/incubator-baremaps.git
commit cda5ec815c820ab9f2c795387c0946f6bce5be32
Merge: 3f702305 d24d57d9
Author: drabble <an...@gmail.com>
AuthorDate: Sat Jun 1 22:01:55 2024 +0200
Merge remote-tracking branch 'origin/849-nested-type-and-jsonb' into 849-default-crs
.../apache/baremaps/database/copy/CopyWriter.java | 14 ++++
...ValueHandler.java => EnvelopeValueHandler.java} | 20 ++++--
.../database/copy/GeometryValueHandler.java | 8 ++-
.../baremaps/database/copy/JsonbValueHandler.java | 80 ++++++++++++++++++++++
.../database/metadata/DatabaseMetadata.java | 36 ++++++----
.../baremaps/geocoder/GeonamesQueryBuilder.java | 2 +-
.../org/apache/baremaps/iploc/IpLocMapper.java | 4 +-
.../storage/flatgeobuf/FlatGeoBufDataTable.java | 7 +-
.../flatgeobuf/FlatGeoBufTypeConversion.java | 6 +-
.../storage/geopackage/GeoPackageDataTable.java | 5 +-
.../geoparquet/GeoParquetTypeConversion.java | 75 ++++++++++++++------
.../storage/postgres/PostgresDataStore.java | 22 ++++--
.../storage/postgres/PostgresTypeConversion.java | 5 +-
.../shapefile/internal/DbaseByteReader.java | 2 +-
.../shapefile/internal/ShapefileByteReader.java | 7 +-
.../shapefile/internal/ShapefileInputStream.java | 4 +-
.../shapefile/internal/ShapefileReader.java | 4 +-
.../org/apache/baremaps/calcite/CalciteTest.java | 11 +--
.../database/postgres/NodeRepositoryTest.java | 4 +-
.../org/apache/baremaps/storage/MockDataTable.java | 11 +--
.../geoparquet/GeoParquetToPostgresTest.java | 7 +-
.../baremaps/data/calcite/SqlTypeConversion.java | 14 ----
.../apache/baremaps/data/storage/DataColumn.java | 22 +++---
.../{DataColumnImpl.java => DataColumnFixed.java} | 3 +-
.../{DataColumnImpl.java => DataColumnNested.java} | 12 ++--
.../baremaps/data/type/DataTypeProvider.java | 35 +++++-----
.../baremaps/geoparquet/data/GeoParquetGroup.java | 26 +++++++
.../geoparquet/data/GeoParquetGroupFactory.java | 30 ++++++--
.../geoparquet/data/GeoParquetGroupImpl.java | 77 ++++++++++++++++++---
29 files changed, 422 insertions(+), 131 deletions(-)
(incubator-baremaps) 02/03: Merge remote-tracking branch 'origin/849-nested-type-and-jsonb' into 849-default-crs
Posted by ad...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
adrabble pushed a commit to branch 849-overture-example
in repository https://gitbox.apache.org/repos/asf/incubator-baremaps.git
commit 6cf69a22d1ef45ed24eac2f66940fa552c19c140
Merge: cda5ec81 adda5123
Author: drabble <an...@gmail.com>
AuthorDate: Sat Jun 1 23:39:05 2024 +0200
Merge remote-tracking branch 'origin/849-nested-type-and-jsonb' into 849-default-crs
.../baremaps/geocoder/GeonamesQueryBuilder.java | 2 +-
.../java/org/apache/baremaps/iploc/IpLocMapper.java | 4 ++--
.../storage/shapefile/internal/DbaseByteReader.java | 2 +-
.../shapefile/internal/ShapefileInputStream.java | 4 ++--
.../storage/shapefile/internal/ShapefileReader.java | 4 ++--
.../geoparquet/data/GeoParquetGroupFactory.java | 21 ++++++++++++---------
.../geoparquet/data/GeoParquetGroupImpl.java | 4 ++--
7 files changed, 22 insertions(+), 19 deletions(-)