You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@baremaps.apache.org by bc...@apache.org on 2023/09/28 19:44:13 UTC
[incubator-baremaps] 12/35: wqRename import osm task and add osc task
This is an automated email from the ASF dual-hosted git repository.
bchapuis pushed a commit to branch 745-daylight
in repository https://gitbox.apache.org/repos/asf/incubator-baremaps.git
commit e4bda42ce9c748a21d2b99117890ed9f5905fa21
Author: Bertil Chapuis <bc...@gmail.com>
AuthorDate: Mon Aug 28 17:03:47 2023 +0200
wqRename import osm task and add osc task
---
.../org/apache/baremaps/cli/database/Database.java | 2 +-
...{ImportOpenStreetMap.java => ImportOsmPbf.java} | 8 +-
.../java/org/apache/baremaps/workflow/Task.java | 2 +-
.../apache/baremaps/workflow/tasks/ImportOsc.java | 91 ++++++++++++++++++++++
...{ImportOpenStreetMap.java => ImportOsmPbf.java} | 2 +-
.../apache/baremaps/workflow/ObjectMapperTest.java | 8 +-
.../org/apache/baremaps/workflow/WorkflowTest.java | 4 +-
.../baremaps/workflow/tasks/ImportMonacoTest.java | 2 +-
...rtOpenStreetMapTest.java => ImportPbfTest.java} | 4 +-
.../workflow/tasks/ImportUpdateDataTest.java | 2 +-
.../tasks/ImportUpdateLiechtensteinTest.java | 2 +-
basemap/daylight/workflow.js | 2 +-
basemap/workflow.js | 2 +-
examples/extrusion/workflow.json | 2 +-
examples/openstreetmap/workflow.json | 2 +-
15 files changed, 114 insertions(+), 21 deletions(-)
diff --git a/baremaps-cli/src/main/java/org/apache/baremaps/cli/database/Database.java b/baremaps-cli/src/main/java/org/apache/baremaps/cli/database/Database.java
index dc83f45b..3d2df584 100644
--- a/baremaps-cli/src/main/java/org/apache/baremaps/cli/database/Database.java
+++ b/baremaps-cli/src/main/java/org/apache/baremaps/cli/database/Database.java
@@ -18,7 +18,7 @@ import picocli.CommandLine;
import picocli.CommandLine.Command;
@Command(name = "database", description = "Database commands.",
- subcommands = {ExecuteSql.class, ImportOpenStreetMap.class, UpdateOpenStreetMap.class},
+ subcommands = {ExecuteSql.class, ImportOsmPbf.class, UpdateOpenStreetMap.class},
sortOptions = false)
public class Database implements Runnable {
diff --git a/baremaps-cli/src/main/java/org/apache/baremaps/cli/database/ImportOpenStreetMap.java b/baremaps-cli/src/main/java/org/apache/baremaps/cli/database/ImportOsmPbf.java
similarity index 87%
rename from baremaps-cli/src/main/java/org/apache/baremaps/cli/database/ImportOpenStreetMap.java
rename to baremaps-cli/src/main/java/org/apache/baremaps/cli/database/ImportOsmPbf.java
index b64cb565..dbfc68e8 100644
--- a/baremaps-cli/src/main/java/org/apache/baremaps/cli/database/ImportOpenStreetMap.java
+++ b/baremaps-cli/src/main/java/org/apache/baremaps/cli/database/ImportOsmPbf.java
@@ -23,7 +23,7 @@ import picocli.CommandLine.Mixin;
import picocli.CommandLine.Option;
@Command(name = "import-osm", description = "Import OpenStreetMap data in Postgres.")
-public class ImportOpenStreetMap implements Callable<Integer> {
+public class ImportOsmPbf implements Callable<Integer> {
@Mixin
private Options options;
@@ -42,8 +42,10 @@ public class ImportOpenStreetMap implements Callable<Integer> {
@Override
public Integer call() throws Exception {
- new org.apache.baremaps.workflow.tasks.ImportOpenStreetMap(file.toAbsolutePath(),
- database, srid).execute(new WorkflowContext());
+ new org.apache.baremaps.workflow.tasks.ImportOsmPbf(
+ file.toAbsolutePath(),
+ database,
+ srid).execute(new WorkflowContext());
return 0;
}
}
diff --git a/baremaps-core/src/main/java/org/apache/baremaps/workflow/Task.java b/baremaps-core/src/main/java/org/apache/baremaps/workflow/Task.java
index 3fb601b1..58a0ca71 100644
--- a/baremaps-core/src/main/java/org/apache/baremaps/workflow/Task.java
+++ b/baremaps-core/src/main/java/org/apache/baremaps/workflow/Task.java
@@ -33,7 +33,7 @@ import org.apache.baremaps.workflow.tasks.*;
@JsonSubTypes.Type(value = ExecuteSqlScript.class, name = "ExecuteSqlScript"),
@JsonSubTypes.Type(value = ExportVectorTiles.class, name = "ExportVectorTiles"),
@JsonSubTypes.Type(value = ImportGeoPackage.class, name = "ImportGeoPackage"),
- @JsonSubTypes.Type(value = ImportOpenStreetMap.class, name = "ImportOpenStreetMap"),
+ @JsonSubTypes.Type(value = ImportOsmPbf.class, name = "ImportOsmPbf"),
@JsonSubTypes.Type(value = ImportShapefile.class, name = "ImportShapefile"),
@JsonSubTypes.Type(value = LogMessage.class, name = "LogMessage"),
@JsonSubTypes.Type(value = UnzipFile.class, name = "UnzipFile"),
diff --git a/baremaps-core/src/main/java/org/apache/baremaps/workflow/tasks/ImportOsc.java b/baremaps-core/src/main/java/org/apache/baremaps/workflow/tasks/ImportOsc.java
new file mode 100644
index 00000000..f0aa8285
--- /dev/null
+++ b/baremaps-core/src/main/java/org/apache/baremaps/workflow/tasks/ImportOsc.java
@@ -0,0 +1,91 @@
+/*
+ * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except
+ * in compliance with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software distributed under the License
+ * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
+ * or implied. See the License for the specific language governing permissions and limitations under
+ * the License.
+ */
+
+package org.apache.baremaps.workflow.tasks;
+
+import static org.apache.baremaps.stream.ConsumerUtils.consumeThenReturn;
+
+import java.io.BufferedInputStream;
+import java.net.MalformedURLException;
+import java.net.URI;
+import java.net.URL;
+import java.nio.file.Path;
+import java.util.zip.GZIPInputStream;
+import org.apache.baremaps.openstreetmap.function.ChangeEntitiesHandler;
+import org.apache.baremaps.openstreetmap.function.EntityGeometryBuilder;
+import org.apache.baremaps.openstreetmap.function.EntityProjectionTransformer;
+import org.apache.baremaps.openstreetmap.model.Header;
+import org.apache.baremaps.openstreetmap.model.Node;
+import org.apache.baremaps.openstreetmap.model.Relation;
+import org.apache.baremaps.openstreetmap.model.Way;
+import org.apache.baremaps.openstreetmap.postgres.*;
+import org.apache.baremaps.openstreetmap.repository.ChangeImporter;
+import org.apache.baremaps.openstreetmap.repository.Repository;
+import org.apache.baremaps.openstreetmap.state.StateReader;
+import org.apache.baremaps.openstreetmap.xml.XmlChangeReader;
+import org.apache.baremaps.workflow.Task;
+import org.apache.baremaps.workflow.WorkflowContext;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+public record ImportOsc(Path file, String database, Integer srid) implements Task {
+
+ private static final Logger logger = LoggerFactory.getLogger(ImportOsc.class);
+
+ @Override
+ public void execute(WorkflowContext context) throws Exception {
+ var datasource = context.getDataSource(database);
+
+ Repository<Long, Node> nodeRepository = new PostgresNodeRepository(datasource);
+ Repository<Long, Way> wayRepository = new PostgresWayRepository(datasource);
+ Repository<Long, Relation> relationRepository = new PostgresRelationRepository(datasource);
+ execute(
+ nodeRepository,
+ wayRepository,
+ relationRepository,
+ srid);
+ }
+
+ public static void execute(
+ Repository<Long, Node> nodeRepository,
+ Repository<Long, Way> wayRepository,
+ Repository<Long, Relation> relationRepository,
+ int srid) throws Exception {
+
+ var createGeometry = new EntityGeometryBuilder(coordinateMap, referenceMap);
+ var reprojectGeometry = new EntityProjectionTransformer(4326, srid);
+ var prepareGeometries = new ChangeEntitiesHandler(createGeometry.andThen(reprojectGeometry));
+ var prepareChange = consumeThenReturn(prepareGeometries);
+ var saveChange = new ChangeImporter(nodeRepository, wayRepository, relationRepository);
+
+ var changeUrl = resolve(replicationUrl, sequenceNumber, "osc.gz");
+ try (var changeInputStream =
+ new GZIPInputStream(new BufferedInputStream(changeUrl.openStream()))) {
+ new XmlChangeReader().stream(changeInputStream).map(prepareChange).forEach(saveChange);
+ }
+
+ var stateUrl = resolve(replicationUrl, sequenceNumber, "state.txt");
+ try (var stateInputStream = new BufferedInputStream(stateUrl.openStream())) {
+ var state = new StateReader().state(stateInputStream);
+ headerRepository.put(new Header(state.getSequenceNumber(), state.getTimestamp(),
+ header.getReplicationUrl(), header.getSource(), header.getWritingProgram()));
+ }
+ }
+
+ public static URL resolve(String replicationUrl, Long sequenceNumber, String extension)
+ throws MalformedURLException {
+ var s = String.format("%09d", sequenceNumber);
+ var uri = String.format("%s/%s/%s/%s.%s", replicationUrl, s.substring(0, 3), s.substring(3, 6),
+ s.substring(6, 9), extension);
+ return URI.create(uri).toURL();
+ }
+}
diff --git a/baremaps-core/src/main/java/org/apache/baremaps/workflow/tasks/ImportOpenStreetMap.java b/baremaps-core/src/main/java/org/apache/baremaps/workflow/tasks/ImportOsmPbf.java
similarity index 99%
rename from baremaps-core/src/main/java/org/apache/baremaps/workflow/tasks/ImportOpenStreetMap.java
rename to baremaps-core/src/main/java/org/apache/baremaps/workflow/tasks/ImportOsmPbf.java
index c387fe0c..37c6de61 100644
--- a/baremaps-core/src/main/java/org/apache/baremaps/workflow/tasks/ImportOpenStreetMap.java
+++ b/baremaps-core/src/main/java/org/apache/baremaps/workflow/tasks/ImportOsmPbf.java
@@ -45,7 +45,7 @@ public record ImportOpenStreetMap(Path file, Object database, Integer databaseSr
implements
Task {
- private static final Logger logger = LoggerFactory.getLogger(ImportOpenStreetMap.class);
+ private static final Logger logger = LoggerFactory.getLogger(ImportOsmPbf.class);
@Override
public void execute(WorkflowContext context) throws Exception {
diff --git a/baremaps-core/src/test/java/org/apache/baremaps/workflow/ObjectMapperTest.java b/baremaps-core/src/test/java/org/apache/baremaps/workflow/ObjectMapperTest.java
index 24d34d69..ac62e875 100644
--- a/baremaps-core/src/test/java/org/apache/baremaps/workflow/ObjectMapperTest.java
+++ b/baremaps-core/src/test/java/org/apache/baremaps/workflow/ObjectMapperTest.java
@@ -19,7 +19,7 @@ import java.io.IOException;
import java.nio.file.Paths;
import java.util.List;
import org.apache.baremaps.workflow.tasks.DownloadUrl;
-import org.apache.baremaps.workflow.tasks.ImportOpenStreetMap;
+import org.apache.baremaps.workflow.tasks.ImportOsmPbf;
import org.junit.Test;
public class ObjectMapperTest {
@@ -36,16 +36,16 @@ public class ObjectMapperTest {
"https://download.geofabrik.de/europe/liechtenstein-latest.osm.pbf",
Paths.get("liechtenstein-latest.osm.pbf")))),
new Step("import", List.of("download"),
- List.of(new ImportOpenStreetMap(Paths.get("liechtenstein-latest.osm.pbf"),
+ List.of(new ImportOsmPbf(Paths.get("liechtenstein-latest.osm.pbf"),
"jdbc:postgresql://localhost:5432/baremaps?&user=baremaps&password=baremaps",
3857)))));
var json = mapper.writeValueAsString(workflow1);
assertTrue(json.contains(DownloadUrl.class.getSimpleName()));
- assertTrue(json.contains(ImportOpenStreetMap.class.getSimpleName()));
+ assertTrue(json.contains(ImportOsmPbf.class.getSimpleName()));
// deserialize the workflow
var workflow2 = mapper.readValue(json, Workflow.class);
assertTrue(workflow2.getSteps().get(0).getTasks().get(0) instanceof DownloadUrl);
- assertTrue(workflow2.getSteps().get(1).getTasks().get(0) instanceof ImportOpenStreetMap);
+ assertTrue(workflow2.getSteps().get(1).getTasks().get(0) instanceof ImportOsmPbf);
}
}
diff --git a/baremaps-core/src/test/java/org/apache/baremaps/workflow/WorkflowTest.java b/baremaps-core/src/test/java/org/apache/baremaps/workflow/WorkflowTest.java
index 57bb1fb1..4dcd682a 100644
--- a/baremaps-core/src/test/java/org/apache/baremaps/workflow/WorkflowTest.java
+++ b/baremaps-core/src/test/java/org/apache/baremaps/workflow/WorkflowTest.java
@@ -19,7 +19,7 @@ import java.util.List;
import org.apache.baremaps.testing.PostgresContainerTest;
import org.apache.baremaps.workflow.tasks.DownloadUrl;
import org.apache.baremaps.workflow.tasks.ImportGeoPackage;
-import org.apache.baremaps.workflow.tasks.ImportOpenStreetMap;
+import org.apache.baremaps.workflow.tasks.ImportOsmPbf;
import org.apache.baremaps.workflow.tasks.ImportShapefile;
import org.apache.baremaps.workflow.tasks.UnzipFile;
import org.junit.jupiter.api.Disabled;
@@ -98,7 +98,7 @@ class WorkflowTest extends PostgresContainerTest {
List.of(new DownloadUrl("https://tiles.baremaps.com/samples/liechtenstein.osm.pbf",
Paths.get("downloads/liechtenstein.osm.pbf")))),
new Step("import-osmpbf", List.of("fetch-osmpbf"),
- List.of(new ImportOpenStreetMap(Paths.get("downloads/liechtenstein.osm.pbf"), jdbcUrl(),
+ List.of(new ImportOsmPbf(Paths.get("downloads/liechtenstein.osm.pbf"), jdbcUrl(),
3857))),
new Step("fetch-shapefile", List.of(), List.of(new DownloadUrl(
"https://osmdata.openstreetmap.de/download/simplified-water-polygons-split-3857.zip",
diff --git a/baremaps-core/src/test/java/org/apache/baremaps/workflow/tasks/ImportMonacoTest.java b/baremaps-core/src/test/java/org/apache/baremaps/workflow/tasks/ImportMonacoTest.java
index aa5d26b5..0a1d05db 100644
--- a/baremaps-core/src/test/java/org/apache/baremaps/workflow/tasks/ImportMonacoTest.java
+++ b/baremaps-core/src/test/java/org/apache/baremaps/workflow/tasks/ImportMonacoTest.java
@@ -52,7 +52,7 @@ class ImportMonacoTest extends PostgresRepositoryTest {
new IndexedDataMap<>(new AppendOnlyBuffer<>(new LongListDataType(), new OnHeapMemory()));
// Import data
- ImportOpenStreetMap.execute(TestFiles.resolve("monaco/monaco-210801.osm.pbf"), coordinateMap,
+ ImportOsmPbf.execute(TestFiles.resolve("monaco/monaco-210801.osm.pbf"), coordinateMap,
referenceMap, headerRepository, nodeRepository, wayRepository, relationRepository, 3857);
assertEquals(3047l, headerRepository.selectLatest().getReplicationSequenceNumber());
diff --git a/baremaps-core/src/test/java/org/apache/baremaps/workflow/tasks/ImportOpenStreetMapTest.java b/baremaps-core/src/test/java/org/apache/baremaps/workflow/tasks/ImportPbfTest.java
similarity index 89%
rename from baremaps-core/src/test/java/org/apache/baremaps/workflow/tasks/ImportOpenStreetMapTest.java
rename to baremaps-core/src/test/java/org/apache/baremaps/workflow/tasks/ImportPbfTest.java
index ba457f2f..60761564 100644
--- a/baremaps-core/src/test/java/org/apache/baremaps/workflow/tasks/ImportOpenStreetMapTest.java
+++ b/baremaps-core/src/test/java/org/apache/baremaps/workflow/tasks/ImportPbfTest.java
@@ -20,7 +20,7 @@ import org.apache.baremaps.workflow.WorkflowContext;
import org.junit.jupiter.api.Tag;
import org.junit.jupiter.api.Test;
-class ImportOpenStreetMapTest extends PostgresContainerTest {
+class ImportOsmPbfTest extends PostgresContainerTest {
@Test
@Tag("integration")
@@ -28,7 +28,7 @@ class ImportOpenStreetMapTest extends PostgresContainerTest {
var file = TestFiles.resolve("data.osm.pbf");
var jdbcUrl = jdbcUrl();
var srid = 3857;
- var task = new ImportOpenStreetMap(file, jdbcUrl, srid);
+ var task = new ImportOsmPbf(file, jdbcUrl, srid);
task.execute(new WorkflowContext());
}
}
diff --git a/baremaps-core/src/test/java/org/apache/baremaps/workflow/tasks/ImportUpdateDataTest.java b/baremaps-core/src/test/java/org/apache/baremaps/workflow/tasks/ImportUpdateDataTest.java
index ff23d891..e1e49dfc 100644
--- a/baremaps-core/src/test/java/org/apache/baremaps/workflow/tasks/ImportUpdateDataTest.java
+++ b/baremaps-core/src/test/java/org/apache/baremaps/workflow/tasks/ImportUpdateDataTest.java
@@ -56,7 +56,7 @@ class ImportUpdateDataTest extends PostgresRepositoryTest {
new IndexedDataMap<>(new AppendOnlyBuffer<>(new LongListDataType(), new OnHeapMemory()));
// Import data
- ImportOpenStreetMap.execute(SIMPLE_DATA_OSM_PBF, coordinateMap, referenceMap, headerRepository,
+ ImportOsmPbf.execute(SIMPLE_DATA_OSM_PBF, coordinateMap, referenceMap, headerRepository,
nodeRepository, wayRepository, relationRepository, 3857);
headerRepository.put(new Header(0l, LocalDateTime.of(2020, 1, 1, 0, 0, 0, 0),
diff --git a/baremaps-core/src/test/java/org/apache/baremaps/workflow/tasks/ImportUpdateLiechtensteinTest.java b/baremaps-core/src/test/java/org/apache/baremaps/workflow/tasks/ImportUpdateLiechtensteinTest.java
index fd1e23c6..83990894 100644
--- a/baremaps-core/src/test/java/org/apache/baremaps/workflow/tasks/ImportUpdateLiechtensteinTest.java
+++ b/baremaps-core/src/test/java/org/apache/baremaps/workflow/tasks/ImportUpdateLiechtensteinTest.java
@@ -53,7 +53,7 @@ class ImportUpdateLiechtensteinTest extends PostgresRepositoryTest {
new IndexedDataMap<>(new AppendOnlyBuffer<>(new LongListDataType(), new OnHeapMemory()));
// Import data
- ImportOpenStreetMap.execute(LIECHTENSTEIN_OSM_PBF, coordinateMap, referenceMap,
+ ImportOsmPbf.execute(LIECHTENSTEIN_OSM_PBF, coordinateMap, referenceMap,
headerRepository,
nodeRepository, wayRepository, relationRepository, 3857);
diff --git a/basemap/daylight/workflow.js b/basemap/daylight/workflow.js
index 258dd3f1..abc12550 100644
--- a/basemap/daylight/workflow.js
+++ b/basemap/daylight/workflow.js
@@ -26,7 +26,7 @@ export default {
// "path": "data/data.osm.pbf"
// },
// {
- // "type": "ImportOpenStreetMap",
+ // "type": "ImportOsmPbf",
// "file": "data/data.osm.pbf",
// "database": config.database,
// "databaseSrid": 3857
diff --git a/basemap/workflow.js b/basemap/workflow.js
index 5ff3b1ec..80eea670 100644
--- a/basemap/workflow.js
+++ b/basemap/workflow.js
@@ -122,7 +122,7 @@ export default {
"path": "data/data.osm.pbf"
},
{
- "type": "ImportOpenStreetMap",
+ "type": "ImportOsmPbf",
"file": "data/data.osm.pbf",
"database": config.database,
"databaseSrid": 3857
diff --git a/examples/extrusion/workflow.json b/examples/extrusion/workflow.json
index de11af58..039ad0d1 100644
--- a/examples/extrusion/workflow.json
+++ b/examples/extrusion/workflow.json
@@ -18,7 +18,7 @@
],
"tasks": [
{
- "type": "ImportOpenStreetMap",
+ "type": "ImportOsmPbf",
"file": "greater-london-latest.osm.pbf",
"database": "jdbc:postgresql://localhost:5432/baremaps?&user=baremaps&password=baremaps",
"databaseSrid": 3857
diff --git a/examples/openstreetmap/workflow.json b/examples/openstreetmap/workflow.json
index 9ebd677a..d488cb4b 100644
--- a/examples/openstreetmap/workflow.json
+++ b/examples/openstreetmap/workflow.json
@@ -18,7 +18,7 @@
],
"tasks": [
{
- "type": "ImportOpenStreetMap",
+ "type": "ImportOsmPbf",
"file": "liechtenstein-latest.osm.pbf",
"database": "jdbc:postgresql://localhost:5432/baremaps?&user=baremaps&password=baremaps",
"databaseSrid": 3857