You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@baremaps.apache.org by bc...@apache.org on 2023/11/17 15:53:01 UTC
(incubator-baremaps) branch workflow updated (e8198440 -> 06a9f62c)
This is an automated email from the ASF dual-hosted git repository.
bchapuis pushed a change to branch workflow
in repository https://gitbox.apache.org/repos/asf/incubator-baremaps.git
from e8198440 Remove unused methods
new e2554064 Simplify file decompression
new 31aee85f Replace UnzipFile by DecompressFile task in workflows
new 06a9f62c Remove UnzipFile task
The 3 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
.../apache/baremaps/cli/database/UpdateOsm.java | 2 +-
.../baremaps/workflow/tasks/DecompressFile.java | 21 ++---
.../apache/baremaps/workflow/tasks/UnzipFile.java | 92 ----------------------
.../org/apache/baremaps/workflow/WorkflowTest.java | 19 ++---
.../workflow/tasks/ImportShapefileTest.java | 15 ++--
.../baremaps/workflow/tasks/UnzipFileTest.java | 40 ----------
basemap/import.js | 21 ++---
examples/geocoding/workflow.js | 2 +-
examples/ip-to-location/workflow.js | 7 +-
examples/naturalearth/workflow.json | 7 +-
examples/shadedrelief/workflow.json | 7 +-
11 files changed, 47 insertions(+), 186 deletions(-)
delete mode 100644 baremaps-core/src/main/java/org/apache/baremaps/workflow/tasks/UnzipFile.java
delete mode 100644 baremaps-core/src/test/java/org/apache/baremaps/workflow/tasks/UnzipFileTest.java
(incubator-baremaps) 01/03: Simplify file decompression
Posted by bc...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
bchapuis pushed a commit to branch workflow
in repository https://gitbox.apache.org/repos/asf/incubator-baremaps.git
commit e2554064a18918322e060e9086f6d4a0b050d9c8
Author: Bertil Chapuis <bc...@gmail.com>
AuthorDate: Fri Nov 17 16:44:34 2023 +0100
Simplify file decompression
---
.../org/apache/baremaps/cli/database/UpdateOsm.java | 2 +-
.../apache/baremaps/workflow/tasks/DecompressFile.java | 18 +++---------------
2 files changed, 4 insertions(+), 16 deletions(-)
diff --git a/baremaps-cli/src/main/java/org/apache/baremaps/cli/database/UpdateOsm.java b/baremaps-cli/src/main/java/org/apache/baremaps/cli/database/UpdateOsm.java
index 832a6f72..11fb8d6c 100644
--- a/baremaps-cli/src/main/java/org/apache/baremaps/cli/database/UpdateOsm.java
+++ b/baremaps-cli/src/main/java/org/apache/baremaps/cli/database/UpdateOsm.java
@@ -42,7 +42,7 @@ public class UpdateOsm implements Callable<Integer> {
private int srid = 3857;
@Option(names = {"--replication-url"}, paramLabel = "REPLICATION_URL",
- description = "The replication url of the OpenStreetMap server.")
+ description = "The replication url of the OpenStreetMap server.")
private String replicationUrl = "https://planet.osm.org/replication/hour";
@Override
diff --git a/baremaps-core/src/main/java/org/apache/baremaps/workflow/tasks/DecompressFile.java b/baremaps-core/src/main/java/org/apache/baremaps/workflow/tasks/DecompressFile.java
index 272ad1a3..e5a4108b 100644
--- a/baremaps-core/src/main/java/org/apache/baremaps/workflow/tasks/DecompressFile.java
+++ b/baremaps-core/src/main/java/org/apache/baremaps/workflow/tasks/DecompressFile.java
@@ -133,11 +133,7 @@ public class DecompressFile implements Task {
Files.createDirectories(path.getParent());
try (BufferedOutputStream outputStream =
new BufferedOutputStream(Files.newOutputStream(path))) {
- int bytesRead;
- byte[] buffer = new byte[4096];
- while ((bytesRead = tarInputStream.read(buffer)) != -1) {
- outputStream.write(buffer, 0, bytesRead);
- }
+ tarInputStream.transferTo(outputStream);
}
}
}
@@ -164,11 +160,7 @@ public class DecompressFile implements Task {
Files.createDirectories(path.getParent());
try (BufferedOutputStream outputStream =
new BufferedOutputStream(Files.newOutputStream(path))) {
- int bytesRead;
- byte[] buffer = new byte[4096];
- while ((bytesRead = tarInputStream.read(buffer)) != -1) {
- outputStream.write(buffer, 0, bytesRead);
- }
+ tarInputStream.transferTo(outputStream);
}
}
}
@@ -193,11 +185,7 @@ public class DecompressFile implements Task {
StandardOpenOption.TRUNCATE_EXISTING);
try (var input = new BufferedInputStream(zipFile.getInputStream(entry));
var output = new BufferedOutputStream(new FileOutputStream(path.toFile()))) {
- int nBytes = -1;
- byte[] buffer = new byte[4096];
- while ((nBytes = input.read(buffer)) > 0) {
- output.write(buffer, 0, nBytes);
- }
+ input.transferTo(output);
}
}
}
(incubator-baremaps) 02/03: Replace UnzipFile by DecompressFile task in workflows
Posted by bc...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
bchapuis pushed a commit to branch workflow
in repository https://gitbox.apache.org/repos/asf/incubator-baremaps.git
commit 31aee85f6eb7fed506e648cf8b1e1d5ea3ec82eb
Author: Bertil Chapuis <bc...@gmail.com>
AuthorDate: Fri Nov 17 16:49:19 2023 +0100
Replace UnzipFile by DecompressFile task in workflows
---
.../baremaps/workflow/tasks/DecompressFile.java | 3 ++-
basemap/import.js | 21 ++++++++++++---------
examples/geocoding/workflow.js | 2 +-
examples/ip-to-location/workflow.js | 7 ++++---
examples/naturalearth/workflow.json | 7 ++++---
examples/shadedrelief/workflow.json | 7 ++++---
6 files changed, 27 insertions(+), 20 deletions(-)
diff --git a/baremaps-core/src/main/java/org/apache/baremaps/workflow/tasks/DecompressFile.java b/baremaps-core/src/main/java/org/apache/baremaps/workflow/tasks/DecompressFile.java
index e5a4108b..f9398307 100644
--- a/baremaps-core/src/main/java/org/apache/baremaps/workflow/tasks/DecompressFile.java
+++ b/baremaps-core/src/main/java/org/apache/baremaps/workflow/tasks/DecompressFile.java
@@ -181,7 +181,8 @@ public class DecompressFile implements Task {
var entry = entries.nextElement();
var path = target.resolve(entry.getName());
Files.createDirectories(path.getParent());
- Files.write(path, new byte[] {}, StandardOpenOption.CREATE,
+ Files.write(path, new byte[] {},
+ StandardOpenOption.CREATE,
StandardOpenOption.TRUNCATE_EXISTING);
try (var input = new BufferedInputStream(zipFile.getInputStream(entry));
var output = new BufferedOutputStream(new FileOutputStream(path.toFile()))) {
diff --git a/basemap/import.js b/basemap/import.js
index fec1e00a..d887461e 100644
--- a/basemap/import.js
+++ b/basemap/import.js
@@ -28,9 +28,10 @@ export default {
"path": "data/natural_earth_vector.gpkg.zip"
},
{
- "type": "UnzipFile",
- "file": "data/natural_earth_vector.gpkg.zip",
- "directory": "data/natural_earth_vector"
+ "type": "DecompressFile",
+ "source": "data/natural_earth_vector.gpkg.zip",
+ "target": "data/natural_earth_vector",
+ "compression": "zip"
},
{
"type": "ImportGeoPackage",
@@ -57,9 +58,10 @@ export default {
"path": "data/water-polygons-split-3857.zip"
},
{
- "type": "UnzipFile",
- "file": "data/water-polygons-split-3857.zip",
- "directory": "data"
+ "type": "DecompressFile",
+ "source": "data/water-polygons-split-3857.zip",
+ "target": "data",
+ "compression": "zip"
},
{
"type": "ImportShapefile",
@@ -80,9 +82,10 @@ export default {
"path": "data/simplified-water-polygons-split-3857.zip"
},
{
- "type": "UnzipFile",
- "file": "data/simplified-water-polygons-split-3857.zip",
- "directory": "data"
+ "type": "DecompressFile",
+ "source": "data/simplified-water-polygons-split-3857.zip",
+ "target": "data",
+ "compression": "zip"
},
{
"type": "ImportShapefile",
diff --git a/examples/geocoding/workflow.js b/examples/geocoding/workflow.js
index bee197f8..ba54fcc0 100644
--- a/examples/geocoding/workflow.js
+++ b/examples/geocoding/workflow.js
@@ -19,7 +19,7 @@ const geonamesUrl = "https://download.geonames.org/export/dump/allCountries.zip"
// Fetch and unzip Geonames
const FetchAndUnzipGeonames = {id: "fetch-geonames-allcountries", needs: [], tasks: [
{type: "DownloadUrl", url: geonamesUrl, path: "downloads/geonames-allcountries.zip", force: true},
- {type: "UnzipFile", file: "downloads/geonames-allcountries.zip", directory: "archives"}
+ {type: "DecompressFile", source: "downloads/geonames-allcountries.zip", target: "archives", compression: "zip"}
]};
// Create the Geocoder index
diff --git a/examples/ip-to-location/workflow.js b/examples/ip-to-location/workflow.js
index c262e179..67c5d9db 100644
--- a/examples/ip-to-location/workflow.js
+++ b/examples/ip-to-location/workflow.js
@@ -77,9 +77,10 @@ export default {"steps": [
force: true
},
{
- type: "UnzipFile",
- file: "downloads/geonames-allcountries.zip",
- directory: "archives"
+ type: "DecompressFile",
+ source: "downloads/geonames-allcountries.zip",
+ target: "archives",
+ compression: "zip"
},
{
type: "CreateGeonamesIndex",
diff --git a/examples/naturalearth/workflow.json b/examples/naturalearth/workflow.json
index bd9d6444..b7a66864 100644
--- a/examples/naturalearth/workflow.json
+++ b/examples/naturalearth/workflow.json
@@ -10,9 +10,10 @@
"path": "natural_earth_vector.gpkg.zip"
},
{
- "type": "UnzipFile",
- "file": "natural_earth_vector.gpkg.zip",
- "directory": "natural_earth_vector"
+ "type": "DecompressFile",
+ "source": "natural_earth_vector.gpkg.zip",
+ "target": "natural_earth_vector",
+ "compression": "zip"
},
{
"type": "ImportGeoPackage",
diff --git a/examples/shadedrelief/workflow.json b/examples/shadedrelief/workflow.json
index 48407d52..f88c5d06 100644
--- a/examples/shadedrelief/workflow.json
+++ b/examples/shadedrelief/workflow.json
@@ -10,9 +10,10 @@
"path": "shadedrelief.zip"
},
{
- "type": "UnzipFile",
- "file": "shadedrelief.zip",
- "directory": "shadedrelief"
+ "type": "DecompressFile",
+ "source": "shadedrelief.zip",
+ "target": "shadedrelief",
+ "compression": "zip"
}
]
}
(incubator-baremaps) 03/03: Remove UnzipFile task
Posted by bc...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
bchapuis pushed a commit to branch workflow
in repository https://gitbox.apache.org/repos/asf/incubator-baremaps.git
commit 06a9f62ce0a7baf921fdebb2d8cd188724a6cad2
Author: Bertil Chapuis <bc...@gmail.com>
AuthorDate: Fri Nov 17 16:52:54 2023 +0100
Remove UnzipFile task
---
.../apache/baremaps/workflow/tasks/UnzipFile.java | 92 ----------------------
.../org/apache/baremaps/workflow/WorkflowTest.java | 19 ++---
.../workflow/tasks/ImportShapefileTest.java | 15 ++--
.../baremaps/workflow/tasks/UnzipFileTest.java | 40 ----------
4 files changed, 16 insertions(+), 150 deletions(-)
diff --git a/baremaps-core/src/main/java/org/apache/baremaps/workflow/tasks/UnzipFile.java b/baremaps-core/src/main/java/org/apache/baremaps/workflow/tasks/UnzipFile.java
deleted file mode 100644
index 0804168b..00000000
--- a/baremaps-core/src/main/java/org/apache/baremaps/workflow/tasks/UnzipFile.java
+++ /dev/null
@@ -1,92 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to you under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.baremaps.workflow.tasks;
-
-import com.fasterxml.jackson.annotation.JsonTypeName;
-import java.io.*;
-import java.nio.file.*;
-import java.util.zip.ZipFile;
-import org.apache.baremaps.workflow.Task;
-import org.apache.baremaps.workflow.WorkflowContext;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-/**
- * Unzip a file.
- */
-@JsonTypeName("UnzipFile")
-public class UnzipFile implements Task {
-
- private static final Logger logger = LoggerFactory.getLogger(UnzipFile.class);
-
- private Path file;
- private Path directory;
-
- /**
- * Constructs an {@code UnzipFile}.
- *
- * @param file the file
- * @param directory the directory
- */
- public UnzipFile(Path file, Path directory) {
- this.file = file;
- this.directory = directory;
- }
-
- /**
- * {@inheritDoc}
- */
- @Override
- public void execute(WorkflowContext context) throws Exception {
- var filePath = file.toAbsolutePath();
- var directoryPath = directory.toAbsolutePath();
-
- try (var zipFile = new ZipFile(filePath.toFile())) {
- var entries = zipFile.entries();
-
- while (entries.hasMoreElements()) {
- var ze = entries.nextElement();
- if (ze.isDirectory()) {
- continue;
- }
-
- var path = directoryPath.resolve(ze.getName());
-
- var file = path.toFile().getCanonicalFile();
- var directory = directoryPath.toFile().getCanonicalFile();
- if (!file.toPath().startsWith(directory.toPath())) {
- throw new IOException("Entry is outside of the target directory");
- }
-
- Files.createDirectories(path.getParent());
- Files.write(path, new byte[] {}, StandardOpenOption.CREATE,
- StandardOpenOption.TRUNCATE_EXISTING);
-
- try (var input = new BufferedInputStream(zipFile.getInputStream(ze));
- var output = new BufferedOutputStream(new FileOutputStream(path.toFile()))) {
-
- int nBytes;
- byte[] buffer = new byte[4096];
- while ((nBytes = input.read(buffer)) > 0) {
- output.write(buffer, 0, nBytes);
- }
- }
- }
- }
- }
-}
diff --git a/baremaps-core/src/test/java/org/apache/baremaps/workflow/WorkflowTest.java b/baremaps-core/src/test/java/org/apache/baremaps/workflow/WorkflowTest.java
index c8ba01d8..6e7d220f 100644
--- a/baremaps-core/src/test/java/org/apache/baremaps/workflow/WorkflowTest.java
+++ b/baremaps-core/src/test/java/org/apache/baremaps/workflow/WorkflowTest.java
@@ -22,11 +22,8 @@ package org.apache.baremaps.workflow;
import java.nio.file.Paths;
import java.util.List;
import org.apache.baremaps.testing.PostgresContainerTest;
-import org.apache.baremaps.workflow.tasks.DownloadUrl;
-import org.apache.baremaps.workflow.tasks.ImportGeoPackage;
-import org.apache.baremaps.workflow.tasks.ImportOsmPbf;
-import org.apache.baremaps.workflow.tasks.ImportShapefile;
-import org.apache.baremaps.workflow.tasks.UnzipFile;
+import org.apache.baremaps.workflow.tasks.*;
+import org.apache.baremaps.workflow.tasks.DecompressFile.Compression;
import org.junit.jupiter.api.Disabled;
import org.junit.jupiter.api.Test;
@@ -38,8 +35,8 @@ class WorkflowTest extends PostgresContainerTest {
var workflow = new Workflow(List.of(new Step("fetch-geopackage", List.of(), List.of(
new DownloadUrl("https://naciscdn.org/naturalearth/packages/natural_earth_vector.gpkg.zip",
Paths.get("natural_earth_vector.gpkg.zip"), false),
- new UnzipFile(Paths.get("natural_earth_vector.gpkg.zip"),
- Paths.get("natural_earth_vector")),
+ new DecompressFile(Paths.get("natural_earth_vector.gpkg.zip"),
+ Paths.get("natural_earth_vector"), Compression.zip),
new ImportGeoPackage(Paths.get("natural_earth_vector/packages/natural_earth_vector.gpkg"),
jdbcUrl(),
4326, 3857)))));
@@ -53,8 +50,8 @@ class WorkflowTest extends PostgresContainerTest {
List.of(
new DownloadUrl("https://osmdata.openstreetmap.de/download/coastlines-split-4326.zip",
Paths.get("coastlines-split-4326.zip"), false),
- new UnzipFile(Paths.get("coastlines-split-4326.zip"),
- Paths.get("coastlines-split-4326")),
+ new DecompressFile(Paths.get("coastlines-split-4326.zip"),
+ Paths.get("coastlines-split-4326"), Compression.zip),
new ImportShapefile(Paths.get("coastlines-split-4326/coastlines-split-4326/lines.shp"),
jdbcUrl(),
4326, 3857)))));
@@ -111,8 +108,8 @@ class WorkflowTest extends PostgresContainerTest {
Paths.get("downloads/simplified-water-polygons-split-3857.zip"), false))),
new Step("unzip-shapefile", List.of("fetch-shapefile"),
List.of(
- new UnzipFile(Paths.get("downloads/simplified-water-polygons-split-3857.zip"),
- Paths.get("archives")))),
+ new DecompressFile(Paths.get("downloads/simplified-water-polygons-split-3857.zip"),
+ Paths.get("archives"), Compression.zip))),
new Step("fetch-projection", List.of("unzip-shapefile"),
List.of(new DownloadUrl("https://spatialreference.org/ref/sr-org/epsg3857/prj/",
Paths.get(
diff --git a/baremaps-core/src/test/java/org/apache/baremaps/workflow/tasks/ImportShapefileTest.java b/baremaps-core/src/test/java/org/apache/baremaps/workflow/tasks/ImportShapefileTest.java
index d611efb0..1f63b986 100644
--- a/baremaps-core/src/test/java/org/apache/baremaps/workflow/tasks/ImportShapefileTest.java
+++ b/baremaps-core/src/test/java/org/apache/baremaps/workflow/tasks/ImportShapefileTest.java
@@ -24,6 +24,7 @@ import org.apache.baremaps.testing.PostgresContainerTest;
import org.apache.baremaps.testing.TestFiles;
import org.apache.baremaps.utils.FileUtils;
import org.apache.baremaps.workflow.WorkflowContext;
+import org.apache.baremaps.workflow.tasks.DecompressFile.Compression;
import org.junit.jupiter.api.Tag;
import org.junit.jupiter.api.Test;
@@ -32,13 +33,13 @@ class ImportShapefileTest extends PostgresContainerTest {
@Test
@Tag("integration")
void execute() throws Exception {
- var zip = TestFiles.resolve("monaco-shapefile.zip");
- var directory = Files.createTempDirectory("tmp_");
- var unzip = new UnzipFile(zip, directory);
- unzip.execute(new WorkflowContext());
- var task = new ImportShapefile(directory.resolve("gis_osm_buildings_a_free_1.shp"),
+ var source = TestFiles.resolve("monaco-shapefile.zip");
+ var target = Files.createTempDirectory("tmp_");
+ var decompressFile = new DecompressFile(source, target, Compression.zip);
+ decompressFile.execute(new WorkflowContext());
+ var importShapefile = new ImportShapefile(target.resolve("gis_osm_buildings_a_free_1.shp"),
jdbcUrl(), 4326, 3857);
- task.execute(new WorkflowContext());
- FileUtils.deleteRecursively(directory);
+ importShapefile.execute(new WorkflowContext());
+ FileUtils.deleteRecursively(target);
}
}
diff --git a/baremaps-core/src/test/java/org/apache/baremaps/workflow/tasks/UnzipFileTest.java b/baremaps-core/src/test/java/org/apache/baremaps/workflow/tasks/UnzipFileTest.java
deleted file mode 100644
index a9350139..00000000
--- a/baremaps-core/src/test/java/org/apache/baremaps/workflow/tasks/UnzipFileTest.java
+++ /dev/null
@@ -1,40 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to you under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.baremaps.workflow.tasks;
-
-
-
-import java.nio.file.Files;
-import org.apache.baremaps.testing.TestFiles;
-import org.apache.baremaps.utils.FileUtils;
-import org.apache.baremaps.workflow.WorkflowContext;
-import org.junit.jupiter.api.Tag;
-import org.junit.jupiter.api.Test;
-
-class UnzipFileTest {
-
- @Test
- @Tag("integration")
- void execute() throws Exception {
- var zip = TestFiles.resolve("monaco-shapefile.zip");
- var directory = Files.createTempDirectory("tmp_");
- var task = new UnzipFile(zip, directory);
- task.execute(new WorkflowContext());
- FileUtils.deleteRecursively(directory);
- }
-}