You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@opennlp.apache.org by kr...@apache.org on 2026/08/14 03:17:25 UTC
(opennlp) 07/16: util: Pin hardened ResourceInstaller contract (failing tests)
This is an automated email from the ASF dual-hosted git repository.
krickert pushed a commit to branch OPENNLP-1909-resource-installer
in repository https://gitbox.apache.org/repos/asf/opennlp.git
commit ae6c3bca21fcf4a92c66aba35f76b984ac40dfe7
Author: Kristian Rickert <kr...@gmail.com>
AuthorDate: Tue Aug 11 22:53:35 2026 -0400
util: Pin hardened ResourceInstaller contract (failing tests)
Red evidence: mvn test-compile fails with "cannot find symbol: class
Limits" and the missing install(URI, Path, String, Limits) and
resolveRedirect seams; the SHA-512, staged-atomicity, and redirect
policy tests pin behavior the current installer does not have.
The new tests cover: SHA-512 digests selected by hex length next to
SHA-256, malformed digests rejected as argument errors, staged
installation that leaves the target untouched when a tar, zip, or
truncated archive fails partway, download and expansion ceilings
against oversized sources and small archives that expand into bombs,
and a scripted loopback HTTP server exercising redirects (absolute,
relative, capped chains, missing Location, non-http targets, https
downgrade), error statuses, stalled responses against the read
timeout, and bodies that exceed the download ceiling with or without
a declared length.
---
.../opennlp/tools/util/InstallerTestSupport.java | 151 +++++++
.../tools/util/ResourceInstallerHttpTest.java | 480 +++++++++++++++++++++
.../opennlp/tools/util/ResourceInstallerTest.java | 428 ++++++++++++++----
3 files changed, 974 insertions(+), 85 deletions(-)
diff --git a/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/util/InstallerTestSupport.java b/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/util/InstallerTestSupport.java
new file mode 100644
index 000000000..5848934bc
--- /dev/null
+++ b/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/util/InstallerTestSupport.java
@@ -0,0 +1,151 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package opennlp.tools.util;
+
+import java.io.ByteArrayOutputStream;
+import java.io.IOException;
+import java.nio.charset.StandardCharsets;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.security.MessageDigest;
+import java.security.NoSuchAlgorithmException;
+import java.util.HexFormat;
+import java.util.List;
+import java.util.stream.Stream;
+import java.util.zip.GZIPOutputStream;
+
+/**
+ * Shared fixtures for the {@code ResourceInstaller} test classes: hand-rolled tar
+ * archive building, gzip compression, digest computation, and installed-file listing.
+ */
+final class InstallerTestSupport {
+
+ static final int BLOCK = 512;
+ static final int TERMINATOR_SIZE = 2 * BLOCK;
+
+ private static final int NAME_LENGTH = 100;
+ private static final int SIZE_OFFSET = 124;
+ private static final int TYPE_OFFSET = 156;
+ private static final String SIZE_FORMAT = "%011o";
+ private static final char TYPE_REGULAR_FILE = '0';
+
+ private InstallerTestSupport() {
+ }
+
+ /**
+ * Writes one tar entry into the given buffer: a 512-byte header block carrying the
+ * name, the octal content size, and the regular-file type flag, followed by the
+ * content padded to the next 512-byte block boundary.
+ *
+ * @param tar The buffer receiving the entry bytes. Must not be {@code null}.
+ * @param name The entry name; at most 100 bytes when encoded as UTF-8.
+ * @param content The entry content. Must not be {@code null}.
+ * @throws IOException Thrown if writing to the buffer fails.
+ * @throws IllegalArgumentException Thrown if the name exceeds the tar name field.
+ */
+ static void tarEntry(ByteArrayOutputStream tar, String name, byte[] content)
+ throws IOException {
+ final byte[] header = new byte[BLOCK];
+ final byte[] nameBytes = name.getBytes(StandardCharsets.UTF_8);
+ if (nameBytes.length > NAME_LENGTH) {
+ throw new IllegalArgumentException(
+ "entry name exceeds " + NAME_LENGTH + " bytes: " + name);
+ }
+ System.arraycopy(nameBytes, 0, header, 0, nameBytes.length);
+ final byte[] size = String.format(SIZE_FORMAT, content.length)
+ .getBytes(StandardCharsets.US_ASCII);
+ System.arraycopy(size, 0, header, SIZE_OFFSET, size.length);
+ header[TYPE_OFFSET] = TYPE_REGULAR_FILE;
+ tar.write(header);
+ tar.write(content);
+ tar.write(new byte[(BLOCK - content.length % BLOCK) % BLOCK]);
+ }
+
+ /**
+ * Builds a gzip-compressed tar archive from name and content pairs, terminated by
+ * the two all-zero blocks that end a tar archive.
+ *
+ * @param entries Pairs of entry name and UTF-8 text content. Must not be {@code null}.
+ * @return The archive bytes. Never {@code null}.
+ * @throws IOException Thrown if assembling the archive fails.
+ */
+ static byte[] tarGz(String[][] entries) throws IOException {
+ final ByteArrayOutputStream tar = new ByteArrayOutputStream();
+ for (final String[] entry : entries) {
+ tarEntry(tar, entry[0], entry[1].getBytes(StandardCharsets.UTF_8));
+ }
+ tar.write(new byte[TERMINATOR_SIZE]);
+ return gzip(tar.toByteArray());
+ }
+
+ /**
+ * Compresses the given bytes with gzip, so tests can wrap hand-built or deliberately
+ * truncated tar content.
+ *
+ * @param content The bytes to compress. Must not be {@code null}.
+ * @return The gzip-compressed bytes. Never {@code null}.
+ * @throws IOException Thrown if compressing fails.
+ */
+ static byte[] gzip(byte[] content) throws IOException {
+ final ByteArrayOutputStream compressed = new ByteArrayOutputStream();
+ try (GZIPOutputStream out = new GZIPOutputStream(compressed)) {
+ out.write(content);
+ }
+ return compressed.toByteArray();
+ }
+
+ /**
+ * Computes the SHA-256 of the given bytes as a lowercase hex string.
+ *
+ * @param content The bytes to digest. Must not be {@code null}.
+ * @return The 64-character lowercase hex digest. Never {@code null}.
+ * @throws NoSuchAlgorithmException Thrown if the digest algorithm is unavailable.
+ */
+ static String sha256(byte[] content) throws NoSuchAlgorithmException {
+ return HexFormat.of().formatHex(MessageDigest.getInstance("SHA-256").digest(content));
+ }
+
+ /**
+ * Computes the SHA-512 of the given bytes as a lowercase hex string.
+ *
+ * @param content The bytes to digest. Must not be {@code null}.
+ * @return The 128-character lowercase hex digest. Never {@code null}.
+ * @throws NoSuchAlgorithmException Thrown if the digest algorithm is unavailable.
+ */
+ static String sha512(byte[] content) throws NoSuchAlgorithmException {
+ return HexFormat.of().formatHex(MessageDigest.getInstance("SHA-512").digest(content));
+ }
+
+ /**
+ * Lists every regular file below the given directory as relative paths with forward
+ * slashes, sorted lexicographically, so tests can assert the exact installed file
+ * set.
+ *
+ * @param root The directory to walk. Must not be {@code null}.
+ * @return The sorted relative paths. Never {@code null}.
+ * @throws IOException Thrown if walking the directory fails.
+ */
+ static List<String> installedFiles(Path root) throws IOException {
+ try (Stream<Path> walk = Files.walk(root)) {
+ return walk.filter(Files::isRegularFile)
+ .map(file -> root.relativize(file).toString().replace('\\', '/'))
+ .sorted()
+ .toList();
+ }
+ }
+}
diff --git a/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/util/ResourceInstallerHttpTest.java b/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/util/ResourceInstallerHttpTest.java
new file mode 100644
index 000000000..4ca618bbf
--- /dev/null
+++ b/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/util/ResourceInstallerHttpTest.java
@@ -0,0 +1,480 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package opennlp.tools.util;
+
+import java.io.ByteArrayOutputStream;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.OutputStream;
+import java.net.InetAddress;
+import java.net.ServerSocket;
+import java.net.Socket;
+import java.net.SocketTimeoutException;
+import java.net.URI;
+import java.nio.charset.StandardCharsets;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.time.Duration;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Map;
+import java.util.concurrent.ConcurrentHashMap;
+
+import org.junit.jupiter.api.AfterEach;
+import org.junit.jupiter.api.Assertions;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.io.TempDir;
+
+import static opennlp.tools.util.InstallerTestSupport.installedFiles;
+import static opennlp.tools.util.InstallerTestSupport.sha256;
+import static opennlp.tools.util.InstallerTestSupport.tarGz;
+
+/**
+ * Exercises {@link ResourceInstaller} against a local scripted HTTP server: happy
+ * downloads, redirect handling and its policy, error statuses, stalled responses
+ * against the read timeout, and download ceilings against lying or oversized bodies.
+ */
+public class ResourceInstallerHttpTest {
+
+ private static final Duration GENEROUS = Duration.ofSeconds(10);
+ private static final long MEBIBYTE = 1024 * 1024;
+
+ private StubServer server;
+
+ @BeforeEach
+ void startServer() throws IOException {
+ server = new StubServer();
+ }
+
+ @AfterEach
+ void stopServer() throws IOException {
+ server.close();
+ }
+
+ /**
+ * Builds installation limits with the given read timeout and otherwise generous
+ * values, so timeout tests state only the value they exercise.
+ *
+ * @param readTimeout The read timeout to apply.
+ * @return The limits. Never {@code null}.
+ */
+ private static ResourceInstaller.Limits withReadTimeout(Duration readTimeout) {
+ return new ResourceInstaller.Limits(GENEROUS, readTimeout, 5, MEBIBYTE, MEBIBYTE);
+ }
+
+ /**
+ * Builds installation limits with the given redirect allowance and otherwise
+ * generous values.
+ *
+ * @param maxRedirects The number of redirects to follow.
+ * @return The limits. Never {@code null}.
+ */
+ private static ResourceInstaller.Limits withMaxRedirects(int maxRedirects) {
+ return new ResourceInstaller.Limits(GENEROUS, GENEROUS, maxRedirects,
+ MEBIBYTE, MEBIBYTE);
+ }
+
+ /**
+ * Builds installation limits with the given download ceiling and otherwise generous
+ * values.
+ *
+ * @param maxDownloadBytes The download ceiling in bytes.
+ * @return The limits. Never {@code null}.
+ */
+ private static ResourceInstaller.Limits withDownloadCeiling(long maxDownloadBytes) {
+ return new ResourceInstaller.Limits(GENEROUS, GENEROUS, 5, maxDownloadBytes,
+ MEBIBYTE);
+ }
+
+ @Test
+ void testHttpDownloadInstallsArchive(@TempDir Path target) throws Exception {
+ final byte[] archive = tarGz(new String[][] {{"corpus/data.txt", "over http"}});
+ server.route("/corpus.tar.gz", out -> StubServer.ok(out, archive));
+
+ ResourceInstaller.install(server.uri("/corpus.tar.gz"), target, sha256(archive));
+
+ Assertions.assertEquals("over http",
+ Files.readString(target.resolve("corpus/data.txt")));
+ }
+
+ @Test
+ void testAbsoluteRedirectIsFollowed(@TempDir Path target) throws Exception {
+ final byte[] archive = tarGz(new String[][] {{"corpus/data.txt", "moved"}});
+ server.route("/old.tar.gz", out -> StubServer.redirect(out,
+ server.uri("/new.tar.gz").toString()));
+ server.route("/new.tar.gz", out -> StubServer.ok(out, archive));
+
+ ResourceInstaller.install(server.uri("/old.tar.gz"), target, sha256(archive));
+
+ Assertions.assertEquals("moved",
+ Files.readString(target.resolve("corpus/data.txt")));
+ }
+
+ @Test
+ void testRelativeRedirectIsResolvedAgainstTheSource(@TempDir Path target)
+ throws Exception {
+ final byte[] archive = tarGz(new String[][] {{"corpus/data.txt", "relative"}});
+ server.route("/mirror/old.tar.gz",
+ out -> StubServer.redirect(out, "new.tar.gz"));
+ server.route("/mirror/new.tar.gz", out -> StubServer.ok(out, archive));
+
+ ResourceInstaller.install(server.uri("/mirror/old.tar.gz"), target, sha256(archive));
+
+ Assertions.assertEquals("relative",
+ Files.readString(target.resolve("corpus/data.txt")));
+ }
+
+ /**
+ * Proves the redirect cap: a location that keeps redirecting to itself fails once
+ * the configured allowance is exhausted, and nothing is installed.
+ */
+ @Test
+ void testRedirectChainBeyondLimitFails(@TempDir Path target) throws Exception {
+ server.route("/loop", out -> StubServer.redirect(out, "/loop"));
+
+ final URI source = server.uri("/loop");
+ final IOException thrown = Assertions.assertThrows(IOException.class,
+ () -> ResourceInstaller.install(source, target, null, withMaxRedirects(3)));
+ Assertions.assertEquals("more than 3 redirects: " + source, thrown.getMessage());
+ Assertions.assertEquals(List.of(), installedFiles(target));
+ }
+
+ /**
+ * Proves that a redirect status without a Location header fails loud instead of
+ * installing the redirect response body as if it were the resource.
+ */
+ @Test
+ void testRedirectWithoutLocationFails(@TempDir Path target) throws Exception {
+ server.route("/broken", out -> StubServer.head(out, "302 Found",
+ "Content-Length: 4", "", "gone"));
+
+ final URI source = server.uri("/broken");
+ final IOException thrown = Assertions.assertThrows(IOException.class,
+ () -> ResourceInstaller.install(source, target));
+ Assertions.assertEquals("redirect from " + source + " carries no Location header",
+ thrown.getMessage());
+ Assertions.assertEquals(List.of(), installedFiles(target));
+ }
+
+ /**
+ * Proves that a redirect leaving the http and https schemes is rejected, so a
+ * hostile mirror cannot steer the installer at local files.
+ */
+ @Test
+ void testRedirectToNonHttpSchemeFails(@TempDir Path target) throws Exception {
+ server.route("/hostile", out -> StubServer.redirect(out, "file:///etc/passwd"));
+
+ final IOException thrown = Assertions.assertThrows(IOException.class,
+ () -> ResourceInstaller.install(server.uri("/hostile"), target));
+ Assertions.assertEquals(
+ "redirect target is not an http or https location: file:///etc/passwd",
+ thrown.getMessage());
+ Assertions.assertEquals(List.of(), installedFiles(target));
+ }
+
+ /**
+ * Proves the downgrade rule on the redirect seam: an https source must not follow a
+ * redirect to plain http, while https to https and http to https are permitted.
+ */
+ @Test
+ void testHttpsToHttpDowngradeIsRejected() {
+ final URI https = URI.create("https://example.invalid/archive.tar.gz");
+
+ final IOException thrown = Assertions.assertThrows(IOException.class,
+ () -> ResourceInstaller.resolveRedirect(https,
+ "http://example.invalid/archive.tar.gz"));
+ Assertions.assertEquals(
+ "redirect downgrades https to http: http://example.invalid/archive.tar.gz",
+ thrown.getMessage());
+
+ Assertions.assertDoesNotThrow(() -> ResourceInstaller.resolveRedirect(https,
+ "https://mirror.invalid/archive.tar.gz"));
+ Assertions.assertDoesNotThrow(() -> ResourceInstaller.resolveRedirect(
+ URI.create("http://example.invalid/archive.tar.gz"),
+ "https://mirror.invalid/archive.tar.gz"));
+ }
+
+ @Test
+ void testHttpErrorStatusFails(@TempDir Path target) throws Exception {
+ server.route("/missing.tar.gz", out -> StubServer.head(out, "404 Not Found",
+ "Content-Length: 0", ""));
+
+ final URI source = server.uri("/missing.tar.gz");
+ final IOException thrown = Assertions.assertThrows(IOException.class,
+ () -> ResourceInstaller.install(source, target));
+ Assertions.assertEquals("download failed with HTTP status 404: " + source,
+ thrown.getMessage());
+ Assertions.assertEquals(List.of(), installedFiles(target));
+ }
+
+ /**
+ * Proves the read timeout against a server that accepts the request and then never
+ * answers: the installer gives up within the configured timeout instead of hanging.
+ */
+ @Test
+ void testStalledResponseHitsReadTimeout(@TempDir Path target) {
+ server.route("/stall", out -> StubServer.sleep(Duration.ofSeconds(30)));
+
+ final IOException thrown = Assertions.assertThrows(IOException.class,
+ () -> ResourceInstaller.install(server.uri("/stall"), target, null,
+ withReadTimeout(Duration.ofMillis(250))));
+ Assertions.assertInstanceOf(SocketTimeoutException.class, thrown);
+ }
+
+ /**
+ * Proves the read timeout against a slow-loris body: headers and a few bytes arrive,
+ * then the connection stalls, and the installer gives up instead of hanging.
+ */
+ @Test
+ void testStalledBodyHitsReadTimeout(@TempDir Path target) throws Exception {
+ server.route("/drip", out -> {
+ StubServer.head(out, "200 OK", "Content-Length: 100000", "");
+ out.write("just a few bytes".getBytes(StandardCharsets.UTF_8));
+ out.flush();
+ StubServer.sleep(Duration.ofSeconds(30));
+ });
+
+ final IOException thrown = Assertions.assertThrows(IOException.class,
+ () -> ResourceInstaller.install(server.uri("/drip"), target, null,
+ withReadTimeout(Duration.ofMillis(250))));
+ Assertions.assertInstanceOf(SocketTimeoutException.class, thrown);
+ Assertions.assertEquals(List.of(), installedFiles(target));
+ }
+
+ /**
+ * Proves that a declared content length beyond the download ceiling fails before
+ * the body is read, so a lying server cannot make the installer fetch it at all.
+ */
+ @Test
+ void testDeclaredContentLengthBeyondCeilingFailsFast(@TempDir Path target)
+ throws Exception {
+ server.route("/liar.tar.gz", out -> StubServer.head(out, "200 OK",
+ "Content-Length: 10000000", ""));
+
+ final IOException thrown = Assertions.assertThrows(IOException.class,
+ () -> ResourceInstaller.install(server.uri("/liar.tar.gz"), target, null,
+ withDownloadCeiling(1024)));
+ Assertions.assertEquals(
+ "declared content length 10000000 exceeds the download ceiling of 1024 bytes",
+ thrown.getMessage());
+ Assertions.assertEquals(List.of(), installedFiles(target));
+ }
+
+ /**
+ * Proves the download ceiling against a close-delimited body with no declared
+ * length: the transfer aborts once the ceiling is crossed and nothing is installed.
+ */
+ @Test
+ void testStreamedBodyBeyondCeilingAborts(@TempDir Path target) throws Exception {
+ server.route("/endless", out -> {
+ StubServer.head(out, "200 OK", "");
+ out.write(new byte[64 * 1024]);
+ out.flush();
+ });
+
+ final IOException thrown = Assertions.assertThrows(IOException.class,
+ () -> ResourceInstaller.install(server.uri("/endless"), target, null,
+ withDownloadCeiling(1024)));
+ Assertions.assertEquals("download exceeds the ceiling of 1024 bytes",
+ thrown.getMessage());
+ Assertions.assertEquals(List.of(), installedFiles(target));
+ }
+
+ /**
+ * A minimal scripted HTTP server on a loopback socket. Each registered route is a
+ * function from the response output stream to the raw bytes it wants on the wire,
+ * which lets tests script redirects, lies, stalls, and floods precisely.
+ */
+ private static final class StubServer implements AutoCloseable {
+
+ /** Writes a raw HTTP response to the connected client. */
+ @FunctionalInterface
+ interface Responder {
+ void respond(OutputStream out) throws IOException;
+ }
+
+ private final ServerSocket socket;
+ private final Map<String, Responder> routes = new ConcurrentHashMap<>();
+ private final List<Socket> connections = new ArrayList<>();
+
+ StubServer() throws IOException {
+ socket = new ServerSocket(0, 50, InetAddress.getLoopbackAddress());
+ final Thread acceptor = new Thread(this::acceptLoop, "stub-http-acceptor");
+ acceptor.setDaemon(true);
+ acceptor.start();
+ }
+
+ /**
+ * Registers the responder serving the given absolute request path.
+ *
+ * @param path The absolute request path, starting with {@code /}.
+ * @param responder The script producing the raw response.
+ */
+ void route(String path, Responder responder) {
+ routes.put(path, responder);
+ }
+
+ /**
+ * Builds the http URI of the given absolute path on this server.
+ *
+ * @param path The absolute request path, starting with {@code /}.
+ * @return The URI. Never {@code null}.
+ */
+ URI uri(String path) {
+ return URI.create("http://127.0.0.1:" + socket.getLocalPort() + path);
+ }
+
+ /**
+ * Writes a response head: the HTTP/1.0 status line, the given header lines, and
+ * the blank separator line, followed by any further lines verbatim.
+ *
+ * @param out The response stream.
+ * @param status The status line content after the protocol, such as {@code 200 OK}.
+ * @param lines Header lines, then an empty string separator, then optional body
+ * text written verbatim.
+ * @throws IOException Thrown if writing fails.
+ */
+ static void head(OutputStream out, String status, String... lines)
+ throws IOException {
+ final StringBuilder response = new StringBuilder("HTTP/1.0 ").append(status)
+ .append("\r\n");
+ for (final String line : lines) {
+ response.append(line).append("\r\n");
+ }
+ out.write(response.toString().getBytes(StandardCharsets.US_ASCII));
+ out.flush();
+ }
+
+ /**
+ * Writes a complete 200 response carrying the given body with its exact length.
+ *
+ * @param out The response stream.
+ * @param body The response body bytes.
+ * @throws IOException Thrown if writing fails.
+ */
+ static void ok(OutputStream out, byte[] body) throws IOException {
+ head(out, "200 OK", "Content-Length: " + body.length, "");
+ out.write(body);
+ out.flush();
+ }
+
+ /**
+ * Writes a 302 redirect to the given location.
+ *
+ * @param out The response stream.
+ * @param location The Location header value, absolute or relative.
+ * @throws IOException Thrown if writing fails.
+ */
+ static void redirect(OutputStream out, String location) throws IOException {
+ head(out, "302 Found", "Location: " + location, "Content-Length: 0", "");
+ }
+
+ /**
+ * Blocks the handler thread, simulating a stalled server. Interruption during
+ * teardown ends the sleep early.
+ *
+ * @param duration How long to stall.
+ */
+ static void sleep(Duration duration) {
+ try {
+ Thread.sleep(duration.toMillis());
+ } catch (InterruptedException e) {
+ Thread.currentThread().interrupt();
+ }
+ }
+
+ private void acceptLoop() {
+ while (!socket.isClosed()) {
+ final Socket connection;
+ try {
+ connection = socket.accept();
+ } catch (IOException e) {
+ return;
+ }
+ synchronized (connections) {
+ connections.add(connection);
+ }
+ final Thread handler = new Thread(() -> handle(connection),
+ "stub-http-handler");
+ handler.setDaemon(true);
+ handler.start();
+ }
+ }
+
+ /**
+ * Reads one request, dispatches it to the registered responder, and closes the
+ * connection. Write failures from clients that abort mid-transfer are expected
+ * and ignored.
+ *
+ * @param connection The accepted client connection.
+ */
+ private void handle(Socket connection) {
+ try (connection) {
+ final String path = readRequestPath(connection.getInputStream());
+ final Responder responder = routes.get(path);
+ if (responder == null) {
+ head(connection.getOutputStream(), "404 Not Found", "Content-Length: 0", "");
+ return;
+ }
+ responder.respond(connection.getOutputStream());
+ } catch (IOException e) {
+ // The client hung up or the server is shutting down; both are test-normal.
+ }
+ }
+
+ /**
+ * Reads the request head up to its terminating blank line and returns the path
+ * from the request line.
+ *
+ * @param in The request stream.
+ * @return The request path. Never {@code null}.
+ * @throws IOException Thrown if the request head is malformed or truncated.
+ */
+ private static String readRequestPath(InputStream in) throws IOException {
+ final ByteArrayOutputStream headBytes = new ByteArrayOutputStream();
+ int last4 = 0;
+ int b;
+ while ((b = in.read()) >= 0) {
+ headBytes.write(b);
+ last4 = (last4 << 8) | b;
+ if (last4 == 0x0D0A0D0A) {
+ break;
+ }
+ }
+ final String head = headBytes.toString(StandardCharsets.US_ASCII);
+ final int firstLineEnd = head.indexOf("\r\n");
+ final String requestLine = firstLineEnd < 0 ? head : head.substring(0, firstLineEnd);
+ final String[] parts = requestLine.split(" ");
+ if (parts.length < 2) {
+ throw new IOException("malformed request line: " + requestLine);
+ }
+ return parts[1];
+ }
+
+ @Override
+ public void close() throws IOException {
+ socket.close();
+ synchronized (connections) {
+ for (final Socket connection : connections) {
+ connection.close();
+ }
+ }
+ }
+ }
+}
diff --git a/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/util/ResourceInstallerTest.java b/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/util/ResourceInstallerTest.java
index aab8b6b5f..8657897ec 100644
--- a/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/util/ResourceInstallerTest.java
+++ b/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/util/ResourceInstallerTest.java
@@ -22,9 +22,9 @@ import java.io.IOException;
import java.nio.charset.StandardCharsets;
import java.nio.file.Files;
import java.nio.file.Path;
-import java.security.MessageDigest;
import java.security.NoSuchAlgorithmException;
-import java.util.HexFormat;
+import java.time.Duration;
+import java.util.Arrays;
import java.util.List;
import java.util.Locale;
import java.util.stream.Stream;
@@ -34,79 +34,23 @@ import java.util.zip.ZipOutputStream;
import org.junit.jupiter.api.Assertions;
import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.function.Executable;
import org.junit.jupiter.api.io.TempDir;
+import org.junit.jupiter.params.ParameterizedTest;
+import org.junit.jupiter.params.provider.Arguments;
+import org.junit.jupiter.params.provider.MethodSource;
+
+import static opennlp.tools.util.InstallerTestSupport.BLOCK;
+import static opennlp.tools.util.InstallerTestSupport.TERMINATOR_SIZE;
+import static opennlp.tools.util.InstallerTestSupport.gzip;
+import static opennlp.tools.util.InstallerTestSupport.installedFiles;
+import static opennlp.tools.util.InstallerTestSupport.sha256;
+import static opennlp.tools.util.InstallerTestSupport.sha512;
+import static opennlp.tools.util.InstallerTestSupport.tarEntry;
+import static opennlp.tools.util.InstallerTestSupport.tarGz;
public class ResourceInstallerTest {
- private static final int BLOCK = 512;
- private static final int TERMINATOR_SIZE = 2 * BLOCK;
- private static final int NAME_LENGTH = 100;
- private static final int SIZE_OFFSET = 124;
- private static final int TYPE_OFFSET = 156;
- private static final String SIZE_FORMAT = "%011o";
- private static final char TYPE_REGULAR_FILE = '0';
-
- /**
- * Writes one tar entry into the given buffer: a 512-byte header block carrying the
- * name, the octal content size, and the regular-file type flag, followed by the
- * content padded to the next 512-byte block boundary.
- *
- * @param tar The buffer receiving the entry bytes. Must not be {@code null}.
- * @param name The entry name; at most 100 bytes when encoded as UTF-8.
- * @param content The entry content. Must not be {@code null}.
- * @throws IOException Thrown if writing to the buffer fails.
- * @throws IllegalArgumentException Thrown if the name exceeds the tar name field.
- */
- private static void tarEntry(ByteArrayOutputStream tar, String name, byte[] content)
- throws IOException {
- final byte[] header = new byte[BLOCK];
- final byte[] nameBytes = name.getBytes(StandardCharsets.UTF_8);
- if (nameBytes.length > NAME_LENGTH) {
- throw new IllegalArgumentException(
- "entry name exceeds " + NAME_LENGTH + " bytes: " + name);
- }
- System.arraycopy(nameBytes, 0, header, 0, nameBytes.length);
- final byte[] size = String.format(SIZE_FORMAT, content.length)
- .getBytes(StandardCharsets.US_ASCII);
- System.arraycopy(size, 0, header, SIZE_OFFSET, size.length);
- header[TYPE_OFFSET] = TYPE_REGULAR_FILE;
- tar.write(header);
- tar.write(content);
- tar.write(new byte[(BLOCK - content.length % BLOCK) % BLOCK]);
- }
-
- /**
- * Builds a gzip-compressed tar archive from name and content pairs, terminated by
- * the two all-zero blocks that end a tar archive.
- *
- * @param entries Pairs of entry name and UTF-8 text content. Must not be {@code null}.
- * @return The archive bytes. Never {@code null}.
- * @throws IOException Thrown if assembling the archive fails.
- */
- private static byte[] tarGz(String[][] entries) throws IOException {
- final ByteArrayOutputStream tar = new ByteArrayOutputStream();
- for (final String[] entry : entries) {
- tarEntry(tar, entry[0], entry[1].getBytes(StandardCharsets.UTF_8));
- }
- tar.write(new byte[TERMINATOR_SIZE]);
- final ByteArrayOutputStream compressed = new ByteArrayOutputStream();
- try (GZIPOutputStream gzip = new GZIPOutputStream(compressed)) {
- gzip.write(tar.toByteArray());
- }
- return compressed.toByteArray();
- }
-
- /**
- * Computes the SHA-256 of the given bytes as a lowercase hex string.
- *
- * @param content The bytes to digest. Must not be {@code null}.
- * @return The 64-character lowercase hex digest. Never {@code null}.
- * @throws NoSuchAlgorithmException Thrown if the digest algorithm is unavailable.
- */
- private static String sha256(byte[] content) throws NoSuchAlgorithmException {
- return HexFormat.of().formatHex(MessageDigest.getInstance("SHA-256").digest(content));
- }
-
/**
* Computes the SHA-256 of the given bytes as an uppercase hex string, so tests can
* prove that checksum comparison does not depend on the hex letter case.
@@ -120,21 +64,17 @@ public class ResourceInstallerTest {
}
/**
- * Lists every regular file below the given directory as relative paths with forward
- * slashes, sorted lexicographically, so tests can assert the exact installed file
- * set.
+ * Builds installation limits with generous timeouts and redirect allowance but the
+ * given size ceilings, so ceiling tests state only the values they exercise.
*
- * @param root The directory to walk. Must not be {@code null}.
- * @return The sorted relative paths. Never {@code null}.
- * @throws IOException Thrown if walking the directory fails.
+ * @param maxDownloadBytes The download ceiling in bytes.
+ * @param maxExpandedBytes The expansion ceiling in bytes.
+ * @return The limits. Never {@code null}.
*/
- private static List<String> installedFiles(Path root) throws IOException {
- try (Stream<Path> walk = Files.walk(root)) {
- return walk.filter(Files::isRegularFile)
- .map(file -> root.relativize(file).toString().replace('\\', '/'))
- .sorted()
- .toList();
- }
+ private static ResourceInstaller.Limits ceilings(long maxDownloadBytes,
+ long maxExpandedBytes) {
+ return new ResourceInstaller.Limits(Duration.ofSeconds(10), Duration.ofSeconds(10),
+ 5, maxDownloadBytes, maxExpandedBytes);
}
/**
@@ -362,7 +302,325 @@ public class ResourceInstallerTest {
thrown = Assertions.assertThrows(IllegalArgumentException.class,
() -> ResourceInstaller.install(target.toUri(), target, " "));
- Assertions.assertEquals("sha256 must not be blank; pass null to skip",
+ Assertions.assertEquals(
+ "checksum must be 64 (SHA-256) or 128 (SHA-512) hex characters; pass null to skip",
+ thrown.getMessage());
+
+ thrown = Assertions.assertThrows(IllegalArgumentException.class,
+ () -> ResourceInstaller.install(target.toUri(), target, null, null));
+ Assertions.assertEquals("limits must not be null", thrown.getMessage());
+ }
+
+ /**
+ * Proves that a 128-character hex digest selects SHA-512 verification: the correct
+ * SHA-512 of the archive passes and the content is installed.
+ */
+ @Test
+ void testSha512ChecksumVerifies(@TempDir Path source, @TempDir Path target)
+ throws Exception {
+ final byte[] archive = tarGz(new String[][] {{"data/entry.txt", "payload"}});
+ final Path file = source.resolve("checked.tar.gz");
+ Files.write(file, archive);
+
+ ResourceInstaller.install(file.toUri(), target, sha512(archive));
+
+ Assertions.assertEquals("payload",
+ Files.readString(target.resolve("data/entry.txt")));
+ }
+
+ @Test
+ void testSha512ChecksumMismatchFailsBeforeUnpacking(@TempDir Path source,
+ @TempDir Path target) throws Exception {
+ final byte[] archive = tarGz(new String[][] {{"a/file.txt", "content"}});
+ final Path file = source.resolve("archive.tar.gz");
+ Files.write(file, archive);
+
+ final String wrong = "0".repeat(128);
+ final IOException thrown = Assertions.assertThrows(IOException.class,
+ () -> ResourceInstaller.install(file.toUri(), target, wrong));
+ Assertions.assertEquals("checksum mismatch: expected " + wrong
+ + " but downloaded " + sha512(archive), thrown.getMessage());
+ Assertions.assertEquals(List.of(), installedFiles(target));
+ }
+
+ /**
+ * Proves that a digest whose length matches neither SHA-256 nor SHA-512 is rejected
+ * as an argument error before anything is fetched.
+ */
+ @Test
+ void testChecksumOfUnsupportedLengthIsRejected(@TempDir Path target) {
+ final IllegalArgumentException thrown = Assertions.assertThrows(
+ IllegalArgumentException.class,
+ () -> ResourceInstaller.install(target.toUri(), target, "abc123"));
+ Assertions.assertEquals(
+ "checksum must be 64 (SHA-256) or 128 (SHA-512) hex characters; pass null to skip",
+ thrown.getMessage());
+ }
+
+ /**
+ * Proves that a digest of the right length but with non-hex characters is rejected
+ * as an argument error, so a typo cannot masquerade as a checksum mismatch.
+ */
+ @Test
+ void testChecksumWithNonHexCharactersIsRejected(@TempDir Path target) {
+ final IllegalArgumentException thrown = Assertions.assertThrows(
+ IllegalArgumentException.class,
+ () -> ResourceInstaller.install(target.toUri(), target, "g".repeat(64)));
+ Assertions.assertEquals(
+ "checksum must be 64 (SHA-256) or 128 (SHA-512) hex characters; pass null to skip",
+ thrown.getMessage());
+ }
+
+ /**
+ * Proves staged installation for tar content: when a later entry escapes the target
+ * directory, the earlier entry that already unpacked cleanly must not appear either.
+ */
+ @Test
+ void testFailedTarUnpackLeavesTargetEmpty(@TempDir Path source, @TempDir Path target)
+ throws Exception {
+ final byte[] archive = tarGz(new String[][] {
+ {"good.txt", "fine"},
+ {"../escape.txt", "bad"}});
+ final Path file = source.resolve("partial.tar.gz");
+ Files.write(file, archive);
+
+ final IOException thrown = Assertions.assertThrows(IOException.class,
+ () -> ResourceInstaller.install(file.toUri(), target));
+ Assertions.assertEquals(
+ "archive entry escapes the target directory: ../escape.txt",
+ thrown.getMessage());
+ Assertions.assertEquals(List.of(), installedFiles(target));
+ }
+
+ /**
+ * Proves staged installation for zip content: a traversal entry after a clean one
+ * leaves the target without the clean entry as well.
+ */
+ @Test
+ void testFailedZipUnpackLeavesTargetEmpty(@TempDir Path source, @TempDir Path target)
+ throws Exception {
+ final ByteArrayOutputStream out = new ByteArrayOutputStream();
+ try (ZipOutputStream zip = new ZipOutputStream(out)) {
+ zip.putNextEntry(new ZipEntry("good.txt"));
+ zip.write("fine".getBytes(StandardCharsets.UTF_8));
+ zip.closeEntry();
+ zip.putNextEntry(new ZipEntry("../zip-escape.txt"));
+ zip.write("bad".getBytes(StandardCharsets.UTF_8));
+ zip.closeEntry();
+ }
+ final Path file = source.resolve("partial.zip");
+ Files.write(file, out.toByteArray());
+
+ final IOException thrown = Assertions.assertThrows(IOException.class,
+ () -> ResourceInstaller.install(file.toUri(), target));
+ Assertions.assertEquals(
+ "archive entry escapes the target directory: ../zip-escape.txt",
+ thrown.getMessage());
+ Assertions.assertEquals(List.of(), installedFiles(target));
+ }
+
+ /**
+ * Proves staged installation for a truncated archive: the tar breaks off inside its
+ * second entry header, and the first entry that unpacked cleanly must not appear.
+ */
+ @Test
+ void testTruncatedTarLeavesTargetEmpty(@TempDir Path source, @TempDir Path target)
+ throws Exception {
+ final ByteArrayOutputStream tar = new ByteArrayOutputStream();
+ tarEntry(tar, "first.txt", "complete".getBytes(StandardCharsets.UTF_8));
+ tarEntry(tar, "second.txt", "never finished".getBytes(StandardCharsets.UTF_8));
+ final byte[] whole = tar.toByteArray();
+ // Cut inside the second entry's header: first entry occupies two 512-byte blocks.
+ final byte[] truncated = Arrays.copyOf(whole, 2 * BLOCK + 100);
+ final Path file = source.resolve("truncated.tar.gz");
+ Files.write(file, gzip(truncated));
+
+ Assertions.assertThrows(IOException.class,
+ () -> ResourceInstaller.install(file.toUri(), target));
+ Assertions.assertEquals(List.of(), installedFiles(target));
+ }
+
+ /**
+ * Proves that a failed installation does not disturb files that already existed in
+ * the target directory before the attempt.
+ */
+ @Test
+ void testFailedInstallKeepsPreexistingTargetContent(@TempDir Path source,
+ @TempDir Path target) throws Exception {
+ Files.writeString(target.resolve("existing.txt"), "keep");
+ final byte[] archive = tarGz(new String[][] {
+ {"good.txt", "fine"},
+ {"../escape.txt", "bad"}});
+ final Path file = source.resolve("partial.tar.gz");
+ Files.write(file, archive);
+
+ Assertions.assertThrows(IOException.class,
+ () -> ResourceInstaller.install(file.toUri(), target));
+ Assertions.assertEquals(List.of("existing.txt"), installedFiles(target));
+ Assertions.assertEquals("keep", Files.readString(target.resolve("existing.txt")));
+ }
+
+ /**
+ * Proves that neither a successful nor a failed installation leaves hidden staging
+ * directories behind in the target.
+ */
+ @Test
+ void testInstallationLeavesNoStagingResidue(@TempDir Path source, @TempDir Path target)
+ throws Exception {
+ final byte[] archive = tarGz(new String[][] {{"corpus/data.txt", "content"}});
+ final Path file = source.resolve("clean.tar.gz");
+ Files.write(file, archive);
+
+ ResourceInstaller.install(file.toUri(), target);
+
+ try (Stream<Path> walk = Files.walk(target)) {
+ final List<String> hidden = walk
+ .filter(path -> !path.equals(target))
+ .map(path -> path.getFileName().toString())
+ .filter(fileName -> fileName.startsWith("."))
+ .toList();
+ Assertions.assertEquals(List.of(), hidden);
+ }
+ Assertions.assertEquals(List.of("corpus/data.txt"), installedFiles(target));
+ }
+
+ /**
+ * Enumerates invalid {@code Limits} constructions with the argument error each one
+ * must raise.
+ *
+ * @return One case per invalid argument. Never {@code null}.
+ */
+ static Stream<Arguments> invalidLimits() {
+ final Duration valid = Duration.ofSeconds(10);
+ return Stream.of(
+ Arguments.of("null connectTimeout", (Executable)
+ () -> new ResourceInstaller.Limits(null, valid, 5, 1024, 1024),
+ "connectTimeout must be positive"),
+ Arguments.of("zero connectTimeout", (Executable)
+ () -> new ResourceInstaller.Limits(Duration.ZERO, valid, 5, 1024, 1024),
+ "connectTimeout must be positive"),
+ Arguments.of("null readTimeout", (Executable)
+ () -> new ResourceInstaller.Limits(valid, null, 5, 1024, 1024),
+ "readTimeout must be positive"),
+ Arguments.of("negative readTimeout", (Executable)
+ () -> new ResourceInstaller.Limits(valid, Duration.ofSeconds(-1),
+ 5, 1024, 1024),
+ "readTimeout must be positive"),
+ Arguments.of("negative maxRedirects", (Executable)
+ () -> new ResourceInstaller.Limits(valid, valid, -1, 1024, 1024),
+ "maxRedirects must not be negative"),
+ Arguments.of("zero maxDownloadBytes", (Executable)
+ () -> new ResourceInstaller.Limits(valid, valid, 5, 0, 1024),
+ "maxDownloadBytes must be positive"),
+ Arguments.of("zero maxExpandedBytes", (Executable)
+ () -> new ResourceInstaller.Limits(valid, valid, 5, 1024, 0),
+ "maxExpandedBytes must be positive"));
+ }
+
+ @ParameterizedTest(name = "{0}")
+ @MethodSource("invalidLimits")
+ void testLimitsRejectInvalidValues(String label, Executable construction,
+ String message) {
+ final IllegalArgumentException thrown =
+ Assertions.assertThrows(IllegalArgumentException.class, construction);
+ Assertions.assertEquals(message, thrown.getMessage());
+ }
+
+ /**
+ * Proves the download ceiling on a file source: a fetch larger than the configured
+ * ceiling is rejected and nothing is installed.
+ */
+ @Test
+ void testDownloadCeilingRejectsOversizedSource(@TempDir Path source,
+ @TempDir Path target) throws Exception {
+ final Path file = source.resolve("large.txt");
+ Files.write(file, new byte[8192]);
+
+ final IOException thrown = Assertions.assertThrows(IOException.class,
+ () -> ResourceInstaller.install(file.toUri(), target, null,
+ ceilings(1024, 1024 * 1024)));
+ Assertions.assertEquals("download exceeds the ceiling of 1024 bytes",
thrown.getMessage());
+ Assertions.assertEquals(List.of(), installedFiles(target));
+ }
+
+ /**
+ * Proves the expansion ceiling on tar content: a small compressed archive whose
+ * entries expand beyond the ceiling is rejected and nothing is installed.
+ */
+ @Test
+ void testTarExpansionCeilingRejectsArchive(@TempDir Path source, @TempDir Path target)
+ throws Exception {
+ final ByteArrayOutputStream tar = new ByteArrayOutputStream();
+ tarEntry(tar, "bomb/zeros.bin", new byte[64 * 1024]);
+ tar.write(new byte[TERMINATOR_SIZE]);
+ final Path file = source.resolve("bomb.tar.gz");
+ Files.write(file, gzip(tar.toByteArray()));
+
+ final IOException thrown = Assertions.assertThrows(IOException.class,
+ () -> ResourceInstaller.install(file.toUri(), target, null,
+ ceilings(1024 * 1024, 1024)));
+ Assertions.assertEquals("expanded content exceeds the ceiling of 1024 bytes",
+ thrown.getMessage());
+ Assertions.assertEquals(List.of(), installedFiles(target));
+ }
+
+ /**
+ * Proves the expansion ceiling on zip content: highly compressible entries that
+ * expand beyond the ceiling are rejected and nothing is installed.
+ */
+ @Test
+ void testZipExpansionCeilingRejectsArchive(@TempDir Path source, @TempDir Path target)
+ throws Exception {
+ final ByteArrayOutputStream out = new ByteArrayOutputStream();
+ try (ZipOutputStream zip = new ZipOutputStream(out)) {
+ zip.putNextEntry(new ZipEntry("bomb/zeros.bin"));
+ zip.write(new byte[64 * 1024]);
+ zip.closeEntry();
+ }
+ final Path file = source.resolve("bomb.zip");
+ Files.write(file, out.toByteArray());
+
+ final IOException thrown = Assertions.assertThrows(IOException.class,
+ () -> ResourceInstaller.install(file.toUri(), target, null,
+ ceilings(1024 * 1024, 1024)));
+ Assertions.assertEquals("expanded content exceeds the ceiling of 1024 bytes",
+ thrown.getMessage());
+ Assertions.assertEquals(List.of(), installedFiles(target));
+ }
+
+ /**
+ * Proves the expansion ceiling on plain gzip content: a small gzip file whose
+ * decompressed form exceeds the ceiling is rejected and nothing is installed.
+ */
+ @Test
+ void testPlainGzipExpansionCeilingRejectsFile(@TempDir Path source,
+ @TempDir Path target) throws Exception {
+ final Path file = source.resolve("zeros.bin.gz");
+ Files.write(file, gzip(new byte[64 * 1024]));
+
+ final IOException thrown = Assertions.assertThrows(IOException.class,
+ () -> ResourceInstaller.install(file.toUri(), target, null,
+ ceilings(1024 * 1024, 1024)));
+ Assertions.assertEquals("expanded content exceeds the ceiling of 1024 bytes",
+ thrown.getMessage());
+ Assertions.assertEquals(List.of(), installedFiles(target));
+ }
+
+ /**
+ * Proves that an installation within custom ceilings succeeds, so the limits
+ * overload is usable for its intended purpose and not only for rejection.
+ */
+ @Test
+ void testInstallWithinCustomCeilingsSucceeds(@TempDir Path source, @TempDir Path target)
+ throws Exception {
+ final byte[] archive = tarGz(new String[][] {{"corpus/data.txt", "small"}});
+ final Path file = source.resolve("small.tar.gz");
+ Files.write(file, archive);
+
+ ResourceInstaller.install(file.toUri(), target, sha256(archive),
+ ceilings(1024 * 1024, 1024 * 1024));
+
+ Assertions.assertEquals("small", Files.readString(target.resolve("corpus/data.txt")));
}
}