You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@opennlp.apache.org by kr...@apache.org on 2026/10/04 09:02:30 UTC

(opennlp) branch fix/download-util-cached-lookup updated: DownloadUtil keeps a local copy of the model index

This is an automated email from the ASF dual-hosted git repository.

krickert pushed a commit to branch fix/download-util-cached-lookup
in repository https://gitbox.apache.org/repos/asf/opennlp.git


The following commit(s) were added to refs/heads/fix/download-util-cached-lookup by this push:
     new 18d85eb3f DownloadUtil keeps a local copy of the model index
18d85eb3f is described below

commit 18d85eb3faaa16eeee93040bb077bd210d42be96
Author: Kristian Rickert <kr...@apache.org>
AuthorDate: Sun Oct 4 05:02:22 2026 -0400

    DownloadUtil keeps a local copy of the model index
    
    Looking up a model by language reads the model index from the model
    host on every run, so a reset connection failed the lookup even when
    the model itself was already downloaded. DownloadUtil now keeps a copy
    of the last index it read in the download directory and uses it when
    the index cannot be read. A failed read no longer gives a partial index.
    
    Assisted-by: Claude Code 2.1.289
    Co-authored-by: Kristian Rickert <kr...@apache.org>
    Co-authored-by: Claude <no...@anthropic.com>
    Reviewed-by: Kristian Rickert <kr...@apache.org>
    Signed-off-by: Kristian Rickert <kr...@apache.org>
---
 .../main/java/opennlp/tools/util/DownloadUtil.java | 124 ++++++++++++++++++++-
 .../opennlp/tools/util/DownloadParserTest.java     |  87 ++++++++++++++-
 2 files changed, 206 insertions(+), 5 deletions(-)

diff --git a/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/util/DownloadUtil.java b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/util/DownloadUtil.java
index a350e7659..3342240a1 100644
--- a/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/util/DownloadUtil.java
+++ b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/util/DownloadUtil.java
@@ -26,6 +26,7 @@ import java.net.URI;
 import java.net.URISyntaxException;
 import java.net.URL;
 import java.nio.charset.StandardCharsets;
+import java.nio.file.AtomicMoveNotSupportedException;
 import java.nio.file.Files;
 import java.nio.file.Path;
 import java.nio.file.Paths;
@@ -61,6 +62,8 @@ public class DownloadUtil {
       System.getProperty("OPENNLP_DOWNLOAD_MODEL_PATH", "models/ud-models-1.3/");
   private static final String OPENNLP_DOWNLOAD_HOME = "OPENNLP_DOWNLOAD_HOME";
   private static final String CHECKSUM_EXTENSION = ".sha512";
+  private static final String INDEX_EXTENSION = ".index.html";
+  private static final String TEMP_EXTENSION = ".tmp";
 
   private static Map<String, Map<ModelType, URL>> availableModels;
 
@@ -181,10 +184,18 @@ public class DownloadUtil {
     }
   }
 
+  /**
+   * Lists the models published in the model index. A copy of the last index read is kept in
+   * the download directory and used when the index cannot be read.
+   *
+   * @return The model URLs per language and {@link ModelType}. Empty if the index could not
+   *     be read.
+   */
   public static Map<String, Map<ModelType, URL>> getAvailableModels() {
     if (availableModels == null) {
       try {
-        DownloadParser p = new DownloadParser(new URI(BASE_URL + MODEL_URI_PATH).toURL());
+        final DownloadParser p = new DownloadParser(new URI(BASE_URL + MODEL_URI_PATH).toURL(),
+            getDownloadHome().resolve(indexFileName(MODEL_URI_PATH)));
         availableModels = p.getAvailableModels();
       } catch (MalformedURLException | URISyntaxException e) {
         throw new RuntimeException(e);
@@ -343,6 +354,21 @@ public class DownloadUtil {
     }
   }
 
+  /**
+   * Names the local copy of the model index after the last segment of the model path, for
+   * example {@code ud-models-1.3.index.html} for {@code models/ud-models-1.3/}.
+   *
+   * @param modelPath The model path, with or without trailing slashes.
+   * @return The file name.
+   */
+  static String indexFileName(String modelPath) {
+    String path = modelPath;
+    while (path.endsWith("/")) {
+      path = path.substring(0, path.length() - 1);
+    }
+    return path.substring(path.lastIndexOf('/') + 1) + INDEX_EXTENSION;
+  }
+
   private static Path getDownloadHome() {
     return Paths.get(System.getProperty(OPENNLP_DOWNLOAD_HOME,
             System.getProperty("user.home"))).resolve(".opennlp");
@@ -356,15 +382,103 @@ public class DownloadUtil {
     private static final char TAG_END = '>';
 
     private final URL indexUrl;
+    private final Path localIndex;
 
+    /**
+     * Initializes a parser that always reads the index page from {@code indexUrl}.
+     *
+     * @param indexUrl The index page.
+     */
     DownloadParser(URL indexUrl) {
+      this(indexUrl, null);
+    }
+
+    /**
+     * Initializes a parser that keeps a copy of the index page.
+     *
+     * @param indexUrl The index page.
+     * @param localIndex The file that keeps a copy of the index page, or {@code null} to
+     *     keep no copy.
+     */
+    DownloadParser(URL indexUrl, Path localIndex) {
       Objects.requireNonNull(indexUrl);
       this.indexUrl = indexUrl;
+      this.localIndex = localIndex;
     }
 
+    /**
+     * Lists the models of the index page. If the page lists models, it is stored as the
+     * local copy. Otherwise, for example if the page cannot be read, the local copy is used.
+     *
+     * @return The model URLs per language and {@link ModelType}. Empty if neither the page
+     *     nor the local copy lists models.
+     * @throws MalformedURLException Thrown if a model URL is malformed.
+     * @throws URISyntaxException Thrown if a model URL is not a valid URI.
+     */
     Map<String, Map<ModelType, URL>> getAvailableModels()
         throws MalformedURLException, URISyntaxException {
-      return toMap(extractLinks(fetchPageIndex()));
+      final String page = fetchPageIndex();
+      if (page != null) {
+        final Map<String, Map<ModelType, URL>> models = toMap(extractLinks(page));
+        if (!models.isEmpty()) {
+          storeLocalIndex(page);
+          return models;
+        }
+      }
+      final String local = readLocalIndex();
+      if (local == null) {
+        return new HashMap<>();
+      }
+      logger.info("Using the local copy of the page index at {}.", localIndex);
+      return toMap(extractLinks(local));
+    }
+
+    /**
+     * Reads the local copy of the index page.
+     *
+     * @return The page content, or {@code null} if there is no readable copy.
+     */
+    private String readLocalIndex() {
+      if (localIndex == null || !Files.isRegularFile(localIndex)) {
+        return null;
+      }
+      try {
+        return Files.readString(localIndex, StandardCharsets.UTF_8);
+      } catch (IOException e) {
+        logger.warn("Could not read the local copy of the page index at {}.", localIndex, e);
+        return null;
+      }
+    }
+
+    /**
+     * Stores the index page as the local copy. The page is written to a temporary file that
+     * then replaces the copy, so a reader never sees a partly written copy. A failure is only
+     * logged.
+     *
+     * @param page The page content.
+     */
+    private void storeLocalIndex(String page) {
+      if (localIndex == null) {
+        return;
+      }
+      try {
+        final Path directory = localIndex.toAbsolutePath().getParent();
+        Files.createDirectories(directory);
+        final Path temporary =
+            Files.createTempFile(directory, localIndex.getFileName().toString(), TEMP_EXTENSION);
+        try {
+          Files.writeString(temporary, page, StandardCharsets.UTF_8);
+          try {
+            Files.move(temporary, localIndex, StandardCopyOption.ATOMIC_MOVE);
+          } catch (AtomicMoveNotSupportedException e) {
+            Files.move(temporary, localIndex, StandardCopyOption.REPLACE_EXISTING);
+          }
+        } finally {
+          Files.deleteIfExists(temporary);
+        }
+      } catch (IOException e) {
+        logger.warn("Could not store the local copy of the page index at {}.", localIndex, e);
+      }
     }
 
     /**
@@ -540,6 +654,11 @@ public class DownloadUtil {
       result.putIfAbsent(locale, models);
     }
 
+    /**
+     * Reads the index page from its URL, joining its lines.
+     *
+     * @return The page content, or {@code null} if the page could not be read completely.
+     */
     private String fetchPageIndex() {
       final StringBuilder html = new StringBuilder();
       try (BufferedReader br = new BufferedReader(
@@ -550,6 +669,7 @@ public class DownloadUtil {
         }
       } catch (IOException e) {
         logger.error("Could not read page index from {}", indexUrl, e);
+        return null;
       }
 
       return html.toString();
diff --git a/opennlp-tools/src/test/java/opennlp/tools/util/DownloadParserTest.java b/opennlp-tools/src/test/java/opennlp/tools/util/DownloadParserTest.java
index cbfe6d01a..e5e89a8e0 100644
--- a/opennlp-tools/src/test/java/opennlp/tools/util/DownloadParserTest.java
+++ b/opennlp-tools/src/test/java/opennlp/tools/util/DownloadParserTest.java
@@ -17,33 +17,44 @@
 
 package opennlp.tools.util;
 
+import java.io.IOException;
 import java.net.MalformedURLException;
 import java.net.URI;
 import java.net.URISyntaxException;
 import java.net.URL;
+import java.nio.charset.StandardCharsets;
+import java.nio.file.Files;
+import java.nio.file.Path;
 import java.util.List;
 import java.util.Map;
 import java.util.stream.Stream;
 
 import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.io.TempDir;
 import org.junit.jupiter.params.ParameterizedTest;
 import org.junit.jupiter.params.provider.Arguments;
 import org.junit.jupiter.params.provider.MethodSource;
+import org.junit.jupiter.params.provider.ValueSource;
 
 import opennlp.tools.models.ModelType;
 
 import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertFalse;
 import static org.junit.jupiter.api.Assertions.assertNotNull;
 import static org.junit.jupiter.api.Assertions.assertThrows;
+import static org.junit.jupiter.api.Assertions.assertTrue;
 import static org.junit.jupiter.api.Assertions.fail;
 
 public class DownloadParserTest {
 
+  private static final String INDEX = "opennlp/tools/util/index.html";
+  private static final int LANGUAGES = 36;
+
   @ParameterizedTest(name = "Verify \"{0}\" available models")
   @MethodSource(value = "expectedModels")
   void testAvailableModels(String language, Map<ModelType, String> expectedModels) {
 
-    final URL baseUrl = fromClasspath("opennlp/tools/util/index.html");
+    final URL baseUrl = fromClasspath(INDEX);
     assertNotNull(baseUrl);
 
     final DownloadUtil.DownloadParser downloadParser = new DownloadUtil.DownloadParser(baseUrl);
@@ -51,7 +62,7 @@ public class DownloadParserTest {
     try {
       Map<String, Map<ModelType, URL>> result = downloadParser.getAvailableModels();
       assertNotNull(result);
-      assertEquals(36, result.size());
+      assertEquals(LANGUAGES, result.size());
 
       final Map<ModelType, URL> availableModels = result.get(language);
       assertNotNull(availableModels);
@@ -119,7 +130,7 @@ public class DownloadParserTest {
   @MethodSource("indexPages")
   void testExtractLinks(String page, List<String> expected) {
     final DownloadUtil.DownloadParser downloadParser =
-        new DownloadUtil.DownloadParser(fromClasspath("opennlp/tools/util/index.html"));
+        new DownloadUtil.DownloadParser(fromClasspath(INDEX));
     assertEquals(expected, downloadParser.extractLinks(page));
   }
 
@@ -136,6 +147,76 @@ public class DownloadParserTest {
     }
   }
 
+  @Test
+  void testLocalIndexIsUsedIfTheUrlCannotBeRead(@TempDir Path home) throws Exception {
+    final Path localIndex = home.resolve("index.html");
+    Files.writeString(localIndex, indexPage(), StandardCharsets.UTF_8);
+    final URL unreachable = new URI("file:/this/does/not/exist/").toURL();
+
+    final Map<String, Map<ModelType, URL>> result =
+        new DownloadUtil.DownloadParser(unreachable, localIndex).getAvailableModels();
+
+    assertEquals(LANGUAGES, result.size());
+    assertEquals(unreachable + OPENNLP + "en-ud-ewt-" + MODEL_SENT + VER + BIN,
+        result.get("en").get(ModelType.SENTENCE_DETECTOR).toExternalForm());
+  }
+
+  @Test
+  void testIndexReadFromTheUrlIsStored(@TempDir Path home) throws Exception {
+    final Path localIndex = home.resolve("not/created/yet/index.html");
+
+    final Map<String, Map<ModelType, URL>> result = new DownloadUtil.DownloadParser(
+        fromClasspath(INDEX), localIndex).getAvailableModels();
+
+    assertEquals(LANGUAGES, result.size());
+    assertEquals(indexPage(), Files.readString(localIndex, StandardCharsets.UTF_8));
+    try (Stream<Path> files = Files.list(localIndex.getParent())) {
+      assertEquals(List.of(localIndex), files.toList());
+    }
+  }
+
+  @Test
+  void testIndexReadFromTheUrlReplacesTheLocalIndex(@TempDir Path home) throws Exception {
+    final Path localIndex = home.resolve("index.html");
+    Files.writeString(localIndex, "<html><body>Old</body></html>", StandardCharsets.UTF_8);
+
+    final Map<String, Map<ModelType, URL>> result = new DownloadUtil.DownloadParser(
+        fromClasspath(INDEX), localIndex).getAvailableModels();
+
+    assertEquals(LANGUAGES, result.size());
+    assertEquals(indexPage(), Files.readString(localIndex, StandardCharsets.UTF_8));
+  }
+
+  @Test
+  void testFailedReadStoresNothing(@TempDir Path home) throws Exception {
+    final Path localIndex = home.resolve("index.html");
+
+    final Map<String, Map<ModelType, URL>> result = new DownloadUtil.DownloadParser(
+        new URI("file:/this/does/not/exist").toURL(), localIndex).getAvailableModels();
+
+    assertTrue(result.isEmpty());
+    assertFalse(Files.exists(localIndex));
+  }
+
+  @ParameterizedTest
+  @ValueSource(strings = {"models/ud-models-1.3/", "models/ud-models-1.3", "models/ud-models-1.3//",
+      "ud-models-1.3", "/ud-models-1.3/"})
+  void testIndexFileName(String modelPath) {
+    assertEquals("ud-models-1.3.index.html", DownloadUtil.indexFileName(modelPath));
+  }
+
+  /**
+   * Reads the test index page as the parser keeps it: its lines joined without separators.
+   *
+   * @return The page content.
+   * @throws IOException Thrown if the resource cannot be read.
+   * @throws URISyntaxException Thrown if the resource URL is not a valid URI.
+   */
+  private String indexPage() throws IOException, URISyntaxException {
+    return String.join("", Files.readAllLines(
+        Path.of(fromClasspath(INDEX).toURI()), StandardCharsets.UTF_8));
+  }
+
   private URL fromClasspath(String file) {
     return Thread.currentThread().getContextClassLoader().getResource(file);
   }