(stormcrawler) 01/01: Resolve relative outlinks against in JSoupParserBolt

[jn...@apache.org]

This is an automated email from the ASF dual-hosted git repository.

jnioche pushed a commit to branch JSOUPbaseURI
in repository https://gitbox.apache.org/repos/asf/stormcrawler.git

commit d5987a12ccb743a557689e218ac420883fdba6a1
Author: Julien Nioche <ju...@digitalpebble.com>
AuthorDate: Tue Jul 21 17:31:03 2026 +0100

    Resolve relative outlinks against <base href> in JSoupParserBolt
    
    Relative links were resolved against the document URL, ignoring any
    <base href> element. On a page served from a sub-path with a base
    pointing at the domain root this doubled the path segment. Resolve
    against jsoupDoc.baseUri() instead, which honours <base> and falls
    back to the document URL when absent.
---
 .../apache/stormcrawler/bolt/JSoupParserBolt.java  | 11 +++++++-
 .../stormcrawler/bolt/JSoupParserBoltTest.java     | 33 ++++++++++++++++++++++
 2 files changed, 43 insertions(+), 1 deletion(-)

diff --git a/core/src/main/java/org/apache/stormcrawler/bolt/JSoupParserBolt.java b/core/src/main/java/org/apache/stormcrawler/bolt/JSoupParserBolt.java
index 3d63fd6c..577d828f 100644
--- a/core/src/main/java/org/apache/stormcrawler/bolt/JSoupParserBolt.java
+++ b/core/src/main/java/org/apache/stormcrawler/bolt/JSoupParserBolt.java
@@ -338,7 +338,16 @@ public class JSoupParserBolt extends StatusEmitterBolt {
                 } else {
                     final Elements links = jsoupDoc.select("a[href]");
                     slinks = new HashMap<>(links.size());
-                    final URL baseUrl = URLUtil.toURL(url);
+                    // Resolve relative links against the document's base URI
+                    // rather than its URL: when the page contains a <base href>
+                    // element jsoup updates baseUri() accordingly, which is what
+                    // browsers use to resolve relative hrefs. Falls back to the
+                    // document URL when no (valid) <base> is present.
+                    String baseUri = jsoupDoc.baseUri();
+                    if (StringUtils.isBlank(baseUri)) {
+                        baseUri = url;
+                    }
+                    final URL baseUrl = URLUtil.toURL(baseUri);
                     for (Element link : links) {
                         // nofollow
                         String[] relkeywords = link.attr("rel").split(" ");
diff --git a/core/src/test/java/org/apache/stormcrawler/bolt/JSoupParserBoltTest.java b/core/src/test/java/org/apache/stormcrawler/bolt/JSoupParserBoltTest.java
index 100344d4..05ae595c 100644
--- a/core/src/test/java/org/apache/stormcrawler/bolt/JSoupParserBoltTest.java
+++ b/core/src/test/java/org/apache/stormcrawler/bolt/JSoupParserBoltTest.java
@@ -304,4 +304,37 @@ class JSoupParserBoltTest extends ParsingTester {
         Assertions.assertEquals(
                 "http://www.javascriptlinks.com/mylink", statusTuples.get(0).get(0));
     }
+
+    /**
+     * Relative outlinks must be resolved against the document's {@code <base href>} element when
+     * one is present, matching browser behaviour, rather than against the document URL. Otherwise a
+     * page served from a sub-path with a base pointing at the domain root doubles the path segment.
+     */
+    @Test
+    void testBaseHrefOutlinkResolution() throws IOException {
+        bolt.prepare(stormConf, TestUtil.getMockedTopologyContext(), new OutputCollector(output));
+
+        String html =
+                "<html><head>"
+                        + "<base href=\"https://www.example.be/\">"
+                        + "</head><body>"
+                        + "<a href=\"fr/rayons/stockage-archivage-de-donnees/218725\">Stockage</a>"
+                        + "</body></html>";
+
+        Metadata metadata = new Metadata();
+        metadata.setValue("Content-Type", "text/html; charset=UTF-8");
+        parse(
+                "https://www.example.be/fr/rayons/actualites-reportages/216943",
+                html.getBytes(StandardCharsets.UTF_8),
+                metadata);
+
+        List<List<Object>> statusTuples = output.getEmitted(Constants.StatusStreamName);
+        Assertions.assertEquals(1, statusTuples.size());
+        Assertions.assertEquals(Status.DISCOVERED, statusTuples.get(0).get(2));
+        // resolved against the <base href> (domain root), NOT against the document path,
+        // which would have produced .../actualites-reportages/fr/rayons/stockage-...
+        Assertions.assertEquals(
+                "https://www.example.be/fr/rayons/stockage-archivage-de-donnees/218725",
+                statusTuples.get(0).get(0));
+    }
 }