(stormcrawler) 01/01: feat(playwright): capture rendered DOM on non-2xx responses

[rz...@apache.org]

This is an automated email from the ASF dual-hosted git repository.

rzo1 pushed a commit to branch 1890
in repository https://gitbox.apache.org/repos/asf/stormcrawler.git

commit b595fb421c796f4e4ad20d2a19c1420a1bc6cc59
Author: Richard Zowalla <rz...@apache.org>
AuthorDate: Mon Apr 27 09:48:45 2026 +0200

    feat(playwright): capture rendered DOM on non-2xx responses
    
    Add two opt-in configuration flags to the Playwright protocol so that
    Single-Page Applications which return a non-2xx stub document and then
    hydrate via JavaScript can be crawled correctly:
    
    - playwright.capture.content.on.error: also capture page.content() when
      the origin response status is not 2xx (default false, preserves
      current behavior).
    - playwright.override.status.on.content: when content was captured for
      a non-2xx response, report status 200 to downstream components and
      preserve the original status in the response metadata under
      playwright.origin.status (default false).
    
    Document both flags in the module README and playwright-conf.yaml.
---
 external/playwright/README.md                      | 19 ++++++++++++++++
 external/playwright/playwright-conf.yaml           | 12 ++++++++++
 .../protocol/playwright/HttpProtocol.java          | 26 ++++++++++++++++++++--
 3 files changed, 55 insertions(+), 2 deletions(-)

diff --git a/external/playwright/README.md b/external/playwright/README.md
index 4cbb1d13..725bea64 100644
--- a/external/playwright/README.md
+++ b/external/playwright/README.md
@@ -28,3 +28,22 @@ mvn exec:java -e -Dexec.mainClass=com.microsoft.playwright.CLI -Dexec.args="inst
 
 The setting `playwright.skip.download` to `true` in the configuration will assume that the browser has been installed and will not trigger the installation of all the different browsers.
 
+## Configuration
+
+| Property | Default | Description |
+|---|---|---|
+| `playwright.cdp.url` | _unset_ | If set, connect to an existing Chrome instance via the Chrome DevTools Protocol (e.g. `http://localhost:9222`) instead of launching a browser. |
+| `playwright.remote.ws` | _unset_ | If set, connect to a remote Playwright server over WebSocket (e.g. `ws://localhost:3000/`). Mutually exclusive with `playwright.cdp.url`. |
+| `playwright.skip.download` | `false` | If `true`, sets `PLAYWRIGHT_SKIP_BROWSER_DOWNLOAD=true` so Playwright will not install browsers. Implicitly forced to `true` when `playwright.cdp.url` or `playwright.remote.ws` is set. |
+| `playwright.load.event` | `load` | The Playwright `WaitUntilState` to wait for before considering the page ready. Accepts `load`, `domcontentloaded`, or `networkidle`. |
+| `playwright.skip.resource.types` | _empty_ | List of resource types to abort during navigation (`document`, `stylesheet`, `image`, `media`, `font`, `script`, `texttrack`, `xhr`, `fetch`, `eventsource`, `websocket`, `manifest`, `other`). |
+| `playwright.evaluations` | _empty_ | List of JavaScript expressions evaluated on the page after load. Each result is JSON-serialized and stored in the response metadata under the expression itself as the key. |
+| `playwright.capture.content.on.error` | `false` | By default the rendered DOM is only captured when the origin returns a 2xx status. Set to `true` to also capture `page.content()` for non-2xx responses — useful for Single-Page Applications that return a non-2xx stub document and then hydrate the real content via JavaScript. |
+| `playwright.override.status.on.content` | `false` | When the rendered DOM was captured for a non-2xx response, override the reported HTTP status with `200` so downstream components treat the URL as `FETCHED`. The original origin status is preserved in the response metadata under the key `playwright.origin.status`. No-op unless `playwright.capture.content.on.error` is also `true`. |
+
+Per-URL metadata triggers:
+
+| Metadata key | Effect |
+|---|---|
+| `playwright.trace` | If present on the input metadata, a Playwright trace zip is recorded for the navigation and its path is returned in the response metadata under the same key. |
+
diff --git a/external/playwright/playwright-conf.yaml b/external/playwright/playwright-conf.yaml
index a6d16c61..1020537b 100644
--- a/external/playwright/playwright-conf.yaml
+++ b/external/playwright/playwright-conf.yaml
@@ -24,3 +24,15 @@ config:
   # com.microsoft.playwright.options.WaitUntilState
   # playwright.load.event: "domcontentloaded"
 
+  # By default the rendered DOM is only captured when the origin returns a 2xx
+  # status. Enable this to also capture page.content() for non-2xx responses
+  # (useful for SPAs that return e.g. a 404 stub and then hydrate via JS).
+  # playwright.capture.content.on.error: false
+
+  # When the rendered DOM was captured for a non-2xx response, override the
+  # reported HTTP status with 200 so downstream components treat the URL as
+  # FETCHED. The original origin status is preserved in the response metadata
+  # under the key "playwright.origin.status". No-op unless
+  # playwright.capture.content.on.error is also true.
+  # playwright.override.status.on.content: false
+
diff --git a/external/playwright/src/main/java/org/apache/stormcrawler/protocol/playwright/HttpProtocol.java b/external/playwright/src/main/java/org/apache/stormcrawler/protocol/playwright/HttpProtocol.java
index ef1f2148..b19e1701 100644
--- a/external/playwright/src/main/java/org/apache/stormcrawler/protocol/playwright/HttpProtocol.java
+++ b/external/playwright/src/main/java/org/apache/stormcrawler/protocol/playwright/HttpProtocol.java
@@ -62,6 +62,10 @@ public class HttpProtocol extends AbstractHttpProtocol {
 
     private int timeout = 10000;
 
+    private boolean captureContentOnError = false;
+
+    private boolean overrideStatusOnContent = false;
+
     private BrowserContext context;
 
     private Set<String> resourceTypesToSkip;
@@ -130,6 +134,12 @@ public class HttpProtocol extends AbstractHttpProtocol {
 
         timeout = ConfUtils.getInt(conf, "http.timeout", timeout);
 
+        captureContentOnError =
+                ConfUtils.getBoolean(conf, "playwright.capture.content.on.error", false);
+
+        overrideStatusOnContent =
+                ConfUtils.getBoolean(conf, "playwright.override.status.on.content", false);
+
         final String ua = getAgentString(conf);
 
         NewContextOptions b_c_options =
@@ -245,12 +255,24 @@ public class HttpProtocol extends AbstractHttpProtocol {
                                         responseMetaData.addValue(k, v);
                                     });
 
-                    if (Status.FETCHED == Status.fromHTTPCode(response.status())) {
+                    int httpStatus = response.status();
+                    boolean fetched = Status.FETCHED == Status.fromHTTPCode(httpStatus);
+                    boolean contentCaptured = false;
+
+                    if (fetched || captureContentOnError) {
                         // retrieve the rendered content
                         content = page.content().getBytes(StandardCharsets.UTF_8);
+                        contentCaptured = true;
                     }
 
-                    status.set(response.status());
+                    if (!fetched && contentCaptured && overrideStatusOnContent) {
+                        // expose the original origin status for diagnostics
+                        responseMetaData.setValue(
+                                "playwright.origin.status", Integer.toString(httpStatus));
+                        status.set(200);
+                    } else {
+                        status.set(httpStatus);
+                    }
 
                     // evaluate an expression and store the results
                     // in the metadata using the same string as key