(incubator-stormcrawler) 01/02: Connect to a remote instance using web sockets

[rz...@apache.org]

This is an automated email from the ASF dual-hosted git repository.

rzo1 pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/incubator-stormcrawler.git

commit b40f9db5e667d607477c4aa8930f27f1fccf0442
Author: Julien Nioche <ju...@digitalpebble.com>
AuthorDate: Fri Oct 18 10:28:49 2024 +0100

    Connect to a remote instance using web sockets
    
    Signed-off-by: Julien Nioche <ju...@digitalpebble.com>
---
 external/playwright/playwright-conf.yaml                 |  1 +
 .../stormcrawler/protocol/playwright/HttpProtocol.java   | 16 ++++++++++++++--
 2 files changed, 15 insertions(+), 2 deletions(-)

diff --git a/external/playwright/playwright-conf.yaml b/external/playwright/playwright-conf.yaml
index 4dff4624..93462be9 100644
--- a/external/playwright/playwright-conf.yaml
+++ b/external/playwright/playwright-conf.yaml
@@ -19,6 +19,7 @@ config:
   http.timeout: 5000
   
   # playwright.cdp.url: "http://localhost:9222"
+  # playwright.remote.ws: "ws://localhost:3000/"
   playwright.skip.download: true
   # com.microsoft.playwright.options.WaitUntilState
   # playwright.load.event: "domcontentloaded"
diff --git a/external/playwright/src/main/java/org/apache/stormcrawler/protocol/playwright/HttpProtocol.java b/external/playwright/src/main/java/org/apache/stormcrawler/protocol/playwright/HttpProtocol.java
index 30d69ec0..82d33e00 100644
--- a/external/playwright/src/main/java/org/apache/stormcrawler/protocol/playwright/HttpProtocol.java
+++ b/external/playwright/src/main/java/org/apache/stormcrawler/protocol/playwright/HttpProtocol.java
@@ -37,6 +37,8 @@ import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
 import java.util.Set;
+
+import org.apache.commons.lang.StringUtils;
 import org.apache.storm.Config;
 import org.apache.storm.utils.MutableInt;
 import org.apache.stormcrawler.Metadata;
@@ -81,6 +83,9 @@ public class HttpProtocol extends AbstractHttpProtocol {
         // "http://localhost:9222"
         final String CDP_URL = ConfUtils.getString(conf, "playwright.cdp.url");
 
+        // "ws://localhost:3000/"
+        final String REMOTE_WS = ConfUtils.getString(conf, "playwright.remote.ws");
+
         boolean skipDownloads = ConfUtils.getBoolean(conf, "playwright.skip.download", false);
 
         String loadEventString = ConfUtils.getString(conf, "playwright.load.event", "load");
@@ -100,7 +105,8 @@ public class HttpProtocol extends AbstractHttpProtocol {
         final CreateOptions creationOptions = new CreateOptions();
         final Map<String, String> env = new HashMap<>();
 
-        if (CDP_URL != null) {
+        // no need to download if we are connecting to a remote instance
+        if (StringUtils.isNotBlank(CDP_URL) || StringUtils.isNotBlank(REMOTE_WS)){
             skipDownloads = true;
         }
 
@@ -112,8 +118,14 @@ public class HttpProtocol extends AbstractHttpProtocol {
         creationOptions.setEnv(env);
 
         BrowserType btype = Playwright.create(creationOptions).chromium();
-        if (CDP_URL != null) {
+        if (StringUtils.isNotBlank(CDP_URL) && StringUtils.isNotBlank(REMOTE_WS)){
+            throw new RuntimeException("Can't specify both cdp.url and remote.ws in the configuration");
+        }
+        else if (StringUtils.isNotBlank(CDP_URL)) {
             browser = btype.connectOverCDP(CDP_URL);
+        }
+        else if (StringUtils.isNotBlank(REMOTE_WS)) {
+            browser = btype.connect(REMOTE_WS);
         } else {
             browser = btype.launch();
         }