You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@stormcrawler.apache.org by rz...@apache.org on 2024/05/17 12:15:17 UTC

(incubator-stormcrawler) branch playwright-dynamic-loading created (now 88e89f40)

This is an automated email from the ASF dual-hosted git repository.

rzo1 pushed a change to branch playwright-dynamic-loading
in repository https://gitbox.apache.org/repos/asf/incubator-stormcrawler.git


      at 88e89f40 Implements a unit test to actually test that playwright is rendering HTML/JS content correctly. Adds an option to specify wait until state to ensure JS is executed properly Removes PDF test, which was a "false friend" because it was returned by the mock server with an HTML mimetype and the actual returned PDF object was corrupted.

This branch includes the following new commits:

     new 88e89f40 Implements a unit test to actually test that playwright is rendering HTML/JS content correctly. Adds an option to specify wait until state to ensure JS is executed properly Removes PDF test, which was a "false friend" because it was returned by the mock server with an HTML mimetype and the actual returned PDF object was corrupted.

The 1 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails.  The revisions
listed as "add" were already present in the repository and have only
been added to this reference.



(incubator-stormcrawler) 01/01: Implements a unit test to actually test that playwright is rendering HTML/JS content correctly. Adds an option to specify wait until state to ensure JS is executed properly Removes PDF test, which was a "false friend" because it was returned by the mock server with an HTML mimetype and the actual returned PDF object was corrupted.

Posted by rz...@apache.org.
This is an automated email from the ASF dual-hosted git repository.

rzo1 pushed a commit to branch playwright-dynamic-loading
in repository https://gitbox.apache.org/repos/asf/incubator-stormcrawler.git

commit 88e89f40120a5892bf882c1038cce06f5cdd15e6
Author: Richard Zowalla <rz...@apache.org>
AuthorDate: Fri May 17 14:15:11 2024 +0200

    Implements a unit test to actually test that playwright is rendering HTML/JS content correctly.
    Adds an option to specify wait until state to ensure JS is executed properly
    Removes PDF test, which was a "false friend" because it was returned by the mock server with an HTML mimetype and the actual returned PDF object was corrupted.
---
 external/playwright/playwright-conf.yaml           |   2 +
 .../protocol/playwright/HttpProtocol.java          |  48 ++++++++++++++++-----
 .../protocol/playwright/LocalResourceHandler.java  |   5 +--
 .../protocol/playwright/ProtocolTest.java          |  14 ++++--
 .../src/test/resources/dynamic-scraping.html       |  32 ++++++++++++++
 .../playwright/src/test/resources/pdf-test.pdf     | Bin 20597 -> 0 bytes
 6 files changed, 85 insertions(+), 16 deletions(-)

diff --git a/external/playwright/playwright-conf.yaml b/external/playwright/playwright-conf.yaml
index 5a51b8a6..4298ac2d 100644
--- a/external/playwright/playwright-conf.yaml
+++ b/external/playwright/playwright-conf.yaml
@@ -5,4 +5,6 @@ config:
   
   # playwright.cdp.url: "http://localhost:9222"
   playwright.skip.download: true
+  # com.microsoft.playwright.options.WaitUntilState
+  # playwright.load.event: "domcontentloaded"
 
diff --git a/external/playwright/src/main/java/org/apache/stormcrawler/protocol/playwright/HttpProtocol.java b/external/playwright/src/main/java/org/apache/stormcrawler/protocol/playwright/HttpProtocol.java
index 937d87ad..30d69ec0 100644
--- a/external/playwright/src/main/java/org/apache/stormcrawler/protocol/playwright/HttpProtocol.java
+++ b/external/playwright/src/main/java/org/apache/stormcrawler/protocol/playwright/HttpProtocol.java
@@ -27,6 +27,8 @@ import com.microsoft.playwright.Playwright;
 import com.microsoft.playwright.Playwright.CreateOptions;
 import com.microsoft.playwright.Tracing;
 import com.microsoft.playwright.options.Proxy;
+import com.microsoft.playwright.options.WaitUntilState;
+import java.nio.charset.StandardCharsets;
 import java.nio.file.Files;
 import java.nio.file.Path;
 import java.nio.file.attribute.FileAttribute;
@@ -70,6 +72,8 @@ public class HttpProtocol extends AbstractHttpProtocol {
     // globals like window and document can be used in evaluate.
     private List<String> evaluations;
 
+    private WaitUntilState loadEvent;
+
     @Override
     public void configure(final Config conf) {
         super.configure(conf);
@@ -79,6 +83,20 @@ public class HttpProtocol extends AbstractHttpProtocol {
 
         boolean skipDownloads = ConfUtils.getBoolean(conf, "playwright.skip.download", false);
 
+        String loadEventString = ConfUtils.getString(conf, "playwright.load.event", "load");
+        // map load event string to playwright native wait stat
+        switch (loadEventString) {
+            case "domcontentloaded":
+                loadEvent = WaitUntilState.DOMCONTENTLOADED;
+                break;
+            case "networkidle":
+                loadEvent = WaitUntilState.NETWORKIDLE;
+                break;
+            default:
+                loadEvent = WaitUntilState.LOAD;
+                break;
+        }
+
         final CreateOptions creationOptions = new CreateOptions();
         final Map<String, String> env = new HashMap<>();
 
@@ -152,8 +170,8 @@ public class HttpProtocol extends AbstractHttpProtocol {
                                         .setSources(true));
             }
 
-            final Metadata responsemetadata = new Metadata();
-            responsemetadata.addValue(MD_KEY_START, Instant.now().toString());
+            final Metadata responseMetaData = new Metadata();
+            responseMetaData.addValue(MD_KEY_START, Instant.now().toString());
 
             final MutableInt status = new MutableInt(-1);
             byte[] content = new byte[0];
@@ -171,7 +189,7 @@ public class HttpProtocol extends AbstractHttpProtocol {
                                     response.allHeaders()
                                             .forEach(
                                                     (k, v) -> {
-                                                        responsemetadata.addValue(k, v);
+                                                        responseMetaData.addValue(k, v);
                                                     });
                                 }
                             }
@@ -200,7 +218,13 @@ public class HttpProtocol extends AbstractHttpProtocol {
                             }
                         });
 
-                com.microsoft.playwright.Response response = page.navigate(url);
+                // let playwright do the content loading
+                com.microsoft.playwright.Response response =
+                        page.navigate(
+                                url,
+                                new Page.NavigateOptions()
+                                        .setTimeout(timeout)
+                                        .setWaitUntil(loadEvent));
 
                 // the status is not set unless
                 // a redirection
@@ -208,10 +232,14 @@ public class HttpProtocol extends AbstractHttpProtocol {
                     response.allHeaders()
                             .forEach(
                                     (k, v) -> {
-                                        responsemetadata.addValue(k, v);
+                                        responseMetaData.addValue(k, v);
                                     });
 
-                    content = response.body();
+                    if (Status.FETCHED == Status.fromHTTPCode(response.status())) {
+                        // retrieve the rendered content
+                        content = page.content().getBytes(StandardCharsets.UTF_8);
+                    }
+
                     status.set(response.status());
 
                     // evaluate an expression and store the results
@@ -222,7 +250,7 @@ public class HttpProtocol extends AbstractHttpProtocol {
                             String json =
                                     mapper.writerWithDefaultPrettyPrinter()
                                             .writeValueAsString(performance);
-                            responsemetadata.setValue(expression, json);
+                            responseMetaData.setValue(expression, json);
                         }
                     }
                 }
@@ -231,12 +259,12 @@ public class HttpProtocol extends AbstractHttpProtocol {
             if (md.containsKey(MD_TRACE)) {
                 Path tmp = Files.createTempFile("trace-", ".zip", new FileAttribute[0]);
                 context.tracing().stop(new Tracing.StopOptions().setPath(tmp));
-                responsemetadata.setValue(MD_TRACE, tmp.toString());
+                responseMetaData.setValue(MD_TRACE, tmp.toString());
             }
 
-            responsemetadata.addValue(MD_KEY_END, Instant.now().toString());
+            responseMetaData.addValue(MD_KEY_END, Instant.now().toString());
 
-            return new ProtocolResponse(content, status.get(), responsemetadata);
+            return new ProtocolResponse(content, status.get(), responseMetaData);
         }
     }
 
diff --git a/external/playwright/src/test/java/org/apache/stormcrawler/protocol/playwright/LocalResourceHandler.java b/external/playwright/src/test/java/org/apache/stormcrawler/protocol/playwright/LocalResourceHandler.java
index 261f3c3e..7c3cf8d9 100644
--- a/external/playwright/src/test/java/org/apache/stormcrawler/protocol/playwright/LocalResourceHandler.java
+++ b/external/playwright/src/test/java/org/apache/stormcrawler/protocol/playwright/LocalResourceHandler.java
@@ -43,7 +43,8 @@ public class LocalResourceHandler extends AbstractHandler {
 
         baseRequest.setHandled(true);
 
-        InputStream stream = getClass().getClassLoader().getResourceAsStream(s.substring(1));
+        final String file = s.substring(1);
+        InputStream stream = getClass().getClassLoader().getResourceAsStream(file);
 
         // not found?
         if (stream == null) {
@@ -57,9 +58,7 @@ public class LocalResourceHandler extends AbstractHandler {
 
         response.setHeader("key", "value");
         response.setStatus(HttpServletResponse.SC_OK);
-        // any content type will do
         response.setContentType("text/html");
-        // response.setContentLength(barray.length);
         try (OutputStream out = response.getOutputStream()) {
             out.write(barray);
         }
diff --git a/external/playwright/src/test/java/org/apache/stormcrawler/protocol/playwright/ProtocolTest.java b/external/playwright/src/test/java/org/apache/stormcrawler/protocol/playwright/ProtocolTest.java
index 61aeae24..41c464d7 100644
--- a/external/playwright/src/test/java/org/apache/stormcrawler/protocol/playwright/ProtocolTest.java
+++ b/external/playwright/src/test/java/org/apache/stormcrawler/protocol/playwright/ProtocolTest.java
@@ -17,6 +17,7 @@
 
 package org.apache.stormcrawler.protocol.playwright;
 
+import java.nio.charset.StandardCharsets;
 import java.time.Instant;
 import org.apache.commons.lang.mutable.MutableBoolean;
 import org.apache.storm.Config;
@@ -75,15 +76,22 @@ public class ProtocolTest extends AbstractProtocolTest {
     }
 
     @Test
-    public void testPDF() throws Exception {
+    public void testHTML() throws Exception {
         HttpProtocol protocol = getProtocol();
-        String url = "http://localhost:" + HTTP_PORT + "/pdf-test.pdf";
+        String url = "http://localhost:" + HTTP_PORT + "/dynamic-scraping.html";
         ProtocolResponse response = protocol.getProtocolOutput(url, new Metadata());
         // check that we have the metadata we expect
         Assert.assertNotNull(response.getMetadata().getFirstValue("key"));
         // the correct code
         Assert.assertEquals(200, response.getStatusCode());
-        Assert.assertEquals(32027, response.getContent().length);
+
+        final String content = new String(response.getContent(), StandardCharsets.UTF_8);
+        Assert.assertNotNull(content);
+
+        // we expect that the given JS was executed, so the content should contain this HTML snippet
+        Assert.assertTrue(
+                content.contains(
+                        "<p><a href=\"https://stormcrawler.apache.org/\">StormCrawler Rocks!</a></p>"));
     }
 
     /** Calls 2 URLs on the same protocol instance - should block * */
diff --git a/external/playwright/src/test/resources/dynamic-scraping.html b/external/playwright/src/test/resources/dynamic-scraping.html
new file mode 100644
index 00000000..9c9d56e9
--- /dev/null
+++ b/external/playwright/src/test/resources/dynamic-scraping.html
@@ -0,0 +1,32 @@
+<!DOCTYPE html>
+<html>
+<head>
+    <title>StormCrawler Example</title>
+</head>
+<body>
+<noscript>
+    Booh booh! No JavaScript!
+</noscript>
+<script>
+    document.addEventListener("DOMContentLoaded", function() {
+        // Create a paragraph element
+        var p = document.createElement("p");
+
+        // Create an anchor (link) element
+        var a = document.createElement("a");
+
+        // Set the href attribute of the anchor element
+        a.href = "https://stormcrawler.apache.org/";
+
+        // Set the text content of the anchor element
+        a.textContent = "StormCrawler Rocks!";
+
+        // Append the anchor element to the paragraph element
+        p.appendChild(a);
+
+        // Output the paragraph element to the document body
+        document.body.appendChild(p);
+    });
+</script>
+</body>
+</html>
diff --git a/external/playwright/src/test/resources/pdf-test.pdf b/external/playwright/src/test/resources/pdf-test.pdf
deleted file mode 100644
index f46dbe56..00000000
Binary files a/external/playwright/src/test/resources/pdf-test.pdf and /dev/null differ