[whimsy] branch master updated (20788cb9 -> b160224a)

[se...@apache.org]

This is an automated email from the ASF dual-hosted git repository.

sebb pushed a change to branch master
in repository https://gitbox.apache.org/repos/asf/whimsy.git


    from 20788cb9 Must replace existing entry
     new 34f37cbf Extend scanner to show raw URLs
     new b160224a Basic page scanner (members only)

The 2 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails.  The revisions
listed as "add" were already present in the repository and have only
been added to this reference.


Summary of changes:
 tools/scan-page.js           | 14 ++++++++++----
 www/members/page-scanner.cgi | 21 +++++++++++++++++++++
 2 files changed, 31 insertions(+), 4 deletions(-)
 create mode 100755 www/members/page-scanner.cgi

[whimsy] 02/02: Basic page scanner (members only)

[se...@apache.org]

This is an automated email from the ASF dual-hosted git repository.

sebb pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/whimsy.git

commit b160224a3d400655e2fbc7a9adec395ddd34ac08
Author: Sebb <se...@apache.org>
AuthorDate: Tue May 3 23:15:27 2022 +0100

    Basic page scanner (members only)
---
 www/members/page-scanner.cgi | 21 +++++++++++++++++++++
 1 file changed, 21 insertions(+)

diff --git a/www/members/page-scanner.cgi b/www/members/page-scanner.cgi
new file mode 100755
index 00000000..81b7b073
--- /dev/null
+++ b/www/members/page-scanner.cgi
@@ -0,0 +1,21 @@
+#!/usr/bin/env ruby
+PAGETITLE = "ASF Page Asset Checker - ALPHA"
+
+# very rudimentary page asset checker - shows references to non-ASF assets
+
+# usage: whimsy.apache.org/members/page-scanner?url=http://apache.org/
+
+print "Content-type: text/plain; charset=UTF-8\r\n\r\n"
+
+# puts ENV['REQUEST_URI']
+qs = ENV['QUERY_STRING']
+if qs =~ %r{^url=(https?://.+)}
+  url = $1
+  print "Checking the page #{url}\n\n"
+  print "=====\n"
+  system('node', '/srv/whimsy/tools/scan-page.js', url, 'all')
+  print "=====\n"
+else
+  print "Expecting: ?url=http://.../\n"
+end
+

[whimsy] 01/02: Extend scanner to show raw URLs

[se...@apache.org]

This is an automated email from the ASF dual-hosted git repository.

sebb pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/whimsy.git

commit 34f37cbf628bb372913a0fed8327a957aee8ee70
Author: Sebb <se...@apache.org>
AuthorDate: Tue May 3 23:15:00 2022 +0100

    Extend scanner to show raw URLs
---
 tools/scan-page.js | 14 ++++++++++----
 1 file changed, 10 insertions(+), 4 deletions(-)

diff --git a/tools/scan-page.js b/tools/scan-page.js
index 811b082c..55b06cdc 100755
--- a/tools/scan-page.js
+++ b/tools/scan-page.js
@@ -7,9 +7,10 @@ module.paths.push('/usr/lib/node_modules')
 const puppeteer = require('puppeteer');
 
 const target = process.argv[2] || 'http://apache.org/';
+const option = process.argv[3] || '';
 
 function isASFhost(host) {
-    return host == 'apache.org' || host.endsWith('.apache.org') || host.endsWith('.apachecon.com');
+    return host == '' || host == 'apache.org' || host.endsWith('.apache.org') || host.endsWith('.apachecon.com');
 }
 
 (async () => {
@@ -26,10 +27,15 @@ function isASFhost(host) {
         interceptedRequest.continue();
     } else {
         let host = new URL(url).host
-        // don't visit non-ASF hosts
         if (!isASFhost(host)) {
-            console.log(host);
-            interceptedRequest.abort();
+            // don't visit non-ASF hosts unless requested
+            if (option == 'all') {
+                console.log(url);
+                interceptedRequest.continue();
+            } else {
+                console.log(host);
+                interceptedRequest.abort();
+            }
         } else { 
             // Need to visit at least an initial redirect
             interceptedRequest.continue();