(bifromq-sites) 01/01: fix: exclude development docs from indexing

[vi...@apache.org]

This is an automated email from the ASF dual-hosted git repository.

visoar pushed a commit to branch agent/fix-development-doc-indexing
in repository https://gitbox.apache.org/repos/asf/bifromq-sites.git

commit 935682ea3ad92ff42c23a4d682527dd613de21fd
Author: Visoar Zhang <vi...@gmail.com>
AuthorDate: Fri Sep 4 22:30:46 2026 +0800

    fix: exclude development docs from indexing
---
 docusaurus.config.ts          |  1 +
 scripts/check-site-policy.mjs | 18 ++++++++++++++++++
 2 files changed, 19 insertions(+)

diff --git a/docusaurus.config.ts b/docusaurus.config.ts
index 4333ea28..fb242956 100644
--- a/docusaurus.config.ts
+++ b/docusaurus.config.ts
@@ -79,6 +79,7 @@ const config: Config = {
               path: 'development',
               banner: 'unreleased',
               badge: true,
+              noIndex: true,
             },
             '4.0.0-incubating': {
               label: `${releaseVersion} (latest)`,
diff --git a/scripts/check-site-policy.mjs b/scripts/check-site-policy.mjs
index d34952a5..b829e8c0 100644
--- a/scripts/check-site-policy.mjs
+++ b/scripts/check-site-policy.mjs
@@ -20,6 +20,7 @@ import {extname, join, relative} from 'node:path';
 
 const buildDir = new URL('../build/', import.meta.url).pathname;
 const canonicalOrigin = 'https://bifromq.apache.org';
+const developmentVersion = 'development';
 const archivedVersions = ['3.2.x', '3.1.x', '3.0.x', '2.1.x', '2.0.0', '1.0.x'];
 const approvedExternalResources = new Set([
   'https://analytics.apache.org/matomo.js',
@@ -128,6 +129,23 @@ if (!sitemap.includes('https://bifromq.apache.org/docs/3.3.x/')) {
   fail('The latest historical 3.x documentation must remain in the sitemap.');
 }
 
+const developmentUrl = `${canonicalOrigin}/docs/${developmentVersion}/`;
+if (sitemap.includes(developmentUrl)) {
+  fail('Unreleased development documentation must not be in the sitemap.');
+}
+
+const developmentDir = join(buildDir, 'docs', developmentVersion);
+const developmentHtmlFiles = findHtmlFiles(developmentDir);
+if (developmentHtmlFiles.length === 0) {
+  fail('Unreleased development documentation was not built.');
+}
+for (const htmlFile of developmentHtmlFiles) {
+  const html = readFileSync(htmlFile, 'utf8');
+  if (!html.includes('name="robots" content="noindex, nofollow"')) {
+    fail(`Development page is missing the Docusaurus noindex directive: ${htmlFile}`);
+  }
+}
+
 for (const version of archivedVersions) {
   const versionUrl = `https://bifromq.apache.org/docs/${version}/`;
   if (sitemap.includes(versionUrl)) {