chore(docs): upgrade Docosaurus to 3.9.2 with Rspack and SWC optimizations (#68600)

2025-12-19 18:14:56 -05:00 · 2025-10-24 07:43:39 +02:00
parent 799eb42060
commit 72b6301730
30 changed files with 6698 additions and 4131 deletions
--- a/.gitignore
+++ b/.gitignore
@@ -118,3 +118,4 @@ airbyte-integrations/connectors/**/airbyte-cdk-load/

 # AI
 AGENT.md
+.claude/settings.local.json
--- a/docusaurus/.gitignore
+++ b/docusaurus/.gitignore
@@ -24,3 +24,4 @@ yarn-debug.log*
 yarn-error.log*

 CLAUDE.local.md
+.claude/settings.local.json
--- a/docusaurus/docusaurus.config.ts
+++ b/docusaurus/docusaurus.config.ts
@@ -1,30 +1,50 @@
-// @ts-check
-// Note: type annotations allow type checking and IDEs autocompletion
 import "dotenv/config.js";
+import type { Config } from "@docusaurus/types";
+import { themes as prismThemes } from "prism-react-renderer";
+import type { Options as ClassicPresetOptions } from "@docusaurus/preset-classic";
+import fs from "fs";
+import { PluginOptions as LLmPluginOptions } from "@signalwire/docusaurus-plugin-llms-txt";

-const { themes } = require("prism-react-renderer");
-const lightCodeTheme = themes.github;
-const darkCodeTheme = themes.dracula;
-const npm2yarn = require("@docusaurus/remark-plugin-npm2yarn");
+// Import remark plugins - lazy load to prevent webpack from bundling Node.js code
+const getRemarkPlugins = () => ({
+  docsHeaderDecoration: require("./src/remark/docsHeaderDecoration"),
+  enterpriseDocsHeaderInformation: require("./src/remark/enterpriseDocsHeaderInformation"),
+  productInformation: require("./src/remark/productInformation"),
+  connectorList: require("./src/remark/connectorList"),
+  specDecoration: require("./src/remark/specDecoration"),
+  docMetaTags: require("./src/remark/docMetaTags"),
+  addButtonToTitle: require("./src/remark/addButtonToTitle"),
+  npm2yarn: require("@docusaurus/remark-plugin-npm2yarn"),
+});

-const docsHeaderDecoration = require("./src/remark/docsHeaderDecoration");
-const enterpriseDocsHeaderInformation = require("./src/remark/enterpriseDocsHeaderInformation");
-const productInformation = require("./src/remark/productInformation");
-const connectorList = require("./src/remark/connectorList");
-const specDecoration = require("./src/remark/specDecoration");
-const docMetaTags = require("./src/remark/docMetaTags");
-const addButtonToTitle = require("./src/remark/addButtonToTitle");
-const fs = require("fs");
+const plugins = getRemarkPlugins();

-const { SPEC_CACHE_PATH, API_SIDEBAR_PATH } = require("./src/scripts/embedded-api/constants");
+// Import constants
+const {
+  SPEC_CACHE_PATH,
+  API_SIDEBAR_PATH,
+} = require("./src/scripts/embedded-api/constants");

-/** @type {import('@docusaurus/types').Config} */
-const config = {
+const lightCodeTheme = prismThemes.github;
+const darkCodeTheme = prismThemes.dracula;
+
+const config: Config = {
  future: {
-    experimental_faster: true,
+    experimental_faster: {
+      swcJsLoader: true,
+      swcJsMinimizer: true,
+      swcHtmlMinimizer: true,
+      lightningCssMinimizer: true,
+      mdxCrossCompilerCache: true,
+      rspackBundler: true,
+      rspackPersistentCache: true,
+    },
  },
  markdown: {
    mermaid: true,
+    hooks: {
+      onBrokenMarkdownLinks: "throw",
+    },
  },
  themes: [
    "@docusaurus/theme-mermaid",
@@ -39,7 +59,7 @@ const config = {
  // anything else should match the repo name
  baseUrl: "/",
  onBrokenLinks: "throw",
-  onBrokenMarkdownLinks: "throw",
+
  favicon: "img/favicon.png",
  organizationName: "airbytehq", // Usually your GitHub org/user name.
  projectName: "airbyte", // Usually your repo name.
@@ -87,22 +107,21 @@ const config = {
      : []),
  ],
  i18n: {
-    defaultLocale: 'en',
-    locales: ['en'],
+    defaultLocale: "en",
+    locales: ["en"],
  },
  // The preset is the "main" docs instance, though in reality, most content does not live under this preset. See the plugins array below, which defines the behavior of each docs instance.
  presets: [
    [
      "classic",
-      /** @type {import('@docusaurus/preset-classic').Options} */
-      ({
+      {
        docs: false, // Disable default docs plugin since we're using a custom page for home
        blog: false,
        pages: {}, // Enable pages plugin for standalone pages
        theme: {
          customCss: require.resolve("./src/css/custom.css"),
        },
-      }),
+      } satisfies ClassicPresetOptions,
    ],
  ],
  plugins: [
@@ -114,7 +133,13 @@ const config = {
        path: "../docs/platform",
        routeBasePath: "/platform",
        sidebarPath: "./sidebar-platform.js",
-        editUrl: ({ version, docPath }) => {
+        editUrl: ({
+          version,
+          docPath,
+        }: {
+          version: string;
+          docPath: string;
+        }) => {
          if (version === "current") {
            // For the "next" (unreleased) version
            return `https://github.com/airbytehq/airbyte/edit/master/docs/platform/${docPath}`;
@@ -124,11 +149,11 @@ const config = {
          }
        },
        remarkPlugins: [
-          docsHeaderDecoration,
-          enterpriseDocsHeaderInformation,
-          productInformation,
-          docMetaTags,
-          addButtonToTitle,
+          plugins.docsHeaderDecoration,
+          plugins.enterpriseDocsHeaderInformation,
+          plugins.productInformation,
+          plugins.docMetaTags,
+          plugins.addButtonToTitle,
        ],
      },
    ],
@@ -142,12 +167,12 @@ const config = {
        sidebarPath: "./sidebar-ai-agents.js",
        editUrl: "https://github.com/airbytehq/airbyte/blob/master/docs",
        remarkPlugins: [
-          docsHeaderDecoration,
-          enterpriseDocsHeaderInformation,
-          productInformation,
-          docMetaTags,
-          addButtonToTitle,
-          [npm2yarn, { sync: true }],
+          plugins.docsHeaderDecoration,
+          plugins.enterpriseDocsHeaderInformation,
+          plugins.productInformation,
+          plugins.docMetaTags,
+          plugins.addButtonToTitle,
+          [plugins.npm2yarn, { sync: true }],
        ],
      },
    ],
@@ -161,11 +186,11 @@ const config = {
        sidebarPath: "./sidebar-release_notes.js",
        editUrl: "https://github.com/airbytehq/airbyte/blob/master/docs",
        remarkPlugins: [
-          docsHeaderDecoration,
-          enterpriseDocsHeaderInformation,
-          productInformation,
-          docMetaTags,
-          addButtonToTitle,
+          plugins.docsHeaderDecoration,
+          plugins.enterpriseDocsHeaderInformation,
+          plugins.productInformation,
+          plugins.docMetaTags,
+          plugins.addButtonToTitle,
        ],
      },
    ],
@@ -178,12 +203,15 @@ const config = {
        routeBasePath: "/integrations",
        sidebarPath: "./sidebar-connectors.js",
        editUrl: "https://github.com/airbytehq/airbyte/blob/master/docs",
-        beforeDefaultRemarkPlugins: [specDecoration, connectorList], // use before-default plugins so TOC rendering picks up inserted headings
+        beforeDefaultRemarkPlugins: [
+          plugins.specDecoration,
+          plugins.connectorList,
+        ], // use before-default plugins so TOC rendering picks up inserted headings
        remarkPlugins: [
-          docsHeaderDecoration,
-          enterpriseDocsHeaderInformation,
-          productInformation,
-          docMetaTags,
+          plugins.docsHeaderDecoration,
+          plugins.enterpriseDocsHeaderInformation,
+          plugins.productInformation,
+          plugins.docMetaTags,
        ],
      },
    ],
@@ -215,14 +243,14 @@ const config = {

            // Load the freshly generated sidebar (not the cached one from module load)
            const sidebarPath = API_SIDEBAR_PATH;
-            let freshSidebar = [];
+            let freshSidebar: any[] = [];

            if (fs.existsSync(sidebarPath)) {
              try {
                const sidebarModule = require("./api-docs/embedded-api/sidebar.ts");
                freshSidebar = sidebarModule.default || sidebarModule;
                console.log("Loaded fresh sidebar from generated files");
-              } catch (sidebarError) {
+              } catch (sidebarError: any) {
                console.warn(
                  "Could not load fresh sidebar, using empty array:",
                  sidebarError.message,
@@ -236,14 +264,14 @@ const config = {
              freshSidebar = [];
            }

-            const allowedTags = data.tags?.map((tag) => tag["name"]) || [];
+            const allowedTags = data.tags?.map((tag: any) => tag["name"]) || [];

            // Use freshly loaded sidebar items from the generated file
            const sidebarItems = Array.isArray(freshSidebar)
              ? freshSidebar
              : [];

-            const filteredItems = sidebarItems.filter((item) => {
+            const filteredItems = sidebarItems.filter((item: any) => {
              if (item.type !== "category") {
                return true;
              }
@@ -252,7 +280,7 @@ const config = {
            });

            return filteredItems;
-          } catch (error) {
+          } catch (error: any) {
            console.warn(
              "Error loading embedded API spec from cache:",
              error.message,
@@ -291,8 +319,9 @@ const config = {
        depth: 4,
        content: {
          includePages: true,
+          excludeRoutes: ["./api-docs/**"],
        },
-      },
+      } satisfies LLmPluginOptions,
    ],
    () => ({
      name: "Yaml loader",
@@ -326,36 +355,32 @@ const config = {
    require.resolve("./src/scripts/fontAwesomeIcons.js"),
  ],

-  themeConfig:
-    /** @type {import('@docusaurus/preset-classic').ThemeConfig} */
-    ({
+  themeConfig: {
    colorMode: {
      disableSwitch: false,
    },
    mermaid: {
      theme: {
-          light: 'base',  // "base" theme is fully customizable
-          dark: 'base'
+        light: "base",
+        dark: "base",
      },
      options: {
        themeVariables: {
-            primaryColor: '#5F5CFF',        // Airbyte blue
-            primaryTextColor: '#FFFFFF',    // white labels on colored shapes
-            primaryBorderColor: '#1A194D',  // slightly darker for contrast
-            secondaryColor: '#FF6A4D',      // accent orange
-            // secondaryTextColor: '#FF6A4D',      // accent orange
-            // secondaryBorderColor: '#FF6A4D',      // accent orange
-            tertiaryColor: '#E8EAF6',        // light neutral fill
-            tertiaryTextColor: '#000000',    // black labels on light shapes
-            tertiaryBorderColor: '#E8EAF6',    // light neutral border
-            background: '#FFFFFF',
-            clusterBkg: '#F5F5F5',
-            fontFamily: 'var(--ifm-font-family-base)',
+          primaryColor: "#5F5CFF",
+          primaryTextColor: "#FFFFFF",
+          primaryBorderColor: "#1A194D",
+          secondaryColor: "#FF6A4D",
+          tertiaryColor: "#E8EAF6",
+          tertiaryTextColor: "#000000",
+          tertiaryBorderColor: "#E8EAF6",
+          background: "#FFFFFF",
+          clusterBkg: "#F5F5F5",
+          fontFamily: "var(--ifm-font-family-base)",
        },
        flowchart: {
-            rankSpacing: 100,   // vertical space
-            subGraphTitleMargin: 10, // space within subgraph border for title
-            nodeSpacing: 100,   // horizontal space
+          rankSpacing: 100,
+          subGraphTitleMargin: 10,
+          nodeSpacing: 100,
        },
      },
    },
@@ -366,7 +391,7 @@ const config = {
    },
    algolia: {
      appId: "OYKDBC51MU",
-        apiKey: "15c487fd9f7722282efd8fcb76746fce", // Public API key: it is safe to commit it
+      apiKey: "15c487fd9f7722282efd8fcb76746fce",
      indexName: "airbyte",
    },
    announcementBar: {
@@ -418,7 +443,6 @@ const config = {
          href: "https://support.airbyte.com/",
          label: "Support",
        },
-
        {
          href: "https://status.airbyte.com",
          label: "Status",
@@ -431,7 +455,7 @@ const config = {
          position: "right",
          docsPluginId: "platform",
          label: "Version",
-            dropdownActiveClassDisabled: true, // do not style the dropdown as active when viewing platform docs
+          dropdownActiveClassDisabled: true,
        },
        {
          href: "https://github.com/airbytehq",
@@ -446,7 +470,7 @@ const config = {
      darkTheme: darkCodeTheme,
      additionalLanguages: ["bash", "diff", "json", "hcl"],
    },
-    }),
+  },
 };

-module.exports = config;
+export default config;
--- a/docusaurus/package.json
+++ b/docusaurus/package.json
@@ -3,11 +3,13 @@
  "version": "0.0.0",
  "private": true,
  "scripts": {
-    "prepare-sidebar": "node src/scripts/prepare-sidebar-data.js",
+    "prepare-registry-cache": "node src/scripts/prepare-registry-cache.js",
    "gen-embedded-api-docs": "pnpm exec docusaurus clean-api-docs all && pnpm exec docusaurus gen-api-docs all",
    "prepare-embedded-api": "pnpm run gen-embedded-api-docs && node src/scripts/embedded-api/prepare-embedded-api-spec.js && pnpm prettier --write src/data/embedded_api_spec.json",
-    "prebuild": "pnpm run prepare-sidebar && pnpm run prepare-embedded-api",
-    "prestart": "pnpm run prepare-sidebar && pnpm run prepare-embedded-api",
+    "cleanup-cache": "node src/scripts/cleanup-cache.js",
+    "prebuild": "pnpm run prepare-registry-cache && pnpm run prepare-embedded-api",
+    "postbuild": "pnpm run cleanup-cache",
+    "prestart": "pnpm run prepare-registry-cache",
    "docusaurus": "docusaurus",
    "start": "node src/scripts/fetchSchema.js && docusaurus start --port 3005",
    "build": "node src/scripts/fetchSchema.js && docusaurus build",
@@ -80,19 +82,20 @@
    "@babel/runtime-corejs3": "7.26.10",
    "@cmfcmf/docusaurus-search-local": "^1.1.0",
    "@docsearch/react": "3.1.0",
-    "@docusaurus/core": "^3.7.0",
-    "@docusaurus/cssnano-preset": "^3.7.0",
-    "@docusaurus/faster": "^3.8.1",
-    "@docusaurus/module-type-aliases": "^3.7.0",
-    "@docusaurus/plugin-debug": "^3.7.0",
-    "@docusaurus/plugin-sitemap": "^3.7.0",
-    "@docusaurus/preset-classic": "^3.7.0",
-    "@docusaurus/remark-plugin-npm2yarn": "^3.7.0",
-    "@docusaurus/theme-classic": "^3.7.0",
-    "@docusaurus/theme-common": "^3.7.0",
-    "@docusaurus/theme-mermaid": "^3.7.0",
-    "@docusaurus/theme-search-algolia": "^3.7.0",
-    "@docusaurus/types": "^3.7.0",
+    "@docusaurus/core": "^3.9.2",
+    "@docusaurus/cssnano-preset": "^3.9.2",
+    "@docusaurus/faster": "^3.9.2",
+    "@docusaurus/module-type-aliases": "^3.9.2",
+    "@docusaurus/plugin-debug": "^3.9.2",
+    "@docusaurus/plugin-rsdoctor": "^3.9.2",
+    "@docusaurus/plugin-sitemap": "^3.9.2",
+    "@docusaurus/preset-classic": "^3.9.2",
+    "@docusaurus/remark-plugin-npm2yarn": "^3.9.2",
+    "@docusaurus/theme-classic": "^3.9.2",
+    "@docusaurus/theme-common": "^3.9.2",
+    "@docusaurus/theme-mermaid": "^3.9.2",
+    "@docusaurus/theme-search-algolia": "^3.9.2",
+    "@docusaurus/types": "^3.9.2",
    "@fortawesome/fontawesome-svg-core": "^6.5.1",
    "@fortawesome/free-brands-svg-icons": "^6.7.1",
    "@fortawesome/free-regular-svg-icons": "^6.5.1",
@@ -104,7 +107,7 @@
    "@mdx-js/react": "^3.0.0",
    "@saucelabs/theme-github-codeblock": "^0.3.0",
    "@seriousme/openapi-schema-validator": "^2.5.0",
-    "@signalwire/docusaurus-plugin-llms-txt": "^1.0.1",
+    "@signalwire/docusaurus-plugin-llms-txt": "^1.2.2",
    "ajv": "^8.17.1",
    "ajv-formats": "^3.0.1",
    "async": "2.6.4",
@@ -169,6 +172,7 @@
    ]
  },
  "devDependencies": {
+    "@rsdoctor/cli": "^1.3.4",
    "prettier": "3.5.3"
  },
  "packageManager": "pnpm@9.4.0+sha1.9217c800d4ab947a7aee520242a7b70d64fc7638"
--- a/docusaurus/pnpm-lock.yaml
+++ b/docusaurus/pnpm-lock.yaml
--- a/docusaurus/sidebar-connectors.js
+++ b/docusaurus/sidebar-connectors.js
@@ -1,17 +1,11 @@
 const fs = require("fs");
 const path = require("path");
-
-const REGISTRY_CACHE_PATH = path.join(
-  __dirname,
-  "src",
-  "data",
-  "connector_registry_slim.json",
-);
-
-const connectorsDocsRoot = "../docs/integrations";
-const sourcesDocs = `${connectorsDocsRoot}/sources`;
-const destinationDocs = `${connectorsDocsRoot}/destinations`;
-const enterpriseConnectorDocs = `${connectorsDocsRoot}/enterprise-connectors`;
+const {
+  REGISTRY_CACHE_PATH,
+  SOURCES_DOCS,
+  DESTINATIONS_DOCS,
+  ENTERPRISE_CONNECTORS_DOCS,
+} = require("./src/scripts/constants");

 function getFilenamesInDir(prefix, dir, excludes) {
  return fs
@@ -52,12 +46,14 @@ function getFilenamesInDir(prefix, dir, excludes) {
        return {
          type: "category",
          label: contentTitle,
+          key: `${prefix}${filename}-category`,
          link: { type: "doc", id: path.join(prefix, filename) },
          items: [
            {
              type: "doc",
              id: path.join(prefix, `${filename}-migrations`),
              label: "Migration Guide",
+              key: `${prefix}${filename}-migrations`,
            },
          ],
        };
@@ -67,6 +63,7 @@ function getFilenamesInDir(prefix, dir, excludes) {
        type: "doc",
        id: prefix + filename,
        label: contentTitle,
+        key: `${prefix}${filename}`,
      };
    });
 }
@@ -108,7 +105,7 @@ function addSupportLevelToConnectors(connectors, registry) {
  });
 }

-function groupConnectorsBySupportLevel(connectors) {
+function groupConnectorsBySupportLevel(connectors, keyPrefix = "") {
  const grouped = connectors.reduce(
    (acc, item) => {
      const supportLevel = item.customProps?.supportLevel || "community";
@@ -131,6 +128,7 @@ function groupConnectorsBySupportLevel(connectors) {
      label: "Airbyte",
      collapsible: true,
      collapsed: true,
+      key: `${keyPrefix}airbyte`,
      items: grouped.certified.sort((a, b) => a.label.localeCompare(b.label)),
    });
  }
@@ -141,6 +139,7 @@ function groupConnectorsBySupportLevel(connectors) {
      label: "Marketplace",
      collapsible: true,
      collapsed: true,
+      key: `${keyPrefix}marketplace`,
      items: grouped.community.sort((a, b) => a.label.localeCompare(b.label)),
    });
  }
@@ -151,6 +150,7 @@ function groupConnectorsBySupportLevel(connectors) {
      label: "Enterprise",
      collapsible: true,
      collapsed: true,
+      key: `${keyPrefix}enterprise`,
      items: grouped.enterprise.sort((a, b) => a.label.localeCompare(b.label)),
    });
  }
@@ -161,6 +161,7 @@ function groupConnectorsBySupportLevel(connectors) {
 const sourcePostgres = {
  type: "category",
  label: "Postgres",
+  key: "sources-postgres-category",
  link: {
    type: "doc",
    id: "sources/postgres",
@@ -170,11 +171,13 @@ const sourcePostgres = {
      type: "doc",
      label: "Cloud SQL for Postgres",
      id: "sources/postgres/cloud-sql-postgres",
+      key: "sources-postgres-cloud-sql",
    },
    {
      type: "doc",
      label: "Troubleshooting",
      id: "sources/postgres/postgres-troubleshooting",
+      key: "sources-postgres-troubleshooting",
    },
  ],
 };
@@ -182,6 +185,7 @@ const sourcePostgres = {
 const sourceMongoDB = {
  type: "category",
  label: "Mongo DB",
+  key: "sources-mongodb-v2-category",
  link: {
    type: "doc",
    id: "sources/mongodb-v2",
@@ -191,11 +195,13 @@ const sourceMongoDB = {
      type: "doc",
      label: "Migration Guide",
      id: "sources/mongodb-v2-migrations",
+      key: "sources-mongodb-v2-migrations",
    },
    {
      type: "doc",
      label: "Troubleshooting",
      id: "sources/mongodb-v2/mongodb-v2-troubleshooting",
+      key: "sources-mongodb-v2-troubleshooting",
    },
  ],
 };
@@ -203,6 +209,7 @@ const sourceMongoDB = {
 const sourceMysql = {
  type: "category",
  label: "MySQL",
+  key: "sources-mysql-category",
  link: {
    type: "doc",
    id: "sources/mysql",
@@ -212,6 +219,7 @@ const sourceMysql = {
      type: "doc",
      label: "Troubleshooting",
      id: "sources/mysql/mysql-troubleshooting",
+      key: "sources-mysql-troubleshooting",
    },
  ],
 };
@@ -219,6 +227,7 @@ const sourceMysql = {
 const sourceMssql = {
  type: "category",
  label: "MS SQL Server (MSSQL)",
+  key: "sources-mssql-category",
  link: {
    type: "doc",
    id: "sources/mssql",
@@ -228,11 +237,12 @@ const sourceMssql = {
      type: "doc",
      label: "Troubleshooting",
      id: "sources/mssql/mssql-troubleshooting",
+      key: "sources-mssql-troubleshooting",
    },
  ],
 };
 function getSourceConnectors(registry) {
-  const sources = getFilenamesInDir("sources/", sourcesDocs, [
+  const sources = getFilenamesInDir("sources/", SOURCES_DOCS, [
    "readme",
    "postgres",
    "mongodb-v2",
@@ -248,7 +258,7 @@ function getSourceConnectors(registry) {
  ];
  const enterpriseSources = getFilenamesInDir(
    "enterprise-connectors/",
-    enterpriseConnectorDocs,
+    ENTERPRISE_CONNECTORS_DOCS,
    ["readme"],
  );
  const enterpriseSourcesWithSupportLevel = enterpriseSources
@@ -273,6 +283,7 @@ function getSourceConnectors(registry) {
 const destinationS3 = {
  type: "category",
  label: "S3",
+  key: "destinations-s3-category",
  link: {
    type: "doc",
    id: "destinations/s3",
@@ -282,11 +293,13 @@ const destinationS3 = {
      type: "doc",
      label: "Migration Guide",
      id: "destinations/s3-migrations",
+      key: "destinations-s3-migrations",
    },
    {
      type: "doc",
      label: "Troubleshooting",
      id: "destinations/s3/s3-troubleshooting",
+      key: "destinations-s3-troubleshooting",
    },
  ],
 };
@@ -294,6 +307,7 @@ const destinationS3 = {
 const destinationPostgres = {
  type: "category",
  label: "Postgres",
+  key: "destinations-postgres-category",
  link: {
    type: "doc",
    id: "destinations/postgres",
@@ -303,6 +317,7 @@ const destinationPostgres = {
      type: "doc",
      label: "Troubleshooting",
      id: "destinations/postgres/postgres-troubleshooting",
+      key: "destinations-postgres-troubleshooting",
    },
  ],
 };
@@ -311,6 +326,7 @@ const destinationPostgres = {
 const destinationMsSql = {
  type: "category",
  label: "MS SQL Server (MSSQL)",
+  key: "destinations-mssql-category",
  link: {
    type: "doc",
    id: "destinations/mssql",
@@ -323,13 +339,14 @@ const destinationMsSql = {
      type: "doc",
      label: "Migration Guide",
      id: "destinations/mssql-migrations",
+      key: "destinations-mssql-migrations",
    },
  ],
 };

 function getDestinationConnectors(registry) {
  const specialDestinationConnectors = [destinationS3, destinationPostgres];
-  const destinations = getFilenamesInDir("destinations/", destinationDocs, [
+  const destinations = getFilenamesInDir("destinations/", DESTINATIONS_DOCS, [
    "s3",
    "postgres",
    "mssql",
@@ -342,7 +359,7 @@ function getDestinationConnectors(registry) {

  const enterpriseDestinations = getFilenamesInDir(
    "enterprise-connectors/",
-    enterpriseConnectorDocs,
+    ENTERPRISE_CONNECTORS_DOCS,
    ["readme"],
  );
  const enterpriseDestinationsWithSupportLevel = enterpriseDestinations
@@ -376,9 +393,11 @@ function buildConnectorSidebar() {

  const sourcesBySupportLevel = groupConnectorsBySupportLevel(
    sourcesWithSupportLevel,
+    "sources-",
  );
  const destinationsBySupportLevel = groupConnectorsBySupportLevel(
    destinationConnectors,
+    "destinations-",
  );

  return {
--- a/docusaurus/src/components/Arcade.jsx
+++ b/docusaurus/src/components/Arcade.jsx
@@ -11,7 +11,6 @@ export const Arcade = (props) => {
      <iframe
        src={`https://demo.arcade.software/${props.id}?embed`}
        title={props.title}
-        frameborder="0"
        loading="lazy"
        allowFullScreen
        style={{
--- a/docusaurus/src/components/ConnectorRegistry.jsx
+++ b/docusaurus/src/components/ConnectorRegistry.jsx
@@ -1,8 +1,8 @@
 import { usePluginData } from "@docusaurus/useGlobalData";
 import TabItem from "@theme/TabItem";
 import Tabs from "@theme/Tabs";
-import React, { useEffect, useState } from "react";
-import { REGISTRY_URL } from "../connector_registry";
+import { useEffect, useState } from "react";
+import { REGISTRY_URL } from "../constants";
 import styles from "./ConnectorRegistry.module.css";

 const iconStyle = { maxWidth: 25, maxHeight: 25 };
--- a/docusaurus/src/components/EntityRelationshipDiagram.jsx
+++ b/docusaurus/src/components/EntityRelationshipDiagram.jsx
@@ -1,7 +1,7 @@
-import React, { useEffect, useState } from "react";
+import { useEffect, useState } from "react";
 import erdBG from "../../static/img/erd-bg-cta.jpg";
 import styles from "./EntityRelationshipDiagram.module.css";
-const { getRegistryEntry } = require("../remark/utils");
+const { getRegistryEntry } = require("../helpers/clientRegistryUtils");
 const { getFromPaths } = require("../helpers/objects");

 export const EntityRelationshipDiagram = ({}) => {
--- a/docusaurus/src/components/HeaderDecoration.jsx
+++ b/docusaurus/src/components/HeaderDecoration.jsx
@@ -1,7 +1,5 @@
 import dayjs from "dayjs";
 import relativeTime from "dayjs/plugin/relativeTime";
-import React from "react";
-import { getSupportLevelDisplay } from "../connector_registry";
 import { Callout } from "./Callout";
 import { Chip } from "./Chip";
 import { CopyPageButton } from "./CopyPageButton/CopyPageButton";
@@ -195,6 +193,21 @@ const boolStringToBool = (boolString) => {
  return null;
 };

+ const getSupportLevelDisplay=(rawSupportLevel) => {
+  switch (rawSupportLevel) {
+    case "certified":
+      return "Airbyte";
+    case "community":
+      return "Marketplace";
+    case "enterprise":
+      return "Enterprise";
+    case "archived":
+      return "Archived";
+    default:
+      return null;
+  }
+}
+
 // COMPONENTS

 const MetricIcon = ({ iconComponent, level }) => {
--- a/docusaurus/src/components/MarkpromptChat/ChatTrigger.jsx
+++ b/docusaurus/src/components/MarkpromptChat/ChatTrigger.jsx
@@ -22,8 +22,8 @@ const Icon = () => {
            y2="10"
            gradientUnits="userSpaceOnUse"
          >
-            <stop stop-color="hsl(241, 100%, 68%)"></stop>
-            <stop offset="1" stop-color="hsl(10, 100%, 65%)"></stop>
+            <stop stopColor="hsl(241, 100%, 68%)"></stop>
+            <stop offset="1" stopColor="hsl(10, 100%, 65%)"></stop>
          </linearGradient>
        </defs>
      </svg>
--- a/docusaurus/src/components/RequestERD/RequestERD.jsx
+++ b/docusaurus/src/components/RequestERD/RequestERD.jsx
@@ -1,10 +1,10 @@
 import { useLocation } from "@docusaurus/router";
-import React, { useEffect, useState } from "react";
+import { useEffect, useState } from "react";
 import { EmailModal } from "./EmailModal";
 import styles from "./RequestERD.module.css";
 import { faPlus } from "@fortawesome/free-solid-svg-icons";
 import { FontAwesomeIcon } from "@fortawesome/react-fontawesome";
-const { getRegistryEntry } = require("../../remark/utils");
+const { getRegistryEntry } = require("../../helpers/clientRegistryUtils");
 const { getFromPaths } = require("../../helpers/objects");

 export const RequestERD = () => {
--- a/docusaurus/src/components/SpecSchema.jsx
+++ b/docusaurus/src/components/SpecSchema.jsx
@@ -1,4 +1,3 @@
-import React from "react";
 import styles from "./SpecSchema.module.css";
 import sanitizeHtml from "sanitize-html";
 import { Disclosure } from "@headlessui/react";
@@ -20,12 +19,12 @@ function JSONSchemaViewer(props) {
  return (
    <div>
      <Heading as="h3">Config fields reference</Heading>
-      <div class={styles.grid}>
-        <div class={className(styles.headerItem, styles.tableHeader)}>
+      <div className={styles.grid}>
+        <div className={className(styles.headerItem, styles.tableHeader)}>
          Field
        </div>
-        <div class={className(styles.headerItem, styles.tableHeader)}>Type</div>
-        <div class={className(styles.headerItem, styles.tableHeader)}>
+        <div className={className(styles.headerItem, styles.tableHeader)}>Type</div>
+        <div className={className(styles.headerItem, styles.tableHeader)}>
          Property name
        </div>
        <JSONSchemaObject schema={props.schema} />
--- a/docusaurus/src/constants.ts
+++ b/docusaurus/src/constants.ts
@@ -0,0 +1,2 @@
+export const REGISTRY_URL =
+  "https://connectors.airbyte.com/files/generated_reports/connector_registry_report.json";
--- a/docusaurus/src/data/embedded_api_spec.json
+++ b/docusaurus/src/data/embedded_api_spec.json
@@ -366,6 +366,94 @@
        }
      }
    },
+    "/api/v1/agents/chat/schema/{source_id}/state": {
+      "get": {
+        "tags": ["Chat - Schema"],
+        "summary": "Get Schema Chat State",
+        "operationId": "get_agents_chat_schema_source_id_state",
+        "security": [
+          {
+            "HTTPBearer": []
+          }
+        ],
+        "parameters": [
+          {
+            "name": "source_id",
+            "in": "path",
+            "required": true,
+            "schema": {
+              "type": "string",
+              "format": "uuid",
+              "title": "Source Id"
+            }
+          },
+          {
+            "name": "x-organization-id",
+            "in": "header",
+            "required": false,
+            "schema": {
+              "type": "string",
+              "format": "uuid",
+              "description": "Select the working organization. Necessary since users can belong to multiple organizations.",
+              "title": "X-Organization-Id"
+            },
+            "description": "Select the working organization. Necessary since users can belong to multiple organizations."
+          }
+        ],
+        "responses": {
+          "200": {
+            "description": "Successful Response",
+            "content": {
+              "application/json": {
+                "schema": {
+                  "$ref": "#/components/schemas/SchemaChatAgentState"
+                }
+              }
+            }
+          },
+          "400": {
+            "description": "Bad request",
+            "content": {
+              "application/json": {
+                "schema": {
+                  "$ref": "#/components/schemas/ApiErrorResponse"
+                }
+              }
+            }
+          },
+          "403": {
+            "description": "Forbidden",
+            "content": {
+              "application/json": {
+                "schema": {
+                  "$ref": "#/components/schemas/ApiErrorResponse"
+                }
+              }
+            }
+          },
+          "422": {
+            "description": "Unprocessable entity",
+            "content": {
+              "application/json": {
+                "schema": {
+                  "$ref": "#/components/schemas/ApiErrorResponse"
+                }
+              }
+            }
+          },
+          "500": {
+            "description": "Internal server error",
+            "content": {
+              "application/json": {
+                "schema": {
+                  "$ref": "#/components/schemas/ApiErrorResponse"
+                }
+              }
+            }
+          }
+        }
+      }
+    },
    "/api/v1/agents/discovery": {
      "post": {
        "tags": ["Agents - Discovery"],
@@ -378,6 +466,18 @@
          }
        ],
        "parameters": [
+          {
+            "name": "annotate",
+            "in": "query",
+            "required": false,
+            "schema": {
+              "type": "boolean",
+              "description": "Whether to annotate the discovered catalogs using the catalog annotator agent",
+              "default": false,
+              "title": "Annotate"
+            },
+            "description": "Whether to annotate the discovered catalogs using the catalog annotator agent"
+          },
          {
            "name": "x-organization-id",
            "in": "header",
@@ -467,6 +567,18 @@
              "title": "Source Definition Id"
            }
          },
+          {
+            "name": "annotate",
+            "in": "query",
+            "required": false,
+            "schema": {
+              "type": "boolean",
+              "description": "Whether to annotate the discovered catalog using the catalog annotator agent",
+              "default": false,
+              "title": "Annotate"
+            },
+            "description": "Whether to annotate the discovered catalog using the catalog annotator agent"
+          },
          {
            "name": "x-organization-id",
            "in": "header",
@@ -3730,7 +3842,7 @@
            "content": {
              "application/json": {
                "schema": {
-                  "$ref": "#/components/schemas/SourceDiscoverResponse"
+                  "$ref": "#/components/schemas/SourceCatalogPatchResponse"
                }
              }
            }
@@ -6859,95 +6971,6 @@
        }
      }
    },
-    "/api/v1/internal/account/organizations/{organization_id}/signup": {
-      "post": {
-        "tags": ["Organizations"],
-        "summary": "Organization Signup",
-        "description": "Sign up an organization with synchronous provisioning.\nSets up Orb billing, Stigg subscription, and payment status in a single request.\nReturns immediately with subscription details.",
-        "operationId": "create_internal_account_organizations_organization_id_signup",
-        "security": [
-          {
-            "HTTPBearer": []
-          }
-        ],
-        "parameters": [
-          {
-            "name": "organization_id",
-            "in": "path",
-            "required": true,
-            "schema": {
-              "type": "string",
-              "format": "uuid",
-              "title": "Organization Id"
-            }
-          },
-          {
-            "name": "x-organization-id",
-            "in": "header",
-            "required": false,
-            "schema": {
-              "type": "string",
-              "format": "uuid",
-              "description": "Select the working organization. Necessary since users can belong to multiple organizations.",
-              "title": "X-Organization-Id"
-            },
-            "description": "Select the working organization. Necessary since users can belong to multiple organizations."
-          }
-        ],
-        "responses": {
-          "200": {
-            "description": "Successful Response",
-            "content": {
-              "application/json": {
-                "schema": {
-                  "$ref": "#/components/schemas/OrganizationSignupResponse"
-                }
-              }
-            }
-          },
-          "400": {
-            "description": "Bad request",
-            "content": {
-              "application/json": {
-                "schema": {
-                  "$ref": "#/components/schemas/ApiErrorResponse"
-                }
-              }
-            }
-          },
-          "403": {
-            "description": "Forbidden",
-            "content": {
-              "application/json": {
-                "schema": {
-                  "$ref": "#/components/schemas/ApiErrorResponse"
-                }
-              }
-            }
-          },
-          "422": {
-            "description": "Unprocessable entity",
-            "content": {
-              "application/json": {
-                "schema": {
-                  "$ref": "#/components/schemas/ApiErrorResponse"
-                }
-              }
-            }
-          },
-          "500": {
-            "description": "Internal server error",
-            "content": {
-              "application/json": {
-                "schema": {
-                  "$ref": "#/components/schemas/ApiErrorResponse"
-                }
-              }
-            }
-          }
-        }
-      }
-    },
    "/api/v1/internal/account/organizations/{organization_id}/subscription": {
      "get": {
        "tags": ["Organizations"],
@@ -9752,6 +9775,7 @@
          "get_source_definition",
          "list_source_templates",
          "get_source_template",
+          "get_source_definition_catalog",
          "run_source_template_creator",
          "create_source_template",
          "query_source_catalog",
@@ -11248,6 +11272,12 @@
            "format": "uri",
            "title": "Return Url",
            "description": "The URL to redirect to after the customer portal session"
+          },
+          "flow": {
+            "type": "string",
+            "enum": ["payment_method", "portal"],
+            "title": "Flow",
+            "default": "portal"
          }
        },
        "type": "object",
@@ -11268,6 +11298,11 @@
        "title": "CustomerPortalResponse",
        "description": "Response schema for customer portal link."
      },
+      "DataCategory": {
+        "type": "string",
+        "enum": ["person", "transaction", "event", "object", "file"],
+        "title": "DataCategory"
+      },
      "DestinationDefinitionListItem": {
        "properties": {
          "id": {
@@ -12372,56 +12407,6 @@
        "required": ["organizationId", "organizationName", "sso"],
        "title": "OrganizationInfoResponse"
      },
-      "OrganizationSignupResponse": {
-        "properties": {
-          "organization_id": {
-            "type": "string",
-            "format": "uuid",
-            "title": "Organization Id"
-          },
-          "provisioning_state": {
-            "type": "string",
-            "title": "Provisioning State"
-          },
-          "orb_subscription_id": {
-            "anyOf": [
-              {
-                "type": "string"
-              },
-              {
-                "type": "null"
-              }
-            ],
-            "title": "Orb Subscription Id"
-          },
-          "stigg_subscription_id": {
-            "anyOf": [
-              {
-                "type": "string"
-              },
-              {
-                "type": "null"
-              }
-            ],
-            "title": "Stigg Subscription Id"
-          },
-          "error_message": {
-            "anyOf": [
-              {
-                "type": "string"
-              },
-              {
-                "type": "null"
-              }
-            ],
-            "title": "Error Message"
-          }
-        },
-        "type": "object",
-        "required": ["organization_id", "provisioning_state"],
-        "title": "OrganizationSignupResponse",
-        "description": "Response schema for organization signup endpoint."
-      },
      "OrganizationSubscriptionCreditBlock": {
        "properties": {
          "amount": {
@@ -12755,6 +12740,61 @@
        "type": "object",
        "title": "RevokeSourceOauthResponse"
      },
+      "SchemaChatAgentState": {
+        "properties": {
+          "state": {
+            "$ref": "#/components/schemas/SchemaChatAgentStateType"
+          },
+          "source_id": {
+            "type": "string",
+            "format": "uuid",
+            "title": "Source Id"
+          },
+          "source_definition_name": {
+            "type": "string",
+            "title": "Source Definition Name"
+          },
+          "schema_type": {
+            "anyOf": [
+              {
+                "$ref": "#/components/schemas/SonarCatalogSchemaType"
+              },
+              {
+                "type": "null"
+              }
+            ]
+          },
+          "catalog": {
+            "anyOf": [
+              {
+                "$ref": "#/components/schemas/SonarCatalogOutput"
+              },
+              {
+                "type": "null"
+              }
+            ]
+          },
+          "thread_id": {
+            "anyOf": [
+              {
+                "type": "string"
+              },
+              {
+                "type": "null"
+              }
+            ],
+            "title": "Thread Id"
+          }
+        },
+        "type": "object",
+        "required": ["state", "source_id", "source_definition_name"],
+        "title": "SchemaChatAgentState"
+      },
+      "SchemaChatAgentStateType": {
+        "type": "string",
+        "enum": ["unsupported_source", "first_session", "returning"],
+        "title": "SchemaChatAgentStateType"
+      },
      "ScopedTokenCreateRequest": {
        "properties": {
          "workspace_name": {
@@ -12895,6 +12935,16 @@
            ],
            "title": "Description"
          },
+          "data_category": {
+            "anyOf": [
+              {
+                "$ref": "#/components/schemas/DataCategory"
+              },
+              {
+                "type": "null"
+              }
+            ]
+          },
          "suggested": {
            "type": "boolean",
            "title": "Suggested",
@@ -13017,6 +13067,16 @@
        "type": "object",
        "title": "SonarStream"
      },
+      "SourceCatalogPatchResponse": {
+        "properties": {
+          "catalog": {
+            "$ref": "#/components/schemas/SonarCatalogOutput"
+          }
+        },
+        "type": "object",
+        "required": ["catalog"],
+        "title": "SourceCatalogPatchResponse"
+      },
      "SourceCatalogQueryRequest": {
        "properties": {
          "jmes_query": {
@@ -13302,8 +13362,15 @@
            ]
          },
          "catalog_type": {
+            "anyOf": [
+              {
                "$ref": "#/components/schemas/SonarCatalogSchemaType"
              },
+              {
+                "type": "null"
+              }
+            ]
+          },
          "query_result": {
            "anyOf": [
              {},
@@ -13315,7 +13382,7 @@
          }
        },
        "type": "object",
-        "required": ["id", "name", "source_definition_id", "catalog_type"],
+        "required": ["id", "name", "source_definition_id"],
        "title": "SourceDefinitionCatalogGetResponse"
      },
      "SourceDefinitionListResponse": {
--- a/docusaurus/src/helpers/clientRegistryUtils.js
+++ b/docusaurus/src/helpers/clientRegistryUtils.js
@@ -0,0 +1,90 @@
+// Client-side registry utilities
+// This works with URLs instead of vfiles and fetches data from the connector registry
+
+const connectorPageAlternativeEndings = ["-migrations", "-troubleshooting"];
+const connectorPageAlternativeEndingsRegExp = new RegExp(
+  connectorPageAlternativeEndings.join("|"),
+  "gi",
+);
+
+const REGISTRY_URL =
+  "https://connectors.airbyte.com/files/generated_reports/connector_registry_report.json";
+
+const fetchRegistry = async () => {
+  try {
+    const response = await fetch(REGISTRY_URL);
+    if (!response.ok) {
+      throw new Error(`Failed to fetch registry: ${response.statusText}`);
+    }
+    return await response.json();
+  } catch (error) {
+    console.error("Failed to fetch connector registry:", error);
+    return [];
+  }
+};
+
+const getRegistryEntry = async ({ path }) => {
+  if (
+    !path.includes("integrations/sources") &&
+    !path.includes("integrations/destinations")
+  ) {
+    return;
+  }
+
+  // Extract connector info from pathname
+  const pathParts = path.split("/");
+  const integrationIndex = pathParts.indexOf("integrations");
+  if (integrationIndex === -1) return;
+
+  const connectorType = pathParts[integrationIndex + 1]; // "sources" or "destinations"
+  const connectorNameWithExt = pathParts[integrationIndex + 2]; // e.g., "mysql.md" or "mysql"
+
+  if (!connectorNameWithExt) return;
+
+  let connectorName = connectorNameWithExt.split(".")[0]; // Remove .md extension
+
+  // Remove alternative endings
+  connectorName = connectorName.replace(connectorPageAlternativeEndingsRegExp, "");
+
+  const dockerRepository = `airbyte/${connectorType.replace(/s$/, "")}-${connectorName}`;
+
+  const registry = await fetchRegistry();
+
+  let registryEntry = registry.find(
+    (r) => r.dockerRepository_oss === dockerRepository,
+  );
+
+  if (!registryEntry) {
+    registryEntry = buildArchivedRegistryEntry(
+      connectorName,
+      dockerRepository,
+      connectorType,
+    );
+  }
+
+  return registryEntry;
+};
+
+const buildArchivedRegistryEntry = (
+  connectorName,
+  dockerRepository,
+  connectorType,
+) => {
+  const dockerName = dockerRepository.split("/")[1];
+  const registryEntry = {
+    connectorName,
+    name_oss: dockerName,
+    dockerRepository_oss: dockerRepository,
+    is_oss: false,
+    is_cloud: false,
+    iconUrl_oss: `https://connectors.airbyte.com/files/metadata/airbyte/${dockerName}/latest/icon.svg`,
+    supportLevel_oss: "archived",
+    documentationUrl_oss: `https://docs.airbyte.com/integrations/${connectorType}s/${connectorName}`,
+  };
+
+  return registryEntry;
+};
+
+module.exports = {
+  getRegistryEntry,
+};
--- a/docusaurus/src/plugins/enterpriseConnectors.js
+++ b/docusaurus/src/plugins/enterpriseConnectors.js
@@ -1,7 +1,5 @@
 const fs = require("fs");
-
-const connectorsDocsRoot = "../docs/integrations";
-const enterpriseConnectorDocs = `${connectorsDocsRoot}/enterprise-connectors`;
+const { ENTERPRISE_CONNECTORS_DOCS } = require("../scripts/constants");

 export function getFilenamesInDir(dir, excludes) {
  return fs
@@ -24,7 +22,7 @@ function enterpriseConnectorsPlugin(context, options) {
    name: "enterprise-connectors-plugin",
    async loadContent() {
      try {
-        const enterpriseSources = getFilenamesInDir(enterpriseConnectorDocs, [
+        const enterpriseSources = getFilenamesInDir(ENTERPRISE_CONNECTORS_DOCS, [
          "readme",
        ]);
        return enterpriseSources;
--- a/docusaurus/src/remark/connectorList.js
+++ b/docusaurus/src/remark/connectorList.js
@@ -1,9 +1,10 @@
 const visit = require("unist-util-visit").visit;
-const { catalog, isPypiConnector } = require("../connector_registry");
+const {  isPypiConnector } = require("../scripts/connector_registry");
+const  { fetchRegistry } = require("../scripts/fetch-registry");

 const plugin = () => {
  const transformer = async (ast, vfile) => {
-    const registry = await catalog;
+    const registry = await fetchRegistry();

    visit(ast, "mdxJsxFlowElement", (node) => {
      if (node.name !== "PyAirbyteConnectors") return;
--- a/docusaurus/src/remark/docsHeaderDecoration.js
+++ b/docusaurus/src/remark/docsHeaderDecoration.js
@@ -3,7 +3,7 @@ const { isDocsPage, getRegistryEntry } = require("./utils");
 const {
  getLatestPythonCDKVersion,
  parseCDKVersion,
-} = require("../connector_registry");
+} = require("../scripts/connector_registry");
 const visit = require("unist-util-visit").visit;

 /**
--- a/docusaurus/src/remark/enterpriseDocsHeaderInformation.js
+++ b/docusaurus/src/remark/enterpriseDocsHeaderInformation.js
@@ -1,14 +1,14 @@
 const { isEnterpriseConnectorDocsPage } = require("./utils");
 const { toAttributes } = require("../helpers/objects");
 const visit = require("unist-util-visit").visit;
-const { catalog } = require("../connector_registry");
+const { fetchRegistry } = require("../scripts/fetch-registry");

 const getEnterpriseConnectorVersion = async (dockerRepository) => {
  if (!dockerRepository) {
    return "No version information available";
  }
  try {
-    const registry = await catalog;
+    const registry = await fetchRegistry();

    const registryEntry = registry.find(
      (r) =>
--- a/docusaurus/src/remark/specDecoration.js
+++ b/docusaurus/src/remark/specDecoration.js
@@ -1,5 +1,5 @@
 const visit = require("unist-util-visit").visit;
-const { catalog } = require("../connector_registry");
+const { fetchRegistry } = require("../scripts/fetch-registry");
 const { isDocsPage, getRegistryEntry } = require("./utils");

 const plugin = () => {
@@ -11,7 +11,7 @@ const plugin = () => {
 };

 async function injectSpecSchema(ast) {
-  const registry = await catalog;
+  const registry = await fetchRegistry();
  visit(ast, "mdxJsxFlowElement", (node) => {
    if (node.name !== "SpecSchema" && node.name !== "PyAirbyteExample") return;

--- a/docusaurus/src/remark/utils.js
+++ b/docusaurus/src/remark/utils.js
@@ -1,4 +1,4 @@
-const { catalog } = require("../connector_registry");
+const { fetchRegistry } = require("../scripts/fetch-registry");

 // the migration guide and troubleshooting guide are not connectors, but also not in a sub-folder, e.g. /integrations/sources/mssql-migrations
 const connectorPageAlternativeEndings = ["-migrations", "-troubleshooting"];
@@ -65,7 +65,7 @@ const getRegistryEntry = async (vfile) => {
    "",
  )}-${connectorName}`;

-  const registry = await catalog;
+  const registry = await fetchRegistry();

  let registryEntry = registry.find(
    (r) => r.dockerRepository_oss === dockerRepository,
--- a/docusaurus/src/scripts/cleanup-cache.js
+++ b/docusaurus/src/scripts/cleanup-cache.js
@@ -0,0 +1,20 @@
+/**
+ * This script cleans up the temporary registry cache file after the build is complete.
+ * The cache file is generated during the prebuild phase and should not be
+ * committed to git since it's regenerated on each build.
+ */
+const fs = require("fs");
+const { REGISTRY_CACHE_PATH } = require("./constants");
+
+function cleanupCache() {
+  if (fs.existsSync(REGISTRY_CACHE_PATH)) {
+    try {
+      fs.unlinkSync(REGISTRY_CACHE_PATH);
+      console.log(`Cleaned up cache file: ${REGISTRY_CACHE_PATH}`);
+    } catch (error) {
+      console.warn(`Failed to clean up ${REGISTRY_CACHE_PATH}:`, error.message);
+    }
+  }
+}
+
+cleanupCache();
--- a/docusaurus/src/scripts/connector_registry.js
+++ b/docusaurus/src/scripts/connector_registry.js
@@ -1,8 +1,5 @@
 const memoize = require("lodash/memoize");

-const REGISTRY_URL =
-  "https://connectors.airbyte.com/files/generated_reports/connector_registry_report.json";
-
 const fetchLatestVersionOfPyPackage = memoize(async (packageName) => {
  const json = await fetch(`https://pypi.org/pypi/${packageName}/json`).then(
    (resp) => resp.json(),
@@ -10,14 +7,6 @@ const fetchLatestVersionOfPyPackage = memoize(async (packageName) => {
  return json.info.version;
 });

-const fetchCatalog = async () => {
-  console.log("Fetching connector registry...");
-  const json = await fetch(REGISTRY_URL).then((resp) => resp.json());
-  console.log(`fetched ${json.length} connectors from registry`);
-
-  return json;
-};
-
 const getLatestPythonCDKVersion = async () =>
  fetchLatestVersionOfPyPackage("airbyte-cdk");

@@ -59,8 +48,6 @@ function getSupportLevelDisplay(rawSupportLevel) {
 }

 module.exports = {
-  REGISTRY_URL,
-  catalog: fetchCatalog(),
  isPypiConnector: (connector) => {
    return Boolean(connector.remoteRegistries_oss?.pypi?.enabled);
  },
--- a/docusaurus/src/scripts/constants.js
+++ b/docusaurus/src/scripts/constants.js
@@ -0,0 +1,22 @@
+const path = require("path");
+
+const DATA_DIR = path.join(__dirname, "..", "data");
+const REGISTRY_CACHE_PATH = path.join(DATA_DIR, "connector_registry_slim.json");
+const REGISTRY_URL =
+  "https://connectors.airbyte.com/files/generated_reports/connector_registry_report.json";
+
+// Connector documentation paths
+const CONNECTORS_DOCS_ROOT = "../docs/integrations";
+const SOURCES_DOCS = `${CONNECTORS_DOCS_ROOT}/sources`;
+const DESTINATIONS_DOCS = `${CONNECTORS_DOCS_ROOT}/destinations`;
+const ENTERPRISE_CONNECTORS_DOCS = `${CONNECTORS_DOCS_ROOT}/enterprise-connectors`;
+
+module.exports = {
+  DATA_DIR,
+  REGISTRY_CACHE_PATH,
+  REGISTRY_URL,
+  CONNECTORS_DOCS_ROOT,
+  SOURCES_DOCS,
+  DESTINATIONS_DOCS,
+  ENTERPRISE_CONNECTORS_DOCS,
+};
--- a/docusaurus/src/scripts/embedded-api/openapi-validator.js
+++ b/docusaurus/src/scripts/embedded-api/openapi-validator.js
@@ -55,7 +55,6 @@ async function validateOpenAPISpec(spec) {
    const version = validator.version;
    
    console.log(`✅ OpenAPI spec validation passed!`);
-    console.log(`📋 OpenAPI version: ${version}`);
    
    // Perform additional custom validations for our specific use case
    try {
--- a/docusaurus/src/scripts/fetch-registry.js
+++ b/docusaurus/src/scripts/fetch-registry.js
@@ -0,0 +1,104 @@
+/**
+ * Utility to manage connector registry - fetching, caching, and extracting minimal data.
+ * This is a separate utility so it can be reused by multiple scripts.
+ */
+const fs = require("fs");
+const https = require("https");
+const { DATA_DIR, REGISTRY_CACHE_PATH, REGISTRY_URL } = require("./constants");
+
+function fetchConnectorRegistryFromRemote() {
+  return new Promise((resolve, reject) => {
+    console.log("Fetching connector registry data...");
+
+    https
+      .get(REGISTRY_URL, (response) => {
+        if (response.statusCode !== 200) {
+          reject(new Error(`Failed to fetch registry: ${response.statusCode}`));
+          return;
+        }
+
+        let data = "";
+
+        response.on("data", (chunk) => {
+          data += chunk;
+        });
+
+        response.on("end", () => {
+          try {
+            const registry = JSON.parse(data);
+            resolve(registry);
+          } catch (error) {
+            reject(
+              new Error(`Failed to parse registry data: ${error.message}`),
+            );
+          }
+        });
+      })
+      .on("error", (error) => {
+        reject(new Error(`Network error: ${error.message}`));
+      });
+  });
+}
+
+function extractMinimalRegistryData(fullRegistry) {
+  return fullRegistry.map((connector) => ({
+    id: connector.name_oss || connector.name_cloud
+      .toLowerCase()
+      .replace(/\s+/g, "-")
+      .replace(/[^a-z0-9-]/g, ""),
+    // Properties used by sidebar-connectors.js
+    docUrl:
+      connector.documentationUrl_cloud || connector.documentationUrl_oss || "",
+    supportLevel:
+      connector.supportLevel_cloud || connector.supportLevel_oss || "community",
+    // Properties used by remark/utils.js and remark/specDecoration.js
+    dockerRepository_oss: connector.dockerRepository_oss || "",
+    spec_oss: connector.spec_oss
+      ? {
+          connectionSpecification: connector.spec_oss.connectionSpecification,
+        }
+      : null,
+    // Properties used by remark/utils.js for buildArchivedRegistryEntry
+    name_oss: connector.name_oss || connector.name || "",
+    is_oss: connector.is_oss || false,
+    is_cloud: connector.is_cloud || false,
+    iconUrl_oss: connector.iconUrl_oss || "",
+    supportLevel_oss: connector.supportLevel_oss || "community",
+    documentationUrl_oss: connector.documentationUrl_oss || "",
+    // Properties used by remark/connectorList.js (isPypiConnector)
+    remoteRegistries_oss: connector.remoteRegistries_oss || {},
+  }));
+}
+
+async function fetchRegistry() {
+  // Check if cache already exists
+  if (fs.existsSync(REGISTRY_CACHE_PATH)) {
+    const cachedData = fs.readFileSync(REGISTRY_CACHE_PATH, "utf8");
+    const minimalRegistry = JSON.parse(cachedData);
+    return minimalRegistry;
+  }
+
+  // Fetch if cache doesn't exist
+  const fullRegistry = await fetchConnectorRegistryFromRemote();
+  const minimalRegistry = extractMinimalRegistryData(fullRegistry);
+
+  // Ensure data directory exists
+  if (!fs.existsSync(DATA_DIR)) {
+    fs.mkdirSync(DATA_DIR, { recursive: true });
+  }
+
+  // Save the minimal registry cache
+  fs.writeFileSync(
+    REGISTRY_CACHE_PATH,
+    JSON.stringify(minimalRegistry, null, 2),
+  );
+  console.log(`✓ Cached ${minimalRegistry.length} connectors`);
+
+  return minimalRegistry;
+}
+
+module.exports = {
+  fetchRegistry,
+  fetchConnectorRegistryFromRemote,
+  extractMinimalRegistryData,
+};
--- a/docusaurus/src/scripts/prepare-registry-cache.js
+++ b/docusaurus/src/scripts/prepare-registry-cache.js
@@ -0,0 +1,26 @@
+/**
+ * This script ensures the connector registry cache exists and is populated.
+ * It's called during the prebuild phase to prepare data for the build process.
+ *
+ * The cache is used by:
+ * - remark/specDecoration.js - needs: dockerRepository_oss, spec_oss.connectionSpecification
+ * - remark/connectorList.js - needs: remoteRegistries_oss for isPypiConnector filter
+ * - remark/utils.js - needs: dockerRepository_oss, name_oss, is_oss, is_cloud,
+ *                     iconUrl_oss, supportLevel_oss, documentationUrl_oss
+ * - sidebar-connectors.js - needs: docUrl, supportLevel, dockerRepository_oss
+ *
+ * This avoids network timeouts during the build process by fetching data once upfront.
+ * The cache file is cleaned up after the build completes (see cleanup-cache.js).
+ */
+const { fetchRegistry } = require("./fetch-registry");
+
+async function main() {
+  try {
+    await fetchRegistry();
+  } catch (error) {
+    console.error("Error preparing registry cache:", error);
+    process.exit(1);
+  }
+}
+
+main();
--- a/docusaurus/src/scripts/prepare-sidebar-data.js
+++ b/docusaurus/src/scripts/prepare-sidebar-data.js
@@ -1,101 +1,24 @@
 /**
- * This script fetches the connector registry data and processes it before
- * the build process starts. It ensures the data is available for the sidebar
- * to use even on the first build.
+ * This script ensures the connector registry cache exists and is populated.
+ * It's called during the prebuild phase to prepare data for the build process.
+ *
+ * The cache is used by:
+ * - remark/specDecoration.js - needs: dockerRepository_oss, spec_oss.connectionSpecification
+ * - remark/connectorList.js - needs: remoteRegistries_oss for isPypiConnector filter
+ * - remark/utils.js - needs: dockerRepository_oss, name_oss, is_oss, is_cloud,
+ *                     iconUrl_oss, supportLevel_oss, documentationUrl_oss
+ * - sidebar-connectors.js - needs: docUrl, supportLevel, dockerRepository_oss
+ *
+ * This avoids network timeouts during the build process by fetching data once upfront.
+ * The cache file is cleaned up after the build completes (see cleanup-cache.js).
 */
-const fs = require("fs");
-const https = require("https");
-const path = require("path");
-const REGISTRY_CACHE_PATH = path.join(
-  __dirname,
-  "..",
-  "data",
-  "connector_registry_slim.json",
-);
-
-const REGISTRY_URL =
-  "https://connectors.airbyte.com/files/generated_reports/connector_registry_report.json";
-
-function fetchConnectorRegistry() {
-  return new Promise((resolve, reject) => {
-    console.log("Fetching connector registry data...");
-
-    https
-      .get(REGISTRY_URL, (response) => {
-        if (response.statusCode !== 200) {
-          reject(new Error(`Failed to fetch registry: ${response.statusCode}`));
-          return;
-        }
-
-        let data = "";
-
-        response.on("data", (chunk) => {
-          data += chunk;
-        });
-
-        response.on("end", () => {
-          try {
-            const registry = JSON.parse(data);
-            resolve(registry);
-          } catch (error) {
-            reject(
-              new Error(`Failed to parse registry data: ${error.message}`),
-            );
-          }
-        });
-      })
-      .on("error", (error) => {
-        reject(new Error(`Network error: ${error.message}`));
-      });
-  });
-}
-
-function processRegistryData(registry) {
-  return registry
-    .map((connector) => {
-      const name = connector.name_oss || connector.name_cloud || "";
-      if (name) {
-        return {
-          id: name
-            .toLowerCase()
-            .replace(/\s+/g, "-")
-            .replace(/[^a-z0-9-]/g, ""),
-          type: connector.connector_type?.toLowerCase() || "unknown",
-          supportLevel:
-            connector.supportLevel_cloud ||
-            connector.supportLevel_oss ||
-            "community",
-          docUrl:
-            connector.documentationUrl_cloud ||
-            connector.documentationUrl_oss ||
-            "",
-        };
-      }
-    })
-    .filter(Boolean);
-}
+const { fetchRegistry } = require("./fetch-registry");

 async function main() {
  try {
-    const registry = await fetchConnectorRegistry();
-
-    const processedRegistry = processRegistryData(registry);
-
-    const dir = path.dirname(REGISTRY_CACHE_PATH);
-    if (!fs.existsSync(dir)) {
-      fs.mkdirSync(dir, { recursive: true });
-    }
-
-    fs.writeFileSync(
-      REGISTRY_CACHE_PATH,
-      JSON.stringify(processedRegistry, null, 2),
-    );
-
-    console.log(
-      `Connector registry data processed and saved to ${REGISTRY_CACHE_PATH}`,
-    );
+    await fetchRegistry();
  } catch (error) {
-    console.error("Error preparing sidebar data:", error);
+    console.error("Error preparing registry cache:", error);
    process.exit(1);
  }
 }
--- a/docusaurus/tsconfig.json
+++ b/docusaurus/tsconfig.json
@@ -1,7 +1,20 @@
 {
-  // This file is not used in compilation. It is here just for a nice editor experience.
-  "extends": "@tsconfig/docusaurus/tsconfig.json",
  "compilerOptions": {
-    "baseUrl": "."
-  }
+    "target": "ES2020",
+    "lib": ["ES2020", "DOM", "DOM.Iterable"],
+    "module": "ESNext",
+    "moduleResolution": "bundler",
+    "strict": true,
+    "esModuleInterop": true,
+    "skipLibCheck": true,
+    "forceConsistentCasingInFileNames": true,
+    "jsx": "react-jsx",
+    "declaration": true,
+    "declarationMap": true,
+    "sourceMap": true,
+    "resolveJsonModule": true,
+    "allowSyntheticDefaultImports": true
+  },
+  "include": ["docosaurus.config.ts"],
+  "exclude": ["node_modules", "build", ".docusaurus", "dist"]
 }