feat: add Embedded Api docs to docusaurus (#62875)

Co-authored-by: Claude <noreply@anthropic.com>

docusaurus/api-docs/embedded-api/.gitignore

@@ -0,0 +1,8 @@
+# This directory contains generated API documentation files
+# Generated during the build process from OpenAPI specifications
+# Only keep .gitignore and README.txt in version control
+
+# Ignore all files except .gitignore and README.txt
+*
+!.gitignore
+!README.txt

docusaurus/api-docs/embedded-api/README.txt

@@ -0,0 +1,21 @@
+Embedded API Documentation
+==========================
+
+This directory contains auto-generated API documentation files that are created during the build process.
+
+The files in this directory are generated from the OpenAPI specification defined in:
+- docusaurus/src/data/embedded_api_spec.json
+- docusaurus/src/scripts/embedded-api/prepare-embedded-api-spec.js
+
+Why is this folder gitignored?
+-------------------------------
+
+The API documentation files (*.api.mdx, sidebar.ts, etc.) are generated during `pnpm build` and should NOT be committed to git.
+This folder must exist for the build to succeed, but the generated files will be recreated each build.
+
+To regenerate the documentation files locally, run:
+  pnpm build
+
+For more information about the embedded API docs generation, see:
+- docusaurus/src/scripts/embedded-api/openapi-validator.js
+- docusaurus/src/scripts/embedded-api/prepare-embedded-api-spec.js

docusaurus/docusaurus.config.js

@@ -14,6 +14,9 @@ const connectorList = require("./src/remark/connectorList");
 const specDecoration = require("./src/remark/specDecoration");
 const docMetaTags = require("./src/remark/docMetaTags");
 const addButtonToTitle = require("./src/remark/addButtonToTitle");
+const fs = require("fs");
+
+const { SPEC_CACHE_PATH, API_SIDEBAR_PATH } = require("./src/scripts/embedded-api/constants");
 
 /** @type {import('@docusaurus/types').Config} */
 const config = {
@@ -184,6 +187,100 @@ const config = {
         ],
       },
     ],
+    [
+      "@docusaurus/plugin-content-docs",
+      {
+        id: "embedded-api",
+        path: "api-docs/embedded-api",
+        routeBasePath: "/embedded-api/",
+        docItemComponent: "@theme/ApiItem",
+        async sidebarItemsGenerator() {
+          // We only want to include visible endpoints on the sidebar. We need to filter out endpoints with tags
+          // that are not included in the spec. Even if we didn't need to filter out elements the OpenAPI plugin generates a sidebar.ts
+          // file that exports a nested object, but Docusaurus expects just the array of sidebar items, so we need to extracts the actual sidebar
+          // items from the generated file structure.
+
+          try {
+            const specPath = SPEC_CACHE_PATH; 
+
+            if (!fs.existsSync(specPath)) {
+              console.warn(
+                "Embedded API spec file not found, using empty sidebar",
+              );
+              return [];
+            }
+
+            const data = JSON.parse(fs.readFileSync(specPath, "utf8"));
+            console.log("Loaded embedded API spec from cache");
+
+            // Load the freshly generated sidebar (not the cached one from module load)
+            const sidebarPath = API_SIDEBAR_PATH;
+            let freshSidebar = [];
+
+            if (fs.existsSync(sidebarPath)) {
+              try {
+                const sidebarModule = require("./api-docs/embedded-api/sidebar.ts");
+                freshSidebar = sidebarModule.default || sidebarModule;
+                console.log("Loaded fresh sidebar from generated files");
+              } catch (sidebarError) {
+                console.warn(
+                  "Could not load fresh sidebar, using empty array:",
+                  sidebarError.message,
+                );
+                freshSidebar = [];
+              }
+            } else {
+              console.warn(
+                "Generated sidebar file not found, using empty array",
+              );
+              freshSidebar = [];
+            }
+
+            const allowedTags = data.tags?.map((tag) => tag["name"]) || [];
+
+            // Use freshly loaded sidebar items from the generated file
+            const sidebarItems = Array.isArray(freshSidebar)
+              ? freshSidebar
+              : [];
+
+            const filteredItems = sidebarItems.filter((item) => {
+              if (item.type !== "category") {
+                return true;
+              }
+
+              return allowedTags.includes(item.label);
+            });
+
+            return filteredItems;
+          } catch (error) {
+            console.warn(
+              "Error loading embedded API spec from cache:",
+              error.message,
+            );
+            return [];
+          }
+        },
+      },
+    ],
+    [
+      "docusaurus-plugin-openapi-docs",
+      {
+        id: "embedded-api",
+        docsPluginId: "embedded-api",
+        config: {
+          embedded: {
+            specPath: "src/data/embedded_api_spec.json",
+            outputDir: "api-docs/embedded-api",
+            sidebarOptions: {
+              groupPathsBy: "tag",
+              categoryLinkSource: "tag",
+              sidebarCollapsed: false,
+              sidebarCollapsible: false,
+            },
+          },
+        },
+      },
+    ],
     require.resolve("./src/plugins/enterpriseConnectors"),
     [
       "@signalwire/docusaurus-plugin-llms-txt",

docusaurus/package.json

@@ -4,8 +4,10 @@
   "private": true,
   "scripts": {
     "prepare-sidebar": "node src/scripts/prepare-sidebar-data.js",
-    "prebuild": "pnpm run prepare-sidebar",
-    "prestart": "pnpm run prepare-sidebar",
+    "gen-embedded-api-docs": "pnpm exec docusaurus clean-api-docs all && pnpm exec docusaurus gen-api-docs all",
+    "prepare-embedded-api": "pnpm run gen-embedded-api-docs && node src/scripts/embedded-api/prepare-embedded-api-spec.js && pnpm prettier --write src/data/embedded_api_spec.json",
+    "prebuild": "pnpm run prepare-sidebar && pnpm run prepare-embedded-api",
+    "prestart": "pnpm run prepare-sidebar && pnpm run prepare-embedded-api",
     "docusaurus": "docusaurus",
     "start": "node src/scripts/fetchSchema.js && docusaurus start --port 3005",
     "build": "node src/scripts/fetchSchema.js && docusaurus build",
@@ -85,8 +87,9 @@
     "@docusaurus/plugin-debug": "^3.7.0",
     "@docusaurus/plugin-sitemap": "^3.7.0",
     "@docusaurus/preset-classic": "^3.7.0",
-    "@docusaurus/remark-plugin-npm2yarn": "^3.9.1",
+    "@docusaurus/remark-plugin-npm2yarn": "^3.7.0",
     "@docusaurus/theme-classic": "^3.7.0",
+    "@docusaurus/theme-common": "^3.7.0",
     "@docusaurus/theme-mermaid": "^3.7.0",
     "@docusaurus/theme-search-algolia": "^3.7.0",
     "@docusaurus/types": "^3.7.0",
@@ -100,7 +103,10 @@
     "@markprompt/react": "^0.62.1",
     "@mdx-js/react": "^3.0.0",
     "@saucelabs/theme-github-codeblock": "^0.3.0",
+    "@seriousme/openapi-schema-validator": "^2.5.0",
     "@signalwire/docusaurus-plugin-llms-txt": "^1.0.1",
+    "ajv": "^8.17.1",
+    "ajv-formats": "^3.0.1",
     "async": "2.6.4",
     "autoprefixer": "10.4.16",
     "classnames": "^2.3.2",
@@ -164,5 +170,6 @@
   },
   "devDependencies": {
     "prettier": "3.5.3"
-  }
+  },
+  "packageManager": "pnpm@9.4.0+sha1.9217c800d4ab947a7aee520242a7b70d64fc7638"
 }

docusaurus/sidebar-ai-agents.js

@@ -13,36 +13,40 @@ export default {
           type: "category",
           label: "Embedded",
           items: [
-        {
-          type: "category",
-          label: "Widget",
-          items: [
-            "embedded/widget/quickstart",
             {
               type: "category",
-              label: "Tutorials",
+              label: "Widget",
               items: [
-                "embedded/widget/tutorials/prerequisites-setup",
-                "embedded/widget/tutorials/develop-your-app",
-                "embedded/widget/tutorials/use-embedded",
-              ]
+                "embedded/widget/quickstart",
+                {
+                  type: "category",
+                  label: "Tutorials",
+                  items: [
+                    "embedded/widget/tutorials/prerequisites-setup",
+                    "embedded/widget/tutorials/develop-your-app",
+                    "embedded/widget/tutorials/use-embedded",
+                  ],
+                },
+                "embedded/widget/managing-embedded",
+                "embedded/widget/template-tags",
+              ],
             },
-            "embedded/widget/managing-embedded",
-            "embedded/widget/template-tags",
-          ]
-        },
-        {
-          type: "category",
-          label: "API",
-          items: [
-            "embedded/api/README",
-            "embedded/api/connection-templates",
-            "embedded/api/source-templates",
-            "embedded/api/configuring-sources",
-          ]
-        },
-
-          ]
+            {
+              type: "category",
+              label: "API",
+              items: [
+                "embedded/api/README",
+                {
+                  type: "link",
+                  label: "Sonar API reference",
+                  href: "/embedded-api/sonar",
+                },
+                "embedded/api/connection-templates",
+                "embedded/api/source-templates",
+                "embedded/api/configuring-sources",
+              ],
+            },
+          ],
         },
         {
           type: "category",

docusaurus/src/css/custom.css

@@ -31,6 +31,12 @@
   --ifm-table-head-color: var(--color-white);
   --ifm-table-border-color: var(--ifm-color-primary-lightest);
   --docusaurus-highlighted-code-line-bg: rgba(0, 0, 0, 0.2);
+  
+  /* OpenAPI method badge colors */
+  --openapi-code-green: #49cc90;
+  --openapi-code-red: #f93e3e;
+  --openapi-code-blue: #61affe;
+  --openapi-code-orange: #fca130;
 
   --color-white: hsl(0, 0%, 100%);
   --color-grey-40: hsl(240, 25%, 98%);
@@ -406,3 +412,60 @@ nav a.navbar__link--active {
 header h1 {
   margin-bottom: 0;
 }
+
+article span.badge {
+  margin-bottom: 0;
+}
+
+/* OpenAPI HTTP Method Badges */
+li.theme-doc-sidebar-item-link.menu__list-item .menu__link {
+  display: flex;
+  align-items: center;
+  justify-content: start;
+}
+
+/* Target the parent li element that has the method class */
+li.api-method.post .menu__link::before,
+li.api-method.get .menu__link::before,
+li.api-method.put .menu__link::before,
+li.api-method.patch .menu__link::before,
+li.api-method.delete .menu__link::before {
+  content: "";
+  width: 42px;
+  height: 14px;
+  font-size: 10px;
+  text-transform: uppercase;
+  font-weight: 600;
+  border-radius: 0.25rem;
+  border: 1px solid;
+  margin-right: var(--ifm-spacing-horizontal);
+  text-align: center;
+  flex-shrink: 0;
+  border-color: transparent;
+  color: white;
+}
+
+li.api-method.get .menu__link::before {
+  content: "get";
+  background-color: var(--ifm-color-primary);
+}
+
+li.api-method.post .menu__link::before {
+  content: "post";
+  background-color: var(--openapi-code-green);
+}
+
+li.api-method.delete .menu__link::before {
+  content: "del";
+  background-color: var(--openapi-code-red);
+}
+
+li.api-method.put .menu__link::before {
+  content: "put";
+  background-color: var(--openapi-code-blue);
+}
+
+li.api-method.patch .menu__link::before {
+  content: "patch";
+  background-color: var(--openapi-code-orange);
+}

docusaurus/src/scripts/embedded-api/constants.js

@@ -0,0 +1,32 @@
+/**
+ * Shared constants for the Docusaurus documentation build process
+ * 
+ * This file contains paths and configuration values that are used across
+ * multiple scripts and configuration files.
+ */
+
+const path = require("path");
+
+// Get the project root directory (docusaurus folder)
+const PROJECT_ROOT = path.resolve(__dirname, "..", "..", "..");
+
+// Path to the cached embedded API OpenAPI specification
+const SPEC_CACHE_PATH = path.join(PROJECT_ROOT, "src", "data", "embedded_api_spec.json");
+
+// URL for fetching the latest embedded API specification
+
+const EMBEDDED_API_SPEC_URL = "https://airbyte-sonar-prod.s3.us-east-2.amazonaws.com/openapi/latest/app.json";
+
+// API documentation output directory (relative to project root)
+const API_DOCS_OUTPUT_DIR = "api-docs/embedded-api";
+
+// Sidebar file path for generated API docs
+const API_SIDEBAR_PATH = path.join(PROJECT_ROOT, API_DOCS_OUTPUT_DIR, "sidebar.ts");
+
+module.exports = {
+  PROJECT_ROOT,
+  SPEC_CACHE_PATH,
+  EMBEDDED_API_SPEC_URL,
+  API_DOCS_OUTPUT_DIR,
+  API_SIDEBAR_PATH,
+};

docusaurus/src/scripts/embedded-api/openapi-validator.js

@@ -0,0 +1,287 @@
+/**
+ * OpenAPI specification validator using @seriousme/openapi-schema-validator
+ * 
+ * This validator uses a purpose-built OpenAPI validation library that supports
+ * OpenAPI 2.0, 3.0.x, and 3.1.x specifications. It's been tested on over 2,000 
+ * real-world APIs from AWS, Microsoft, Google, etc.
+ * 
+ * Ensures compatibility with:
+ * 1. The docusaurus-plugin-openapi-docs generator
+ * 2. Our custom sidebar filtering logic in docusaurus.config.js
+ * 3. The Docusaurus theme-openapi-docs components
+ */
+
+// Note: Using dynamic import since this package is ESM-only
+// We'll import it inside the async function to maintain CommonJS compatibility
+
+/**
+ * Validates an OpenAPI specification using the schema validator
+ * @param {Object} spec - The OpenAPI spec to validate
+ * @throws {Error} If validation fails
+ * @returns {Object} The validated spec with additional metadata
+ */
+async function validateOpenAPISpec(spec) {
+  console.log("🔍 Validating OpenAPI spec with @seriousme/openapi-schema-validator...");
+  
+  try {
+    // Dynamic import to handle ESM module in CommonJS context
+    const { Validator } = await import("@seriousme/openapi-schema-validator");
+    
+    // Create validator instance
+    const validator = new Validator();
+    
+    // Validate the spec
+    const result = await validator.validate(spec);
+    
+    if (!result.valid) {
+      let errorMessages = "Unknown validation errors";
+      
+      if (result.errors && Array.isArray(result.errors)) {
+        errorMessages = result.errors.map(err => {
+          const path = err.instancePath || err.schemaPath || 'unknown';
+          const message = err.message || 'validation failed';
+          return `  • ${path}: ${message}`;
+        }).join('\n');
+      } else if (result.errors) {
+        // Handle case where errors is not an array
+        errorMessages = `  • ${String(result.errors)}`;
+      }
+      
+      throw new Error(`OpenAPI spec validation failed:\n${errorMessages}`);
+    }
+
+    // Get the validated specification
+    const validatedSpec = validator.specification;
+    const version = validator.version;
+    
+    console.log(`✅ OpenAPI spec validation passed!`);
+    console.log(`📋 OpenAPI version: ${version}`);
+    
+    // Perform additional custom validations for our specific use case
+    try {
+      performDocumentationSpecificValidations(validatedSpec);
+    } catch (validationError) {
+      // Convert validation warnings to non-fatal warnings for undefined tags
+      if (validationError.message && validationError.message.includes('undefined tags')) {
+        console.warn(`⚠️  ${validationError.message}`);
+        console.warn(`📝 This may cause missing sidebar sections in the documentation`);
+      } else {
+        // Re-throw other validation errors
+        throw validationError;
+      }
+    }
+    
+    // Log validation success with stats
+    const stats = generateValidationStats(validatedSpec);
+    console.log(`📊 Spec stats: ${stats.pathCount} paths, ${stats.tagCount} tags, ${stats.operationCount} operations`);
+    
+    return validatedSpec;
+    
+  } catch (importError) {
+    // Fallback to basic validation if the import fails
+    console.warn("⚠️  Could not load OpenAPI schema validator, using basic validation:", importError.message);
+    return performBasicValidation(spec);
+  }
+}
+
+/**
+ * Performs additional validations specific to our documentation needs
+ * @param {Object} spec - The validated OpenAPI spec
+ */
+function performDocumentationSpecificValidations(spec) {
+  // 1. Ensure tags are defined (critical for sidebar generation)
+  if (!spec.tags || !Array.isArray(spec.tags) || spec.tags.length === 0) {
+    throw new Error("OpenAPI spec must have tags array (required for sidebar generation)");
+  }
+
+  // 2. Validate that every defined tag has at least one operation (critical for docs)
+  const definedTags = new Set(spec.tags.map(tag => tag.name));
+  const usedTags = new Set();
+
+  for (const [path, pathItem] of Object.entries(spec.paths)) {
+    const httpMethods = ['get', 'put', 'post', 'delete', 'options', 'head', 'patch', 'trace'];
+    
+    for (const method of httpMethods) {
+      const operation = pathItem[method];
+      if (operation?.tags) {
+        operation.tags.forEach(tag => {
+          usedTags.add(tag);
+        });
+      }
+    }
+  }
+
+  // 3. Critical: Every defined tag must have at least one operation
+  const unusedTags = Array.from(definedTags).filter(tag => !usedTags.has(tag));
+  if (unusedTags.length > 0) {
+    throw new Error(`Defined tags have no operations and will not appear in docs: ${unusedTags.join(', ')}`);
+  }
+
+  // 4. Warn about operations using undefined tags (but don't fail - let them be uncategorized)
+  const undefinedTags = Array.from(usedTags).filter(tag => !definedTags.has(tag));
+  if (undefinedTags.length > 0) {
+    console.warn(`⚠️  Operations reference undefined tags (will be uncategorized): ${undefinedTags.join(', ')}`);
+  }
+
+  // 4. Validate that operations have required fields for documentation
+  const criticalIssues = [];
+  let operationsWithIssues = 0;
+  
+  for (const [path, pathItem] of Object.entries(spec.paths)) {
+    const httpMethods = ['get', 'put', 'post', 'delete', 'options', 'head', 'patch', 'trace'];
+    
+    for (const method of httpMethods) {
+      const operation = pathItem[method];
+      if (operation) {
+        const operationRef = `${path}[${method.toUpperCase()}]`;
+        
+        // Critical: operationId is required for MDX generation
+        if (!operation.operationId) {
+          criticalIssues.push(`${operationRef} missing operationId (required for MDX file generation)`);
+        }
+        
+        // Critical: tags are required for sidebar organization
+        if (!operation.tags || operation.tags.length === 0) {
+          criticalIssues.push(`${operationRef} missing tags (required for sidebar grouping)`);
+        }
+        
+        // Critical: responses are required
+        if (!operation.responses || Object.keys(operation.responses).length === 0) {
+          criticalIssues.push(`${operationRef} missing responses (required for documentation)`);
+        }
+        
+        // Warning: summary is recommended
+        if (!operation.summary) {
+          console.warn(`⚠️  ${operationRef} missing summary (recommended for UI display)`);
+          operationsWithIssues++;
+        }
+
+        // Warning: should have at least one success response
+        if (operation.responses) {
+          const responseCodes = Object.keys(operation.responses);
+          const hasSuccessResponse = responseCodes.some(code => 
+            code.startsWith('2') || code === 'default'
+          );
+          
+          if (!hasSuccessResponse) {
+            console.warn(`⚠️  ${operationRef} has no success response (2xx or default)`);
+          }
+        }
+      }
+    }
+  }
+  
+  // Throw error if there are critical issues
+  if (criticalIssues.length > 0) {
+    throw new Error(`Critical OpenAPI documentation issues found:\n${criticalIssues.map(issue => `  • ${issue}`).join('\n')}`);
+  }
+
+  if (operationsWithIssues > 0) {
+    console.warn(`⚠️  Found ${operationsWithIssues} operation(s) with potential documentation issues`);
+  }
+
+  console.log(`✅ Documentation-specific validations passed`);
+  console.log(`🏷️  Tags: ${definedTags.size} defined, all have operations (will appear in docs)`);
+  if (undefinedTags.length > 0) {
+    console.log(`🏷️  Additional tags: ${undefinedTags.length} used by operations but not formally defined`);
+  }
+}
+
+/**
+ * Fallback basic validation if the schema validator can't be loaded
+ * @param {Object} spec - The OpenAPI spec
+ * @returns {Object} The spec with basic validation
+ */
+function performBasicValidation(spec) {
+  console.log("🔍 Performing basic OpenAPI spec validation...");
+  
+  // Basic structure validation
+  if (!spec || typeof spec !== "object") {
+    throw new Error("Invalid spec: not an object");
+  }
+
+  if (!spec.openapi && !spec.swagger) {
+    throw new Error("Invalid spec: missing openapi or swagger version field");
+  }
+
+  if (!spec.info || !spec.info.title) {
+    throw new Error("Invalid spec: missing info.title");
+  }
+
+  if (!spec.info.version) {
+    throw new Error("Invalid spec: missing info.version");
+  }
+  
+  // Check for empty title
+  if (spec.info.title === "") {
+    throw new Error("Invalid spec: info.title cannot be empty");
+  }
+
+  if (!spec.paths || typeof spec.paths !== "object") {
+    throw new Error("Invalid spec: missing or invalid paths");
+  }
+  
+  // Check for empty paths
+  if (Object.keys(spec.paths).length === 0) {
+    throw new Error("Invalid spec: paths object cannot be empty");
+  }
+
+  if (!spec.tags || !Array.isArray(spec.tags)) {
+    throw new Error("Invalid spec: missing or invalid tags array (required for sidebar generation)");
+  }
+  
+  // Check for empty tags
+  if (spec.tags.length === 0) {
+    throw new Error("Invalid spec: tags array cannot be empty (required for sidebar generation)");
+  }
+  
+  // Check tag structure
+  for (let i = 0; i < spec.tags.length; i++) {
+    const tag = spec.tags[i];
+    if (!tag || typeof tag !== "object") {
+      throw new Error(`Invalid spec: tag[${i}] must be an object`);
+    }
+    if (!tag.name || typeof tag.name !== "string" || tag.name === "") {
+      throw new Error(`Invalid spec: tag[${i}] must have a non-empty name`);
+    }
+  }
+
+  console.log(`✅ Basic OpenAPI spec validation passed`);
+  console.log(`📋 API: ${spec.info.title} v${spec.info.version}`);
+  
+  const stats = generateValidationStats(spec);
+  console.log(`📊 Spec stats: ${stats.pathCount} paths, ${stats.tagCount} tags, ${stats.operationCount} operations`);
+  
+  return spec;
+}
+
+/**
+ * Generates validation statistics for logging
+ * @param {Object} spec - The OpenAPI spec
+ * @returns {Object} Statistics object
+ */
+function generateValidationStats(spec) {
+  const pathCount = Object.keys(spec.paths || {}).length;
+  const tagCount = spec.tags?.length || 0;
+  
+  let operationCount = 0;
+  for (const pathItem of Object.values(spec.paths || {})) {
+    const methods = ['get', 'put', 'post', 'delete', 'options', 'head', 'patch', 'trace'];
+    operationCount += methods.filter(method => pathItem[method]).length;
+  }
+
+  const schemaCount = spec.components?.schemas ? Object.keys(spec.components.schemas).length : 0;
+
+  return {
+    pathCount,
+    tagCount,
+    operationCount,
+    schemaCount,
+    version: spec.info?.version || 'unknown',
+    title: spec.info?.title || 'Unknown API'
+  };
+}
+
+module.exports = {
+  validateOpenAPISpec,
+};

docusaurus/src/scripts/embedded-api/prepare-embedded-api-spec.js

@@ -0,0 +1,154 @@
+/**
+ * This script fetches the embedded API OpenAPI spec and processes it before
+ * the build process starts. It ensures the spec is available and validated
+ * for both the OpenAPI plugin and sidebar generation.
+ */
+const fs = require("fs");
+const https = require("https");
+const path = require("path");
+const { validateOpenAPISpec } = require("./openapi-validator");
+const { SPEC_CACHE_PATH, EMBEDDED_API_SPEC_URL } = require("./constants");
+
+function fetchEmbeddedApiSpec() {
+  return new Promise((resolve, reject) => {
+    console.log("Fetching embedded API spec...");
+
+    https
+      .get(EMBEDDED_API_SPEC_URL, (response) => {
+        if (response.statusCode !== 200) {
+          reject(new Error(`Failed to fetch spec: ${response.statusCode}`));
+          return;
+        }
+
+        let data = "";
+
+        response.on("data", (chunk) => {
+          data += chunk;
+        });
+
+        response.on("end", () => {
+          try {
+            const spec = JSON.parse(data);
+            resolve(spec);
+          } catch (error) {
+            reject(
+              new Error(`Failed to parse spec data: ${error.message}`),
+            );
+          }
+        });
+      })
+      .on("error", (error) => {
+        reject(new Error(`Network error: ${error.message}`));
+      });
+  });
+}
+
+// validateSpec function is now handled by AJV validator
+// This provides comprehensive OpenAPI 3.1 schema validation
+
+function processSpec(spec) {
+  // For now, return the spec as-is
+  // In the future, we could add processing/transformations here if needed
+  return spec;
+}
+
+function loadPreviousSpec() {
+  try {
+    if (fs.existsSync(SPEC_CACHE_PATH)) {
+      const previousSpec = JSON.parse(fs.readFileSync(SPEC_CACHE_PATH, 'utf8'));
+      console.log("📁 Found previous spec version:", previousSpec.info?.version || "unknown");
+      return previousSpec;
+    }
+  } catch (error) {
+    console.warn("⚠️  Could not load previous spec:", error.message);
+  }
+  return null;
+}
+
+async function main() {
+  
+  const previousSpec = loadPreviousSpec();
+  try {
+    console.log("🔄 Attempting to fetch latest embedded API spec...");
+    const spec = await fetchEmbeddedApiSpec();
+    
+    // Validate using comprehensive OpenAPI schema validator
+    const validatedSpec = await validateOpenAPISpec(spec);
+    const processedSpec = processSpec(validatedSpec);
+
+    // Ensure the data directory exists
+    const dir = path.dirname(SPEC_CACHE_PATH);
+    if (!fs.existsSync(dir)) {
+      fs.mkdirSync(dir, { recursive: true });
+    }
+
+    fs.writeFileSync(
+      SPEC_CACHE_PATH,
+      JSON.stringify(processedSpec, null, 2),
+    );
+
+    console.log(
+      `✅ Embedded API spec processed and saved to ${SPEC_CACHE_PATH}`,
+    );
+    
+    if (previousSpec && previousSpec.info?.version !== processedSpec.info?.version) {
+      //TODO: we don't use versioning yet, so we should find another way to compare specs and output changes?
+      console.log(`📝 Spec updated from ${previousSpec.info?.version} to ${processedSpec.info?.version}`);
+    }
+    
+  } catch (error) {
+    console.error("❌ Error fetching/processing latest spec:", error.message);
+    
+    if (previousSpec) {
+      console.log("🔄 Using previous cached spec version to continue build...");
+      console.log(`📋 Previous spec info: ${previousSpec.info?.title} v${previousSpec.info?.version}`);
+      
+      // Ensure the previous spec is still written to the expected location
+      const dir = path.dirname(SPEC_CACHE_PATH);
+      if (!fs.existsSync(dir)) {
+        fs.mkdirSync(dir, { recursive: true });
+      }
+      
+      fs.writeFileSync(
+        SPEC_CACHE_PATH,
+        JSON.stringify(previousSpec, null, 2),
+      );
+      
+      console.log("✅ Build will continue with previous spec version");
+    } else {
+      console.error("💥 No previous spec found and latest fetch failed");
+      console.error("📝 Creating minimal fallback spec to allow build to continue");
+      console.error("💡 Tip: Run this script successfully once to create an initial cache");
+      
+      // Create minimal valid OpenAPI spec that will result in empty docs
+      const fallbackSpec = {
+        openapi: "3.1.0",
+        info: {
+          title: "Embedded API (Unavailable)",
+          version: "0.0.0",
+          description: "The embedded API specification could not be fetched. Please check your network connection and try again."
+        },
+        paths: {},
+        tags: [],
+        components: {}
+      };
+      
+      // Ensure the data directory exists
+      const dir = path.dirname(SPEC_CACHE_PATH);
+      if (!fs.existsSync(dir)) {
+        fs.mkdirSync(dir, { recursive: true });
+      }
+      
+      // Cleanup is now handled by package.json scripts before doc generation
+
+      fs.writeFileSync(
+        SPEC_CACHE_PATH,
+        JSON.stringify(fallbackSpec, null, 2),
+      );
+      
+      console.log("✅ Build will continue with empty embedded API documentation");
+    }
+  }
+}
+
+main();

poe-tasks/repo-root-tasks.toml

@@ -20,7 +20,7 @@
 help = "Build the docs.airbyte.com site documentation using Docusaurus."
 shell = '''
 cd $POE_ROOT/docusaurus
-pnpm install
+pnpm install --ignore-scripts
 pnpm build
 '''