1
0
mirror of synced 2025-12-19 18:14:56 -05:00
This commit is contained in:
letiescanciano
2025-12-16 15:41:29 +01:00
parent 40f2384cdb
commit a392e28724
9 changed files with 16479 additions and 2 deletions

View File

@@ -0,0 +1,6 @@
# Generated API documentation files
# These are automatically generated by the docusaurus-plugin-openapi-docs
# and should not be committed to version control
*.mdx
sidebar.ts

View File

@@ -12,11 +12,11 @@
"scripts": {
"prepare-registry-cache": "NODE_OPTIONS='--max-old-space-size=4096' node src/scripts/prepare-registry-cache.js",
"prepare-embedded-api": "NODE_OPTIONS='--max-old-space-size=4096' node src/scripts/embedded-api/prepare-embedded-api-spec.js && NODE_OPTIONS='--max-old-space-size=4096' pnpm prettier --write src/data/embedded_api_spec.json && NODE_OPTIONS='--max-old-space-size=4096' pnpm exec docusaurus clean-api-docs embedded-api --plugin-id embedded-api && NODE_OPTIONS='--max-old-space-size=4096' pnpm exec docusaurus gen-api-docs embedded-api --plugin-id embedded-api",
"prepare-public-api": "NODE_OPTIONS='--max-old-space-size=4096' node src/scripts/public-api/prepare-public-api-spec.js && NODE_OPTIONS='--max-old-space-size=4096' pnpm exec docusaurus clean-api-docs public-api --plugin-id public-api && NODE_OPTIONS='--max-old-space-size=4096' pnpm exec docusaurus gen-api-docs public-api --plugin-id public-api",
"prepare-public-api": "NODE_OPTIONS='--max-old-space-size=4096' node src/scripts/public-api/prepare-public-api-spec.js && NODE_OPTIONS='--max-old-space-size=4096' pnpm exec docusaurus clean-api-docs public-api --plugin-id public-api && NODE_OPTIONS='--max-old-space-size=4096' pnpm exec docusaurus gen-api-docs public-api --plugin-id public-api && node src/scripts/public-api/replace-request-schema.js",
"cleanup-cache": "node src/scripts/cleanup-cache.js",
"prebuild": "pnpm run prepare-registry-cache && pnpm run prepare-embedded-api && pnpm run prepare-public-api",
"postbuild": "pnpm run cleanup-cache",
"prestart": "pnpm run prepare-registry-cache && pnpm run prepare-embedded-api",
"prestart": "pnpm run prepare-registry-cache",
"docusaurus": "docusaurus",
"start": "NODE_OPTIONS='--max-old-space-size=8192' node src/scripts/fetchSchema.js && NODE_OPTIONS='--max-old-space-size=8192' docusaurus start --port 3005",
"build": "NODE_OPTIONS='--max-old-space-size=8192' node src/scripts/fetchSchema.js && NODE_OPTIONS='--max-old-space-size=8192' docusaurus build",

View File

@@ -0,0 +1,176 @@
import React, { useState, useEffect, useRef, useMemo } from 'react';
import styles from './index.module.css';
import Schema from '@theme/Schema';
export const CreateSourceRequestSchema = () => {
const [sourceConfigs, setSourceConfigs] = useState([]);
const [loading, setLoading] = useState(true);
const [error, setError] = useState(null);
const [searchQuery, setSearchQuery] = useState('');
const [selectedSource, setSelectedSource] = useState(null);
const [isOpen, setIsOpen] = useState(false);
const inputRef = useRef(null);
const dropdownRef = useRef(null);
// Load source configurations from the pre-processed JSON file
useEffect(() => {
const loadSourceConfigs = async () => {
try {
setLoading(true);
const response = await fetch('/data/source-configs-dereferenced.json');
if (!response.ok) {
throw new Error(`Failed to load source configurations: ${response.statusCode}`);
}
const data = await response.json();
setSourceConfigs(data);
// Auto-select first source
if (data.length > 0) {
setSelectedSource(data[0]);
}
} catch (err) {
setError(err.message);
console.error('Error loading source configs:', err);
} finally {
setLoading(false);
}
};
loadSourceConfigs();
}, []);
// Filter sources based on search query
const filteredSources = useMemo(() => {
if (!searchQuery.trim()) {
return sourceConfigs;
}
const query = searchQuery.toLowerCase();
return sourceConfigs.filter(
source =>
source.displayName.toLowerCase().includes(query) ||
source.id.toLowerCase().includes(query)
);
}, [searchQuery, sourceConfigs]);
// Handle click outside to close dropdown
useEffect(() => {
const handleClickOutside = (event) => {
if (
dropdownRef.current &&
!dropdownRef.current.contains(event.target) &&
inputRef.current &&
!inputRef.current.contains(event.target)
) {
setIsOpen(false);
}
};
document.addEventListener('mousedown', handleClickOutside);
return () => document.removeEventListener('mousedown', handleClickOutside);
}, []);
// Handle keyboard navigation
const handleKeyDown = (e) => {
if (e.key === 'Escape') {
setIsOpen(false);
} else if (e.key === 'ArrowDown' || e.key === 'ArrowUp') {
e.preventDefault();
setIsOpen(true);
}
};
if (loading) {
return (
<div className={styles.container}>
<div className={styles.skeleton}>
<div className={styles.skeletonPlaceholder} />
</div>
</div>
);
}
if (error) {
return (
<div className={styles.container}>
<div className={styles.error}>
<strong>Error loading source configurations:</strong> {error}
</div>
</div>
);
}
return (
<div className={styles.container}>
<div className={styles.selectorWrapper}>
<label className={styles.label}>Select a source connector</label>
<div className={styles.inputWrapper}>
<input
ref={inputRef}
type="text"
className={styles.input}
placeholder="Search sources..."
value={searchQuery}
onChange={(e) => {
setSearchQuery(e.target.value);
setIsOpen(true);
}}
onFocus={() => setIsOpen(true)}
onKeyDown={handleKeyDown}
aria-label="Search source connectors"
aria-expanded={isOpen}
aria-controls="source-dropdown"
/>
<div className={styles.chevron} data-open={isOpen}>
</div>
</div>
{isOpen && (
<div
ref={dropdownRef}
className={styles.dropdown}
id="source-dropdown"
role="listbox"
>
{filteredSources.length > 0 ? (
filteredSources.map((source) => (
<button
key={source.id}
className={`${styles.option} ${
selectedSource?.id === source.id ? styles.selected : ''
}`}
onClick={() => {
setSelectedSource(source);
setSearchQuery('');
setIsOpen(false);
}}
role="option"
aria-selected={selectedSource?.id === source.id}
>
<span className={styles.optionName}>{source.displayName}</span>
<span className={styles.optionId}>{source.id}</span>
</button>
))
) : (
<div className={styles.noResults}>
No sources found for "{searchQuery}"
</div>
)}
</div>
)}
</div>
{selectedSource && (
<div className={styles.schemaWrapper}>
<h3 className={styles.schemaTitle}>
Configuration for {selectedSource.displayName}
</h3>
<div className={styles.schemaContent}>
<Schema schema={selectedSource.schema} />
</div>
</div>
)}
</div>
);
};
export default CreateSourceRequestSchema;

View File

@@ -0,0 +1,289 @@
/* Container */
.container {
display: flex;
flex-direction: column;
gap: 2rem;
margin: 1.5rem 0;
}
/* Selector Wrapper */
.selectorWrapper {
position: relative;
width: 100%;
max-width: 600px;
}
.label {
display: block;
margin-bottom: 0.75rem;
font-weight: 600;
font-size: 0.95rem;
color: var(--ifm-color-emphasis-900);
letter-spacing: -0.3px;
}
/* Input Styling */
.inputWrapper {
position: relative;
display: flex;
align-items: center;
}
.input {
width: 100%;
padding: 0.75rem 1rem 0.75rem 1rem;
border: 1px solid var(--ifm-color-border);
border-radius: 6px;
background-color: var(--ifm-background-surface-primary);
color: var(--ifm-color-content);
font-size: 0.95rem;
font-family: inherit;
transition: all 0.2s cubic-bezier(0.4, 0, 0.2, 1);
box-shadow: 0 1px 2px rgba(0, 0, 0, 0.05);
}
.input:hover {
border-color: var(--ifm-color-primary-lighter);
box-shadow: 0 2px 4px rgba(0, 0, 0, 0.08);
}
.input:focus {
outline: none;
border-color: var(--ifm-color-primary);
box-shadow: 0 0 0 3px var(--ifm-color-primary-lightest),
0 2px 8px rgba(0, 0, 0, 0.1);
}
/* Chevron Icon */
.chevron {
position: absolute;
right: 0.75rem;
pointer-events: none;
font-size: 0.7rem;
color: var(--ifm-color-emphasis-600);
transition: transform 0.2s cubic-bezier(0.4, 0, 0.2, 1);
display: flex;
align-items: center;
}
.chevron[data-open='true'] {
transform: rotate(180deg);
}
/* Dropdown Menu */
.dropdown {
position: absolute;
top: calc(100% + 0.5rem);
left: 0;
right: 0;
background-color: var(--ifm-background-surface-primary);
border: 1px solid var(--ifm-color-border);
border-radius: 6px;
box-shadow: 0 8px 24px rgba(0, 0, 0, 0.12);
z-index: 1000;
max-height: 320px;
overflow-y: auto;
animation: slideDown 0.2s cubic-bezier(0.4, 0, 0.2, 1);
}
@keyframes slideDown {
from {
opacity: 0;
transform: translateY(-8px);
}
to {
opacity: 1;
transform: translateY(0);
}
}
/* Dropdown Scrollbar Styling */
.dropdown::-webkit-scrollbar {
width: 6px;
}
.dropdown::-webkit-scrollbar-track {
background: transparent;
}
.dropdown::-webkit-scrollbar-thumb {
background: var(--ifm-color-emphasis-400);
border-radius: 3px;
}
.dropdown::-webkit-scrollbar-thumb:hover {
background: var(--ifm-color-emphasis-600);
}
/* Option Items */
.option {
width: 100%;
padding: 0.75rem 1rem;
border: none;
background: transparent;
color: var(--ifm-color-content);
text-align: left;
cursor: pointer;
font-size: 0.95rem;
font-family: inherit;
transition: background-color 0.15s ease;
display: flex;
flex-direction: column;
gap: 0.25rem;
border-bottom: 1px solid transparent;
}
.option:first-child {
border-top-left-radius: 5px;
border-top-right-radius: 5px;
}
.option:last-child {
border-bottom: none;
border-bottom-left-radius: 5px;
border-bottom-right-radius: 5px;
}
.option:hover {
background-color: var(--ifm-color-emphasis-100);
}
.option.selected {
background-color: var(--ifm-color-primary-lightest);
color: var(--ifm-color-primary-darker);
font-weight: 500;
}
.option.selected:hover {
background-color: var(--ifm-color-primary-lighter);
}
.optionName {
font-weight: 500;
color: var(--ifm-color-content);
}
.optionId {
font-size: 0.8rem;
color: var(--ifm-color-emphasis-600);
font-family: 'Courier New', monospace;
}
/* No Results */
.noResults {
padding: 1.5rem 1rem;
text-align: center;
color: var(--ifm-color-emphasis-600);
font-size: 0.9rem;
}
/* Schema Wrapper */
.schemaWrapper {
display: flex;
flex-direction: column;
gap: 1rem;
padding-top: 1rem;
border-top: 1px solid var(--ifm-color-border);
}
.schemaTitle {
margin: 0;
font-size: 1.1rem;
font-weight: 600;
color: var(--ifm-color-emphasis-900);
letter-spacing: -0.3px;
}
.schemaContent {
flex: 1;
}
/* Skeleton Loader */
.skeleton {
display: flex;
flex-direction: column;
gap: 1rem;
}
.skeletonPlaceholder {
height: 2.5rem;
border-radius: 6px;
background: linear-gradient(
90deg,
var(--ifm-color-emphasis-200) 0%,
var(--ifm-color-emphasis-100) 50%,
var(--ifm-color-emphasis-200) 100%
);
background-size: 200% 100%;
animation: shimmer 1.5s infinite;
}
@keyframes shimmer {
0% {
background-position: 200% 0;
}
100% {
background-position: -200% 0;
}
}
/* Error State */
.error {
padding: 1rem;
border-radius: 6px;
background-color: var(--ifm-color-danger-lightest);
border: 1px solid var(--ifm-color-danger-light);
color: var(--ifm-color-danger-darker);
font-size: 0.95rem;
line-height: 1.5;
}
/* Responsive Design */
@media (max-width: 768px) {
.container {
gap: 1.5rem;
}
.selectorWrapper {
max-width: 100%;
}
.input {
font-size: 1rem; /* Prevent zoom on iOS */
}
.dropdown {
max-height: 250px;
}
.option {
padding: 0.65rem 0.875rem;
font-size: 0.9rem;
}
.label {
font-size: 0.9rem;
}
.schemaTitle {
font-size: 1rem;
}
}
/* Dark mode adjustments (handled by Docusaurus CSS variables) */
html[data-theme='dark'] .input {
box-shadow: 0 1px 2px rgba(0, 0, 0, 0.3);
}
html[data-theme='dark'] .input:hover {
box-shadow: 0 2px 4px rgba(0, 0, 0, 0.4);
}
html[data-theme='dark'] .input:focus {
box-shadow: 0 0 0 3px var(--ifm-color-primary-lightest),
0 2px 8px rgba(0, 0, 0, 0.3);
}
html[data-theme='dark'] .dropdown {
box-shadow: 0 8px 24px rgba(0, 0, 0, 0.4);
}

File diff suppressed because it is too large Load Diff

View File

@@ -19,6 +19,14 @@ const PUBLIC_API_SPEC_CACHE_PATH = path.join(
"public_api_spec.json",
);
// Path to the dereferenced source configurations (extracted at build time)
const SOURCE_CONFIGS_DEREFERENCED_PATH = path.join(
PROJECT_ROOT,
"src",
"data",
"source-configs-dereferenced.json",
);
// URL for fetching the latest public API specification
const PUBLIC_API_SPEC_BASE_URL =
"https://raw.githubusercontent.com/airbytehq/airbyte-platform/refs/heads/main/airbyte-api/server-api/src/main/openapi/";
@@ -60,6 +68,7 @@ const PUBLIC_API_SIDEBAR_PATH = path.join(
module.exports = {
PROJECT_ROOT,
PUBLIC_API_SPEC_CACHE_PATH,
SOURCE_CONFIGS_DEREFERENCED_PATH,
PUBLIC_API_SPEC_BASE_URL,
PUBLIC_SPEC_FILE_NAMES,
CONFIG_API_SPEC_URL,

View File

@@ -19,6 +19,7 @@ const {
PUBLIC_API_SPEC_BASE_URL,
PUBLIC_SPEC_FILE_NAMES,
CONFIG_API_SPEC_URL,
SOURCE_CONFIGS_DEREFERENCED_PATH,
} = require("./constants");
// Create a YAML version of the cache path
@@ -403,6 +404,130 @@ function filterSourceConfigurationToCertified(spec, certifiedSourceNames) {
return spec;
}
/**
* Recursively dereference a schema by resolving all $ref pointers
* @param {Object} schema - The schema to dereference
* @param {Object} allSchemas - All available schemas for reference resolution
* @param {Set<string>} visited - Track visited schemas to avoid infinite loops
* @returns {Object} The dereferenced schema
*/
function dereferenceSchema(schema, allSchemas, visited = new Set()) {
if (!schema || typeof schema !== 'object' || Array.isArray(schema)) {
return schema;
}
// Handle $ref pointers
if (schema.$ref) {
const refPath = schema.$ref;
const schemaName = refPath.split('/').pop(); // Extract name from "#/components/schemas/source-postgres"
// Prevent infinite loops
if (visited.has(schemaName)) {
return schema; // Return as-is if we've already visited
}
visited.add(schemaName);
const referencedSchema = allSchemas[schemaName];
if (!referencedSchema) {
console.warn(`⚠️ Could not find schema: ${schemaName}`);
return schema;
}
// Recursively dereference the referenced schema
return dereferenceSchema(referencedSchema, allSchemas, new Set(visited));
}
// Recursively process nested objects
const dereferenced = {};
for (const [key, value] of Object.entries(schema)) {
if (value && typeof value === 'object') {
if (Array.isArray(value)) {
dereferenced[key] = value.map(item =>
typeof item === 'object' ? dereferenceSchema(item, allSchemas, visited) : item
);
} else {
dereferenced[key] = dereferenceSchema(value, allSchemas, new Set(visited));
}
} else {
dereferenced[key] = value;
}
}
return dereferenced;
}
/**
* Formats a source name for display
* @param {string} name - The source name (e.g., "source-postgres")
* @returns {string} Formatted name (e.g., "Postgres")
*/
function formatSourceName(name) {
if (!name.startsWith('source-')) {
return name;
}
// Remove "source-" prefix and convert to title case
const withoutPrefix = name.substring(7); // Remove "source-"
return withoutPrefix
.split('-')
.map(word => word.charAt(0).toUpperCase() + word.slice(1))
.join(' ');
}
/**
* Extract and dereference all source schemas from SourceConfiguration
* @param {Object} spec - The OpenAPI specification
* @returns {Array} Array of source configs with dereferenced schemas
*/
function extractAndDereferenceSourceSchemas(spec) {
if (!spec.components?.schemas?.SourceConfiguration?.oneOf) {
console.warn('⚠️ SourceConfiguration.oneOf not found - skipping extraction');
return [];
}
const sourceConfig = spec.components.schemas.SourceConfiguration;
const allSchemas = spec.components.schemas;
const sourceConfigs = [];
console.log(`🔍 Extracting ${sourceConfig.oneOf.length} source configurations...`);
for (const sourceRef of sourceConfig.oneOf) {
if (!sourceRef.title) {
console.warn('⚠️ Source config without title found, skipping');
continue;
}
const sourceId = sourceRef.title; // e.g., "source-postgres"
const sourceSchema = allSchemas[sourceId];
if (!sourceSchema) {
console.warn(`⚠️ Could not find schema for ${sourceId}`);
continue;
}
try {
// Dereference the schema
const dereferenced = dereferenceSchema(sourceSchema, allSchemas);
sourceConfigs.push({
id: sourceId,
displayName: formatSourceName(sourceId),
schema: dereferenced
});
} catch (error) {
console.warn(`⚠️ Error dereferencing ${sourceId}:`, error instanceof Error ? error.message : String(error));
}
}
// Sort alphabetically by displayName
sourceConfigs.sort((a, b) => a.displayName.localeCompare(b.displayName));
console.log(`✅ Extracted and dereferenced ${sourceConfigs.length} source configurations`);
return sourceConfigs;
}
/**
* Main processing function
*/
@@ -476,6 +601,25 @@ async function main() {
const validatedSpec = await validateOpenAPISpec(enhancedSpec);
const processedSpec = validatedSpec;
// Extract and dereference source schemas for the component
const sourceConfigsDeref = extractAndDereferenceSourceSchemas(processedSpec);
// Save dereferenced source configurations to JSON file
if (sourceConfigsDeref.length > 0) {
const sourceConfigsJson = JSON.stringify(sourceConfigsDeref, null, 2);
// Ensure the data directory exists
const dataDir = path.dirname(SOURCE_CONFIGS_DEREFERENCED_PATH);
if (!fs.existsSync(dataDir)) {
fs.mkdirSync(dataDir, { recursive: true });
}
fs.writeFileSync(SOURCE_CONFIGS_DEREFERENCED_PATH, sourceConfigsJson);
console.log(
`✅ Source configurations extracted and saved to ${SOURCE_CONFIGS_DEREFERENCED_PATH}`
);
}
// Ensure the data directory exists
const dir = path.dirname(PUBLIC_API_SPEC_CACHE_PATH);
if (!fs.existsSync(dir)) {

View File

@@ -0,0 +1,84 @@
#!/usr/bin/env node
/**
* Script to replace RequestSchema components with CreateSourceRequestSchema
* Runs after prepare-public-api-spec to process regenerated MDX files
*/
const fs = require('fs');
const path = require('path');
const MDX_FILES = [
'docs/developers/api-reference/create-source.api.mdx',
'docs/developers/api-reference/create-destination.api.mdx'
];
const PROJECT_ROOT = path.join(__dirname, '../../../..');
function replaceRequestSchema(filePath) {
console.log(`\n📝 Processing: ${filePath}`);
const fullPath = path.join(PROJECT_ROOT, filePath);
if (!fs.existsSync(fullPath)) {
console.warn(`⚠️ File not found: ${fullPath}`);
return false;
}
let content = fs.readFileSync(fullPath, 'utf-8');
const originalLength = content.length;
// Replace the import statement
const importPattern = /import\s+RequestSchema\s+from\s+["']@theme\/RequestSchema["'];?/g;
content = content.replace(importPattern, '// CreateSourceRequestSchema is auto-registered in MDXComponents');
// Replace <RequestSchema ... /> components
// This regex handles components that may span multiple lines
const componentPattern = /<RequestSchema\s+[^>]*\/>/gs;
content = content.replace(componentPattern, '<CreateSourceRequestSchema />');
// Replace <RequestSchema ... > ... </RequestSchema> components
// This handles the case where the component has children (though it shouldn't)
const multilinePattern = /<RequestSchema\s+[^>]*>[\s\S]*?<\/RequestSchema>/g;
content = content.replace(multilinePattern, '<CreateSourceRequestSchema />');
const newLength = content.length;
const sizeDiff = originalLength - newLength;
if (originalLength !== newLength) {
fs.writeFileSync(fullPath, content, 'utf-8');
console.log(`✅ Updated: ${filePath}`);
console.log(` Size reduction: ${(sizeDiff / 1024).toFixed(2)} KB`);
return true;
} else {
console.log(` No changes needed: ${filePath}`);
return false;
}
}
function main() {
console.log('🔄 Replacing RequestSchema components...\n');
let filesUpdated = 0;
for (const mdxFile of MDX_FILES) {
if (replaceRequestSchema(mdxFile)) {
filesUpdated++;
}
}
console.log(`\n✨ Complete! Updated ${filesUpdated} file(s)`);
if (filesUpdated > 0) {
console.log('\n📋 Summary:');
console.log(' - RequestSchema imports replaced with comments');
console.log(' - RequestSchema components replaced with CreateSourceRequestSchema');
console.log(' - MDX files are now much smaller');
process.exit(0);
} else {
console.log('\n⚠ No files were updated');
process.exit(0);
}
}
main();

View File

@@ -8,6 +8,7 @@ import { Navattic } from "@site/src/components/Navattic";
import { ProductInformation } from "@site/src/components/ProductInformation";
import { PyAirbyteExample } from "@site/src/components/PyAirbyteExample";
import { SpecSchema } from "@site/src/components/SpecSchema";
import { CreateSourceRequestSchema } from "@site/src/components/CreateSourceRequestSchema";
import MDXComponents from "@theme-original/MDXComponents";
import { CardWithIcon } from "../../components/Card/Card";
import { CopyPageButton } from "../../components/CopyPageButton/CopyPageButton";
@@ -27,6 +28,7 @@ export default {
HeaderDecoration,
Navattic,
SpecSchema,
CreateSourceRequestSchema,
PyAirbyteExample,
ProductInformation,
Details,