airbyte/docusaurus/sidebar-connectors.js

const fs = require("fs");
const path = require("path");
const {
  REGISTRY_CACHE_PATH,
  SOURCES_DOCS,
  DESTINATIONS_DOCS,
  ENTERPRISE_CONNECTORS_DOCS,
} = require("./src/scripts/constants");

function getFilenamesInDir(prefix, dir, excludes) {
  return fs
    .readdirSync(dir)
    .filter(
      (fileName) =>
        !(
          fileName.endsWith(".inapp.md") ||
          fileName.endsWith("-migrations.md") ||
          fileName.endsWith(".js") ||
          fileName === "low-code.md"
        ),
    )
    .map((fileName) => fileName.replace(".md", ""))
    .filter((fileName) => excludes.indexOf(fileName.toLowerCase()) === -1)
    .map((filename) => {
      let contentTitle = filename;

      // Get the first header of the markdown document
      try {
        const filePath = path.join(dir, `${filename}.md`);
        const fileContent = fs.readFileSync(filePath, "utf8");
        const firstLine = fileContent
          .split("\n")
          .find((line) => line.trim().startsWith("# "));
        contentTitle = firstLine
          ? firstLine.replace(/^#\s*/, "").trim()
          : filename;
      } catch (error) {
        console.warn(
          `Warning: Using filename as title for ${path.join(prefix, filename)}`,
        );
      }

      // If there is a migration doc for this connector nest this under the original doc as "Migration Guide"
      const migrationDocPath = path.join(dir, `${filename}-migrations.md`);
      if (fs.existsSync(migrationDocPath)) {
        return {
          type: "category",
          label: contentTitle,
          key: `${prefix}${filename}-category`,
          link: { type: "doc", id: path.join(prefix, filename) },
          items: [
            {
              type: "doc",
              id: path.join(prefix, `${filename}-migrations`),
              label: "Migration Guide",
              key: `${prefix}${filename}-migrations`,
            },
          ],
        };
      }

      return {
        type: "doc",
        id: prefix + filename,
        label: contentTitle,
        key: `${prefix}${filename}`,
      };
    });
}

function loadConnectorRegistry() {
  try {
    if (fs.existsSync(REGISTRY_CACHE_PATH)) {
      const data = fs.readFileSync(REGISTRY_CACHE_PATH, "utf8");
      return JSON.parse(data);
    }
  } catch (error) {
    console.warn("Error loading connector registry data:", error.message);
  }
  return [];
}

function addSupportLevelToConnectors(connectors, registry) {
  return connectors.map((item) => {
    // Get the ID from either doc-type or category-type items
    const id = item.type === "doc" ? item.id : item.link && item.link.id;

    if (!id) {
      return item;
    }

    let connectorInfo = registry.find((record) => record.docUrl.includes(id));

    if (connectorInfo) {
      return {
        ...item,
        customProps: {
          ...item.customProps,
          supportLevel: connectorInfo.supportLevel,
        },
      };
    }

    return item;
  });
}

function groupConnectorsBySupportLevel(connectors, keyPrefix = "") {
  const grouped = connectors.reduce(
    (acc, item) => {
      const supportLevel = item.customProps?.supportLevel || "community";
      if (acc[supportLevel]) {
        acc[supportLevel].push(item);
      } else {
        acc.community.push(item);
      }
      return acc;
    },
    { certified: [], community: [], enterprise: [] },
  );

  // Create categories for each support level
  const categories = [];

  if (grouped.certified.length > 0) {
    categories.push({
      type: "category",
      label: "Airbyte",
      collapsible: true,
      collapsed: true,
      key: `${keyPrefix}airbyte`,
      items: grouped.certified.sort((a, b) => a.label.localeCompare(b.label)),
    });
  }

  if (grouped.community.length > 0) {
    categories.push({
      type: "category",
      label: "Marketplace",
      collapsible: true,
      collapsed: true,
      key: `${keyPrefix}marketplace`,
      items: grouped.community.sort((a, b) => a.label.localeCompare(b.label)),
    });
  }

  if (grouped.enterprise.length > 0) {
    categories.push({
      type: "category",
      label: "Enterprise",
      collapsible: true,
      collapsed: true,
      key: `${keyPrefix}enterprise`,
      items: grouped.enterprise.sort((a, b) => a.label.localeCompare(b.label)),
    });
  }

  return categories;
}

const sourcePostgres = {
  type: "category",
  label: "Postgres",
  key: "sources-postgres-category",
  link: {
    type: "doc",
    id: "sources/postgres",
  },
  items: [
    {
      type: "doc",
      label: "Cloud SQL for Postgres",
      id: "sources/postgres/cloud-sql-postgres",
      key: "sources-postgres-cloud-sql",
    },
    {
      type: "doc",
      label: "Troubleshooting",
      id: "sources/postgres/postgres-troubleshooting",
      key: "sources-postgres-troubleshooting",
    },
  ],
};

const sourceMongoDB = {
  type: "category",
  label: "Mongo DB",
  key: "sources-mongodb-v2-category",
  link: {
    type: "doc",
    id: "sources/mongodb-v2",
  },
  items: [
    {
      type: "doc",
      label: "Migration Guide",
      id: "sources/mongodb-v2-migrations",
      key: "sources-mongodb-v2-migrations",
    },
    {
      type: "doc",
      label: "Troubleshooting",
      id: "sources/mongodb-v2/mongodb-v2-troubleshooting",
      key: "sources-mongodb-v2-troubleshooting",
    },
  ],
};

const sourceMysql = {
  type: "category",
  label: "MySQL",
  key: "sources-mysql-category",
  link: {
    type: "doc",
    id: "sources/mysql",
  },
  items: [
    {
      type: "doc",
      label: "Troubleshooting",
      id: "sources/mysql/mysql-troubleshooting",
      key: "sources-mysql-troubleshooting",
    },
  ],
};

const sourceMssql = {
  type: "category",
  label: "MS SQL Server (MSSQL)",
  key: "sources-mssql-category",
  link: {
    type: "doc",
    id: "sources/mssql",
  },
  items: [
    {
      type: "doc",
      label: "Troubleshooting",
      id: "sources/mssql/mssql-troubleshooting",
      key: "sources-mssql-troubleshooting",
    },
  ],
};
function getSourceConnectors(registry) {
  const sources = getFilenamesInDir("sources/", SOURCES_DOCS, [
    "readme",
    "postgres",
    "mongodb-v2",
    "mssql",
    "mysql",
  ]);

  const specialSources = [
    sourcePostgres,
    sourceMongoDB,
    sourceMysql,
    sourceMssql,
  ];
  const enterpriseSources = getFilenamesInDir(
    "enterprise-connectors/",
    ENTERPRISE_CONNECTORS_DOCS,
    ["readme"],
  );
  const enterpriseSourcesWithSupportLevel = enterpriseSources
  .filter((item) => {
    const itemId = item.id || item.link?.id;
    return itemId && itemId.includes("source");
  })
  .map((item) => {
      return {
        ...item,
        customProps: { ...item.customProps, supportLevel: "enterprise" },
      };
    });
  const sourcesWithSupportLevel = addSupportLevelToConnectors(
    [...specialSources, ...sources],
    registry,
  );

  return [...sourcesWithSupportLevel, ...enterpriseSourcesWithSupportLevel];
}

const destinationS3 = {
  type: "category",
  label: "S3",
  key: "destinations-s3-category",
  link: {
    type: "doc",
    id: "destinations/s3",
  },
  items: [
    {
      type: "doc",
      label: "Migration Guide",
      id: "destinations/s3-migrations",
      key: "destinations-s3-migrations",
    },
    {
      type: "doc",
      label: "Troubleshooting",
      id: "destinations/s3/s3-troubleshooting",
      key: "destinations-s3-troubleshooting",
    },
  ],
};

const destinationPostgres = {
  type: "category",
  label: "Postgres",
  key: "destinations-postgres-category",
  link: {
    type: "doc",
    id: "destinations/postgres",
  },
  items: [
    {
      type: "doc",
      label: "Migration Guide",
      id: "destinations/postgres-migrations",
      key: "destinations-postgres-migrations",
    },
    {
      type: "doc",
      label: "Troubleshooting",
      id: "destinations/postgres/postgres-troubleshooting",
      key: "destinations-postgres-troubleshooting",
    },
  ],
};

// Mssql destination is on the connector registry as mssql-v2, so we need to manually create the sidebar item
const destinationMsSql = {
  type: "category",
  label: "MS SQL Server (MSSQL)",
  key: "destinations-mssql-category",
  link: {
    type: "doc",
    id: "destinations/mssql",
  },
  customProps: {
    supportLevel: "certified",
  },
  items: [
    {
      type: "doc",
      label: "Migration Guide",
      id: "destinations/mssql-migrations",
      key: "destinations-mssql-migrations",
    },
  ],
};

function getDestinationConnectors(registry) {
  const specialDestinationConnectors = [destinationS3, destinationPostgres];
  const destinations = getFilenamesInDir("destinations/", DESTINATIONS_DOCS, [
    "s3",
    "postgres",
    "mssql",
    "readme",
  ]);
  const destinationsWithSupportLevel = addSupportLevelToConnectors(
    [...specialDestinationConnectors, ...destinations],
    registry,
  );

  const enterpriseDestinations = getFilenamesInDir(
    "enterprise-connectors/",
    ENTERPRISE_CONNECTORS_DOCS,
    ["readme"],
  );
  const enterpriseDestinationsWithSupportLevel = enterpriseDestinations
  .filter((item) => {
    const itemId = item.id || item.link?.id;
    return itemId && itemId.includes("destination");
  })
  .map((item) => {
      return {
        ...item,
        customProps: {
          ...item.customProps,
          supportLevel: "enterprise",
        },
      };
    });

  return [
    ...destinationsWithSupportLevel,
    ...enterpriseDestinationsWithSupportLevel,
    destinationMsSql,
  ];
}

function buildConnectorSidebar() {
  const registry = loadConnectorRegistry();

  const sourcesWithSupportLevel = getSourceConnectors(registry);

  const destinationConnectors = getDestinationConnectors(registry);

  const sourcesBySupportLevel = groupConnectorsBySupportLevel(
    sourcesWithSupportLevel,
    "sources-",
  );
  const destinationsBySupportLevel = groupConnectorsBySupportLevel(
    destinationConnectors,
    "destinations-",
  );

  return {
    type: "category",
    label: "Connectors",
    collapsible: false,
    link: {
      type: "doc",
      id: "README",
    },
    items: [
      {
        type: "category",
        label: "Sources",
        link: {
          type: "doc",
          id: "sources/README",
        },
        collapsible: true,
        collapsed: false,
        items: sourcesBySupportLevel,
      },
      {
        type: "category",
        label: "Destinations",
        link: {
          type: "doc",
          id: "destinations/README",
        },
        collapsible: true,
        collapsed: false,
        items: destinationsBySupportLevel,
      },
      "connector-support-levels",
      {
        type: "doc",
        id: "custom-connectors",
      },
      "locating-files-local-destination",
      "speed-improvements",
    ],
  };
}

const connectorSidebar = buildConnectorSidebar();

module.exports = {
  connectors: [connectorSidebar],
};