From 99658b9a8b771a3f90d97faf186bb99ba3caff4f Mon Sep 17 00:00:00 2001 From: "Aaron (\"AJ\") Steers" Date: Tue, 28 Oct 2025 10:26:58 -0700 Subject: [PATCH] feat(metadata-service): Add JSON schema bundling for IDE validation and CDK models (#68672) Co-authored-by: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- .gitignore | 3 + .../connectors/metadata_service/lib/README.md | 10 + .../lib/bin/bundle-schemas.js | 48 + ...classes.sh => generate-metadata-models.sh} | 6 +- .../ConnectorMetadataDefinitionV0.json | 855 ++++++++++++++++++ .../metadata_service/lib/package-lock.json | 62 ++ .../metadata_service/lib/package.json | 12 + .../metadata_service/lib/pyproject.toml | 2 +- 8 files changed, 996 insertions(+), 2 deletions(-) create mode 100755 airbyte-ci/connectors/metadata_service/lib/bin/bundle-schemas.js rename airbyte-ci/connectors/metadata_service/lib/bin/{generate-python-classes.sh => generate-metadata-models.sh} (81%) create mode 100644 airbyte-ci/connectors/metadata_service/lib/metadata_service/models/generated/ConnectorMetadataDefinitionV0.json create mode 100644 airbyte-ci/connectors/metadata_service/lib/package-lock.json create mode 100644 airbyte-ci/connectors/metadata_service/lib/package.json diff --git a/.gitignore b/.gitignore index 328282f487f..8d2d5864528 100644 --- a/.gitignore +++ b/.gitignore @@ -59,6 +59,9 @@ coverage.xml .pytest_cache/ cover/ +# Node.js +node_modules/ + # dbt profiles.yml diff --git a/airbyte-ci/connectors/metadata_service/lib/README.md b/airbyte-ci/connectors/metadata_service/lib/README.md index 90b191eef9a..fb2981e9d90 100644 --- a/airbyte-ci/connectors/metadata_service/lib/README.md +++ b/airbyte-ci/connectors/metadata_service/lib/README.md @@ -10,6 +10,16 @@ To use this submodule, it is recommended that you use Poetry to manage dependenc poetry install ``` +### Node.js Requirement + +The model generation process also requires Node.js to bundle JSON schemas. Install Node.js: + +- On macOS: `brew install node` +- On Ubuntu/Debian: `sudo apt-get install nodejs npm` +- On other systems: https://nodejs.org/ + +Node.js dependencies will be automatically installed when running `poetry run poe generate-models`. + ## Generating Models This submodule includes a tool for generating Python models from JSON Schema specifications. To generate the models, we use the library [datamodel-code-generator](https://github.com/koxudaxi/datamodel-code-generator). The generated models are stored in `models/generated`. diff --git a/airbyte-ci/connectors/metadata_service/lib/bin/bundle-schemas.js b/airbyte-ci/connectors/metadata_service/lib/bin/bundle-schemas.js new file mode 100755 index 00000000000..003d71e7a26 --- /dev/null +++ b/airbyte-ci/connectors/metadata_service/lib/bin/bundle-schemas.js @@ -0,0 +1,48 @@ +#!/usr/bin/env node + +/** + * Bundle JSON schemas using @apidevtools/json-schema-ref-parser + * This script resolves all $ref references in the schema files and creates a single bundled schema. + */ + +const $RefParser = require('@apidevtools/json-schema-ref-parser'); +const fs = require('fs'); +const path = require('path'); + +const YAML_DIR = 'metadata_service/models/src'; +const OUTPUT_DIR = 'metadata_service/models/generated'; +const ENTRY_SCHEMA = path.join(YAML_DIR, 'ConnectorMetadataDefinitionV0.yaml'); +const BUNDLE_OUTPUT = path.join(OUTPUT_DIR, 'ConnectorMetadataDefinitionV0.json'); + +async function bundleSchemas() { + try { + console.log('📦 Bundling JSON schemas...'); + console.log(` Entry schema: ${ENTRY_SCHEMA}`); + console.log(` Output: ${BUNDLE_OUTPUT}`); + + if (!fs.existsSync(YAML_DIR)) { + console.error(`❌ Error: The yaml directory does not exist: ${YAML_DIR}`); + process.exit(1); + } + + if (!fs.existsSync(OUTPUT_DIR)) { + fs.mkdirSync(OUTPUT_DIR, { recursive: true }); + } + + const schema = await $RefParser.bundle(ENTRY_SCHEMA, { + dereference: { + circular: 'ignore' // Handle circular references gracefully + } + }); + + fs.writeFileSync(BUNDLE_OUTPUT, JSON.stringify(schema, null, 2)); + + console.log('✅ Successfully bundled schema to', BUNDLE_OUTPUT); + console.log(' This bundled schema can be used for IDE validation and other tools.'); + } catch (error) { + console.error('❌ Error bundling schemas:', error.message); + process.exit(1); + } +} + +bundleSchemas(); diff --git a/airbyte-ci/connectors/metadata_service/lib/bin/generate-python-classes.sh b/airbyte-ci/connectors/metadata_service/lib/bin/generate-metadata-models.sh similarity index 81% rename from airbyte-ci/connectors/metadata_service/lib/bin/generate-python-classes.sh rename to airbyte-ci/connectors/metadata_service/lib/bin/generate-metadata-models.sh index 4bab5ffea1f..4f5790990b9 100755 --- a/airbyte-ci/connectors/metadata_service/lib/bin/generate-python-classes.sh +++ b/airbyte-ci/connectors/metadata_service/lib/bin/generate-metadata-models.sh @@ -16,7 +16,7 @@ fi rm -rf "$OUTPUT_DIR"/*.py mkdir -p "$OUTPUT_DIR" -echo "# generated by generate-python-classes" > "$OUTPUT_DIR"/__init__.py +echo "# generated by generate-metadata-models" > "$OUTPUT_DIR"/__init__.py for f in "$YAML_DIR"/*.yaml; do filename_wo_ext=$(basename "$f" | cut -d . -f 1) @@ -30,3 +30,7 @@ for f in "$YAML_DIR"/*.yaml; do --enum-field-as-literal all \ --disable-timestamp done + +echo "" +echo "Generating bundled JSON schema..." +node bin/bundle-schemas.js diff --git a/airbyte-ci/connectors/metadata_service/lib/metadata_service/models/generated/ConnectorMetadataDefinitionV0.json b/airbyte-ci/connectors/metadata_service/lib/metadata_service/models/generated/ConnectorMetadataDefinitionV0.json new file mode 100644 index 00000000000..8407ee5a634 --- /dev/null +++ b/airbyte-ci/connectors/metadata_service/lib/metadata_service/models/generated/ConnectorMetadataDefinitionV0.json @@ -0,0 +1,855 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "$id": "https://github.com/airbytehq/airbyte/airbyte-ci/connectors_ci/metadata_service/lib/models/src/ConnectorMetadataDefinitionV0.yml", + "title": "ConnectorMetadataDefinitionV0", + "description": "describes the metadata of a connector", + "type": "object", + "required": [ + "metadataSpecVersion", + "data" + ], + "additionalProperties": false, + "properties": { + "metadataSpecVersion": { + "type": "string" + }, + "data": { + "type": "object", + "required": [ + "name", + "definitionId", + "connectorType", + "dockerRepository", + "dockerImageTag", + "license", + "documentationUrl", + "githubIssueLabel", + "connectorSubtype", + "releaseStage" + ], + "additionalProperties": false, + "properties": { + "name": { + "type": "string" + }, + "icon": { + "type": "string" + }, + "definitionId": { + "type": "string", + "format": "uuid" + }, + "connectorBuildOptions": { + "$schema": "http://json-schema.org/draft-07/schema#", + "$id": "https://github.com/airbytehq/airbyte/airbyte-ci/connectors/metadata_service/lib/metadata_service/models/src/ConnectorBuildOptions.yaml", + "title": "ConnectorBuildOptions", + "description": "metadata specific to the build process.", + "type": "object", + "additionalProperties": false, + "properties": { + "baseImage": { + "type": "string" + } + } + }, + "connectorTestSuitesOptions": { + "type": "array", + "items": { + "$schema": "http://json-schema.org/draft-07/schema#", + "$id": "https://github.com/airbytehq/airbyte/airbyte-ci/connectors/metadata_service/lib/metadata_service/models/src/ConnectorTestOptions.yaml", + "title": "ConnectorTestSuiteOptions", + "description": "Options for a specific connector test suite.", + "type": "object", + "required": [ + "suite" + ], + "additionalProperties": false, + "properties": { + "suite": { + "description": "Name of the configured test suite", + "type": "string", + "enum": [ + "unitTests", + "integrationTests", + "acceptanceTests", + "liveTests" + ] + }, + "testSecrets": { + "description": "List of secrets required to run the test suite", + "type": "array", + "items": { + "$schema": "http://json-schema.org/draft-07/schema#", + "$id": "https://github.com/airbytehq/airbyte/airbyte-ci/connectors/metadata_service/lib/metadata_service/models/src/TestSecret.yaml", + "title": "Secret", + "description": "An object describing a secret's metadata", + "type": "object", + "required": [ + "name", + "secretStore" + ], + "additionalProperties": false, + "properties": { + "name": { + "type": "string", + "description": "The secret name in the secret store" + }, + "fileName": { + "type": "string", + "description": "The name of the file to which the secret value would be persisted" + }, + "secretStore": { + "$schema": "http://json-schema.org/draft-07/schema#", + "$id": "https://github.com/airbytehq/airbyte/airbyte-ci/connectors/metadata_service/lib/metadata_service/models/src/TestSecret.yaml", + "title": "SecretStore", + "description": "An object describing a secret store metadata", + "type": "object", + "required": [ + "name", + "secretStore" + ], + "additionalProperties": false, + "properties": { + "alias": { + "type": "string", + "description": "The alias of the secret store which can map to its actual secret address" + }, + "type": { + "type": "string", + "description": "The type of the secret store", + "enum": [ + "GSM" + ] + } + } + } + } + } + }, + "testConnections": { + "description": "List of sandbox cloud connections that tests can be run against", + "type": "array", + "items": { + "$schema": "http://json-schema.org/draft-07/schema#", + "$id": "https://github.com/airbytehq/airbyte/airbyte-ci/connectors/metadata_service/lib/metadata_service/models/src/TestConnections.yaml", + "title": "TestConnections", + "description": "List of sandbox cloud connections that tests can be run against", + "type": "object", + "required": [ + "name", + "id" + ], + "additionalProperties": false, + "properties": { + "name": { + "type": "string", + "description": "The connection name" + }, + "id": { + "type": "string", + "description": "The connection ID" + } + } + } + } + } + } + }, + "connectorType": { + "type": "string", + "enum": [ + "destination", + "source" + ] + }, + "dockerRepository": { + "type": "string" + }, + "dockerImageTag": { + "type": "string" + }, + "supportsDbt": { + "type": "boolean" + }, + "supportsNormalization": { + "type": "boolean" + }, + "license": { + "type": "string" + }, + "documentationUrl": { + "type": "string", + "format": "uri" + }, + "githubIssueLabel": { + "type": "string" + }, + "maxSecondsBetweenMessages": { + "description": "Maximum delay between 2 airbyte protocol messages, in second. The source will timeout if this delay is reached", + "type": "integer" + }, + "releaseDate": { + "description": "The date when this connector was first released, in yyyy-mm-dd format.", + "type": "string", + "format": "date" + }, + "protocolVersion": { + "type": "string", + "description": "the Airbyte Protocol version supported by the connector" + }, + "erdUrl": { + "type": "string", + "description": "The URL where you can visualize the ERD" + }, + "connectorSubtype": { + "type": "string", + "enum": [ + "api", + "database", + "datalake", + "file", + "custom", + "message_queue", + "unknown", + "vectorstore" + ] + }, + "releaseStage": { + "$schema": "http://json-schema.org/draft-07/schema#", + "$id": "https://github.com/airbytehq/airbyte-platform/blob/main/airbyte-config/config-models/src/main/resources/types/ReleaseStage.yaml", + "title": "ReleaseStage", + "description": "enum that describes a connector's release stage", + "type": "string", + "enum": [ + "alpha", + "beta", + "generally_available", + "custom" + ] + }, + "supportLevel": { + "$schema": "http://json-schema.org/draft-07/schema#", + "$id": "https://github.com/airbytehq/airbyte/airbyte-ci/connectors_ci/metadata_service/lib/models/src/SupportLevel.yaml", + "title": "SupportLevel", + "description": "enum that describes a connector's release stage", + "type": "string", + "enum": [ + "community", + "certified", + "archived" + ] + }, + "tags": { + "type": "array", + "description": "An array of tags that describe the connector. E.g: language:python, keyword:rds, etc.", + "items": { + "type": "string" + }, + "default": [] + }, + "registryOverrides": { + "anyOf": [ + { + "type": "object", + "additionalProperties": false, + "properties": { + "oss": { + "anyOf": [ + { + "$schema": "http://json-schema.org/draft-07/schema#", + "$id": "https://github.com/airbytehq/airbyte/airbyte-ci/connectors_ci/metadata_service/lib/models/src/RegistryOverrides.yml", + "title": "RegistryOverrides", + "description": "describes the overrides per registry of a connector", + "type": "object", + "additionalProperties": false, + "required": [ + "enabled" + ], + "properties": { + "enabled": { + "type": "boolean", + "default": false + }, + "name": { + "type": "string" + }, + "dockerRepository": { + "type": "string" + }, + "dockerImageTag": { + "type": "string" + }, + "supportsDbt": { + "type": "boolean" + }, + "supportsNormalization": { + "type": "boolean" + }, + "license": { + "type": "string" + }, + "documentationUrl": { + "type": "string", + "format": "uri" + }, + "connectorSubtype": { + "type": "string" + }, + "allowedHosts": { + "$ref": "#/properties/data/properties/allowedHosts" + }, + "normalizationConfig": { + "$ref": "#/properties/data/properties/normalizationConfig" + }, + "suggestedStreams": { + "$ref": "#/properties/data/properties/suggestedStreams" + }, + "resourceRequirements": { + "$ref": "#/properties/data/properties/resourceRequirements" + } + } + } + ] + }, + "cloud": { + "anyOf": [ + { + "$ref": "#/properties/data/properties/registryOverrides/anyOf/0/properties/oss/anyOf/0" + } + ] + } + } + } + ] + }, + "allowedHosts": { + "$schema": "http://json-schema.org/draft-07/schema#", + "$id": "https://github.com/airbytehq/airbyte/airbyte-ci/connectors/metadata_service/lib/metadata_service/models/src/AllowedHosts.yaml", + "title": "AllowedHosts", + "description": "A connector's allowed hosts. If present, the platform will limit communication to only hosts which are listed in `AllowedHosts.hosts`.", + "type": "object", + "additionalProperties": true, + "properties": { + "hosts": { + "type": "array", + "description": "An array of hosts that this connector can connect to. AllowedHosts not being present for the source or destination means that access to all hosts is allowed. An empty list here means that no network access is granted.", + "items": { + "type": "string" + } + } + } + }, + "releases": { + "$schema": "http://json-schema.org/draft-07/schema#", + "$id": "https://github.com/airbytehq/airbyte/airbyte-ci/connectors/metadata_service/lib/metadata_service/models/src/ConnectorReleases.yaml", + "title": "ConnectorReleases", + "description": "Contains information about different types of releases for a connector.", + "type": "object", + "additionalProperties": false, + "properties": { + "rolloutConfiguration": { + "$schema": "http://json-schema.org/draft-07/schema#", + "$id": "https://github.com/airbytehq/airbyte/airbyte-ci/connectors/metadata_service/lib/metadata_service/models/src/RolloutConfiguration.yaml", + "title": "RolloutConfiguration", + "description": "configuration for the rollout of a connector", + "type": "object", + "additionalProperties": false, + "properties": { + "enableProgressiveRollout": { + "type": "boolean", + "default": false, + "description": "Whether to enable progressive rollout for the connector." + }, + "initialPercentage": { + "type": "integer", + "minimum": 0, + "maximum": 100, + "default": 0, + "description": "The percentage of users that should receive the new version initially." + }, + "maxPercentage": { + "type": "integer", + "minimum": 0, + "maximum": 100, + "default": 50, + "description": "The percentage of users who should receive the release candidate during the test phase before full rollout." + }, + "advanceDelayMinutes": { + "type": "integer", + "minimum": 10, + "default": 10, + "description": "The number of minutes to wait before advancing the rollout percentage." + } + } + }, + "breakingChanges": { + "$schema": "http://json-schema.org/draft-07/schema#", + "$id": "https://github.com/airbytehq/airbyte/airbyte-ci/connectors/metadata_service/lib/metadata_service/models/src/ConnectorBreakingChanges.yaml", + "title": "ConnectorBreakingChanges", + "description": "Each entry denotes a breaking change in a specific version of a connector that requires user action to upgrade.", + "type": "object", + "additionalProperties": false, + "minProperties": 1, + "patternProperties": { + "^\\d+\\.\\d+\\.\\d+$": { + "$ref": "#/properties/data/properties/releases/properties/breakingChanges/definitions/VersionBreakingChange" + } + }, + "definitions": { + "VersionBreakingChange": { + "description": "Contains information about a breaking change, including the deadline to upgrade and a message detailing the change.", + "type": "object", + "additionalProperties": false, + "required": [ + "upgradeDeadline", + "message" + ], + "properties": { + "upgradeDeadline": { + "description": "The deadline by which to upgrade before the breaking change takes effect.", + "type": "string", + "format": "date" + }, + "message": { + "description": "Descriptive message detailing the breaking change.", + "type": "string" + }, + "deadlineAction": { + "description": "Action to do when the deadline is reached.", + "type": "string", + "enum": [ + "auto_upgrade", + "disable" + ] + }, + "migrationDocumentationUrl": { + "description": "URL to documentation on how to migrate to the current version. Defaults to ${documentationUrl}-migrations#${version}", + "type": "string", + "format": "uri" + }, + "scopedImpact": { + "description": "List of scopes that are impacted by the breaking change. If not specified, the breaking change cannot be scoped to reduce impact via the supported scope types.", + "type": "array", + "minItems": 1, + "items": { + "$ref": "#/properties/data/properties/releases/properties/breakingChanges/definitions/BreakingChangeScope" + } + } + } + }, + "BreakingChangeScope": { + "description": "A scope that can be used to limit the impact of a breaking change.", + "type": "object", + "oneOf": [ + { + "$ref": "#/properties/data/properties/releases/properties/breakingChanges/definitions/StreamBreakingChangeScope" + } + ] + }, + "StreamBreakingChangeScope": { + "description": "A scope that can be used to limit the impact of a breaking change to specific streams.", + "type": "object", + "additionalProperties": false, + "required": [ + "scopeType", + "impactedScopes" + ], + "properties": { + "scopeType": { + "type": "const", + "const": "stream" + }, + "impactedScopes": { + "description": "List of streams that are impacted by the breaking change.", + "type": "array", + "minItems": 1, + "items": { + "type": "string" + } + } + } + } + } + }, + "migrationDocumentationUrl": { + "description": "URL to documentation on how to migrate from the previous version to the current version. Defaults to ${documentationUrl}-migrations", + "type": "string", + "format": "uri" + } + } + }, + "normalizationConfig": { + "$schema": "http://json-schema.org/draft-07/schema#", + "$id": "https://github.com/airbytehq/airbyte/airbyte-ci/connectors/metadata_service/lib/metadata_service/models/src/NormalizationDestinationDefinitionConfig.yaml", + "title": "NormalizationDestinationDefinitionConfig", + "description": "describes a normalization config for destination definition", + "type": "object", + "required": [ + "normalizationRepository", + "normalizationTag", + "normalizationIntegrationType" + ], + "additionalProperties": true, + "properties": { + "normalizationRepository": { + "type": "string", + "description": "a field indicating the name of the repository to be used for normalization. If the value of the flag is NULL - normalization is not used." + }, + "normalizationTag": { + "type": "string", + "description": "a field indicating the tag of the docker repository to be used for normalization." + }, + "normalizationIntegrationType": { + "type": "string", + "description": "a field indicating the type of integration dialect to use for normalization." + } + } + }, + "suggestedStreams": { + "$schema": "http://json-schema.org/draft-07/schema#", + "$id": "https://github.com/airbytehq/airbyte/airbyte-ci/connectors/metadata_service/lib/metadata_service/models/src/SuggestedStreams.yaml", + "title": "SuggestedStreams", + "description": "A source's suggested streams. These will be suggested by default for new connections using this source. Otherwise, all streams will be selected. This is useful for when your source has a lot of streams, but the average user will only want a subset of them synced.", + "type": "object", + "additionalProperties": true, + "properties": { + "streams": { + "type": "array", + "description": "An array of streams that this connector suggests the average user will want. SuggestedStreams not being present for the source means that all streams are suggested. An empty list here means that no streams are suggested.", + "items": { + "type": "string" + } + } + } + }, + "resourceRequirements": { + "$schema": "http://json-schema.org/draft-07/schema#", + "$id": "https://github.com/airbytehq/airbyte/airbyte-ci/connectors/metadata_service/lib/metadata_service/models/src/ActorDefinitionResourceRequirements.yaml", + "title": "ActorDefinitionResourceRequirements", + "description": "actor definition specific resource requirements", + "type": "object", + "additionalProperties": false, + "properties": { + "default": { + "description": "if set, these are the requirements that should be set for ALL jobs run for this actor definition.", + "$ref": "#/properties/data/properties/resourceRequirements/definitions/JobTypeResourceLimit/properties/resourceRequirements" + }, + "jobSpecific": { + "type": "array", + "items": { + "$ref": "#/properties/data/properties/resourceRequirements/definitions/JobTypeResourceLimit" + } + } + }, + "definitions": { + "JobTypeResourceLimit": { + "description": "sets resource requirements for a specific job type for an actor definition. these values override the default, if both are set.", + "type": "object", + "additionalProperties": false, + "required": [ + "jobType", + "resourceRequirements" + ], + "properties": { + "jobType": { + "$schema": "http://json-schema.org/draft-07/schema#", + "$id": "https://github.com/airbytehq/airbyte/airbyte-ci/connectors/metadata_service/lib/metadata_service/models/src/JobType.yaml", + "title": "JobType", + "description": "enum that describes the different types of jobs that the platform runs.", + "type": "string", + "enum": [ + "get_spec", + "check_connection", + "discover_schema", + "sync", + "reset_connection", + "connection_updater", + "replicate" + ] + }, + "resourceRequirements": { + "$schema": "http://json-schema.org/draft-07/schema#", + "$id": "https://github.com/airbytehq/airbyte/airbyte-ci/connectors/metadata_service/lib/metadata_service/models/src/ResourceRequirements.yaml", + "title": "ResourceRequirements", + "description": "generic configuration for pod source requirements", + "type": "object", + "additionalProperties": false, + "properties": { + "cpu_request": { + "type": "string" + }, + "cpu_limit": { + "type": "string" + }, + "memory_request": { + "type": "string" + }, + "memory_limit": { + "type": "string" + } + } + } + } + } + } + }, + "ab_internal": { + "$schema": "http://json-schema.org/draft-07/schema#", + "$id": "https://github.com/airbytehq/airbyte/airbyte-ci/connectors_ci/metadata_service/lib/models/src/AirbyteInternal.yml", + "title": "AirbyteInternal", + "description": "Fields for internal use only", + "type": "object", + "additionalProperties": true, + "properties": { + "sl": { + "type": "integer", + "enum": [ + 0, + 100, + 200, + 300 + ] + }, + "ql": { + "type": "integer", + "enum": [ + 0, + 100, + 200, + 300, + 400, + 500, + 600 + ] + }, + "isEnterprise": { + "type": "boolean", + "default": false + }, + "requireVersionIncrementsInPullRequests": { + "type": "boolean", + "default": true, + "description": "When false, version increment checks will be skipped for this connector" + } + } + }, + "remoteRegistries": { + "$schema": "http://json-schema.org/draft-07/schema#", + "$id": "https://github.com/airbytehq/airbyte/airbyte-ci/connectors_ci/metadata_service/lib/models/src/RemoteRegistries.yml", + "title": "RemoteRegistries", + "description": "describes how the connector is published to remote registries", + "type": "object", + "additionalProperties": false, + "properties": { + "pypi": { + "$ref": "#/properties/data/properties/remoteRegistries/definitions/PyPi" + } + }, + "definitions": { + "PyPi": { + "title": "PyPi", + "description": "describes the PyPi publishing options", + "type": "object", + "additionalProperties": false, + "required": [ + "enabled", + "packageName" + ], + "properties": { + "enabled": { + "type": "boolean" + }, + "packageName": { + "type": "string", + "description": "The name of the package on PyPi." + } + } + } + } + }, + "supportsRefreshes": { + "type": "boolean", + "default": false + }, + "generated": { + "$schema": "http://json-schema.org/draft-07/schema#", + "$id": "https://github.com/airbytehq/airbyte/airbyte-ci/connectors_ci/metadata_service/lib/models/src/GeneratedFields.yaml", + "title": "GeneratedFields", + "description": "Optional schema for fields generated at metadata upload time", + "type": "object", + "properties": { + "git": { + "$schema": "http://json-schema.org/draft-07/schema#", + "$id": "https://github.com/airbytehq/airbyte/airbyte-ci/connectors/metadata_service/lib/metadata_service/models/src/GitInfo.yaml", + "title": "GitInfo", + "description": "Information about the author of the last commit that modified this file. DO NOT DEFINE THIS FIELD MANUALLY. It will be overwritten by the CI.", + "type": "object", + "additionalProperties": false, + "properties": { + "commit_sha": { + "type": "string", + "description": "The git commit sha of the last commit that modified this file." + }, + "commit_timestamp": { + "type": "string", + "format": "date-time", + "description": "The git commit timestamp of the last commit that modified this file." + }, + "commit_author": { + "type": "string", + "description": "The git commit author of the last commit that modified this file." + }, + "commit_author_email": { + "type": "string", + "description": "The git commit author email of the last commit that modified this file." + } + } + }, + "source_file_info": { + "$schema": "http://json-schema.org/draft-07/schema#", + "$id": "https://github.com/airbytehq/airbyte/airbyte-ci/connectors_ci/metadata_service/lib/models/src/SourceFileInfo.yaml", + "title": "SourceFileInfo", + "description": "Information about the source file that generated the registry entry", + "type": "object", + "properties": { + "metadata_etag": { + "type": "string" + }, + "metadata_file_path": { + "type": "string" + }, + "metadata_bucket_name": { + "type": "string" + }, + "metadata_last_modified": { + "type": "string" + }, + "registry_entry_generated_at": { + "type": "string" + } + } + }, + "metrics": { + "$schema": "http://json-schema.org/draft-07/schema#", + "$id": "https://github.com/airbytehq/airbyte/airbyte-ci/connectors_ci/metadata_service/lib/models/src/ConnectorMetrics.yaml", + "title": "ConnectorMetrics", + "description": "Information about the source file that generated the registry entry", + "type": "object", + "properties": { + "all": { + "type": "ConnectorMetric" + }, + "cloud": { + "type": "ConnectorMetric" + }, + "oss": { + "type": "ConnectorMetric" + } + }, + "definitions": { + "ConnectorMetric": { + "type": "object", + "properties": { + "usage": { + "oneOf": [ + { + "type": "string" + }, + { + "type": "string", + "enum": [ + "low", + "medium", + "high" + ] + } + ] + }, + "sync_success_rate": { + "oneOf": [ + { + "type": "string" + }, + { + "type": "string", + "enum": [ + "low", + "medium", + "high" + ] + } + ] + }, + "connector_version": { + "type": "string" + } + }, + "additionalProperties": true + } + } + }, + "sbomUrl": { + "type": "string", + "description": "URL to the SBOM file" + } + } + }, + "supportsFileTransfer": { + "type": "boolean", + "default": false + }, + "supportsDataActivation": { + "type": "boolean", + "default": false + }, + "connectorIPCOptions": { + "$schema": "http://json-schema.org/draft-07/schema#", + "$id": "https://github.com/airbytehq/airbyte/airbyte-ci/connectors/metadata_service/lib/metadata_service/models/src/ConnectorIPCOptions.yaml", + "title": "ConnectorIPCOptions", + "type": "object", + "required": [ + "dataChannel" + ], + "additionalProperties": false, + "properties": { + "dataChannel": { + "type": "object", + "required": [ + "version", + "supportedSerialization", + "supportedTransport" + ], + "additionalProperties": false, + "properties": { + "version": { + "type": "string" + }, + "supportedSerialization": { + "type": "array", + "items": { + "type": "string", + "enum": [ + "JSONL", + "PROTOBUF", + "FLATBUFFERS" + ] + } + }, + "supportedTransport": { + "type": "array", + "items": { + "type": "string", + "enum": [ + "STDIO", + "SOCKET" + ] + } + } + } + } + } + } + } + } + } +} \ No newline at end of file diff --git a/airbyte-ci/connectors/metadata_service/lib/package-lock.json b/airbyte-ci/connectors/metadata_service/lib/package-lock.json new file mode 100644 index 00000000000..aab3f9ca2da --- /dev/null +++ b/airbyte-ci/connectors/metadata_service/lib/package-lock.json @@ -0,0 +1,62 @@ +{ + "name": "metadata-service-schema-tools", + "version": "1.0.0", + "lockfileVersion": 3, + "requires": true, + "packages": { + "": { + "name": "metadata-service-schema-tools", + "version": "1.0.0", + "dependencies": { + "@apidevtools/json-schema-ref-parser": "^11.0.0" + } + }, + "node_modules/@apidevtools/json-schema-ref-parser": { + "version": "11.9.3", + "resolved": "https://registry.npmjs.org/@apidevtools/json-schema-ref-parser/-/json-schema-ref-parser-11.9.3.tgz", + "integrity": "sha512-60vepv88RwcJtSHrD6MjIL6Ta3SOYbgfnkHb+ppAVK+o9mXprRtulx7VlRl3lN3bbvysAfCS7WMVfhUYemB0IQ==", + "license": "MIT", + "dependencies": { + "@jsdevtools/ono": "^7.1.3", + "@types/json-schema": "^7.0.15", + "js-yaml": "^4.1.0" + }, + "engines": { + "node": ">= 16" + }, + "funding": { + "url": "https://github.com/sponsors/philsturgeon" + } + }, + "node_modules/@jsdevtools/ono": { + "version": "7.1.3", + "resolved": "https://registry.npmjs.org/@jsdevtools/ono/-/ono-7.1.3.tgz", + "integrity": "sha512-4JQNk+3mVzK3xh2rqd6RB4J46qUR19azEHBneZyTZM+c456qOrbbM/5xcR8huNCCcbVt7+UmizG6GuUvPvKUYg==", + "license": "MIT" + }, + "node_modules/@types/json-schema": { + "version": "7.0.15", + "resolved": "https://registry.npmjs.org/@types/json-schema/-/json-schema-7.0.15.tgz", + "integrity": "sha512-5+fP8P8MFNC+AyZCDxrB2pkZFPGzqQWUzpSeuuVLvm8VMcorNYavBqoFcxK8bQz4Qsbn4oUEEem4wDLfcysGHA==", + "license": "MIT" + }, + "node_modules/argparse": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/argparse/-/argparse-2.0.1.tgz", + "integrity": "sha512-8+9WqebbFzpX9OR+Wa6O29asIogeRMzcGtAINdpMHHyAg10f05aSFVBbcEqGf/PXw1EjAZ+q2/bEBg3DvurK3Q==", + "license": "Python-2.0" + }, + "node_modules/js-yaml": { + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/js-yaml/-/js-yaml-4.1.0.tgz", + "integrity": "sha512-wpxZs9NoxZaJESJGIZTyDEaYpl0FKSA+FB9aJiyemKhMwkxQg63h4T1KJgUGHpTqPDNRcmmYLugrRjJlBtWvRA==", + "license": "MIT", + "dependencies": { + "argparse": "^2.0.1" + }, + "bin": { + "js-yaml": "bin/js-yaml.js" + } + } + } +} diff --git a/airbyte-ci/connectors/metadata_service/lib/package.json b/airbyte-ci/connectors/metadata_service/lib/package.json new file mode 100644 index 00000000000..af159ca7c3c --- /dev/null +++ b/airbyte-ci/connectors/metadata_service/lib/package.json @@ -0,0 +1,12 @@ +{ + "name": "metadata-service-schema-tools", + "version": "1.0.0", + "description": "Schema bundling tools for Airbyte metadata service", + "private": true, + "scripts": { + "bundle-schemas": "node bin/bundle-schemas.js" + }, + "dependencies": { + "@apidevtools/json-schema-ref-parser": "^11.0.0" + } +} diff --git a/airbyte-ci/connectors/metadata_service/lib/pyproject.toml b/airbyte-ci/connectors/metadata_service/lib/pyproject.toml index e793096c393..90768fa9457 100644 --- a/airbyte-ci/connectors/metadata_service/lib/pyproject.toml +++ b/airbyte-ci/connectors/metadata_service/lib/pyproject.toml @@ -36,7 +36,7 @@ poethepoet = "^0.20.0" metadata_service = "metadata_service.commands:metadata_service" [tool.poe.tasks] -generate-models = { shell = "./bin/generate-python-classes.sh" } +generate-models = { shell = "npm ci --silent && ./bin/generate-metadata-models.sh" } replicate-prod = "gsutil -m rsync -r -d gs://prod-airbyte-cloud-connector-metadata-service gs://$TARGET_BUCKET" copy-connector-from-prod = "gsutil -m rsync -r -d gs://prod-airbyte-cloud-connector-metadata-service/metadata/$CONNECTOR/$VERSION gs://$TARGET_BUCKET/metadata/$CONNECTOR/$VERSION" promote-connector-to-latest = "gsutil -m rsync -r -d gs://$TARGET_BUCKET/metadata/$CONNECTOR/$VERSION gs://$TARGET_BUCKET/metadata/$CONNECTOR/latest"