jprdonnelly/airbyte
1
0
Fork 0
mirror of synced 2025-12-19 10:00:34 -05:00
Code Issues Packages Projects Releases Wiki Activity

feat(metadata-service): Add JSON schema bundling for IDE validation and CDK models (#68672)

Browse Source 
Co-authored-by: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>
This commit is contained in:
Aaron ("AJ") Steers
2025-10-28 10:26:58 -07:00
committed by GitHub
parent a8eb2855ec
commit 99658b9a8b
8 changed files with 996 additions and 2 deletions
Download Patch File Download Diff File Expand all files Collapse all files

3
.gitignore vendored
View File

@@ -59,6 +59,9 @@ coverage.xml
.pytest_cache/
cover/
# Node.js
node_modules/
# dbt
profiles.yml

10
airbyte-ci/connectors/metadata_service/lib/README.md
View File

@@ -10,6 +10,16 @@ To use this submodule, it is recommended that you use Poetry to manage dependenc
poetry install
```
### Node.js Requirement
The model generation process also requires Node.js to bundle JSON schemas. Install Node.js:
- On macOS: `brew install node`
- On Ubuntu/Debian: `sudo apt-get install nodejs npm`
- On other systems: https://nodejs.org/
Node.js dependencies will be automatically installed when running `poetry run poe generate-models`.
## Generating Models
This submodule includes a tool for generating Python models from JSON Schema specifications. To generate the models, we use the library [datamodel-code-generator](https://github.com/koxudaxi/datamodel-code-generator). The generated models are stored in `models/generated`.

48
airbyte-ci/connectors/metadata_service/lib/bin/bundle-schemas.js Executable file
View File

@@ -0,0 +1,48 @@
#!/usr/bin/env node
/**
* Bundle JSON schemas using @apidevtools/json-schema-ref-parser
* This script resolves all $ref references in the schema files and creates a single bundled schema.
*/
const $RefParser = require('@apidevtools/json-schema-ref-parser');
const fs = require('fs');
const path = require('path');
const YAML_DIR = 'metadata_service/models/src';
const OUTPUT_DIR = 'metadata_service/models/generated';
const ENTRY_SCHEMA = path.join(YAML_DIR, 'ConnectorMetadataDefinitionV0.yaml');
const BUNDLE_OUTPUT = path.join(OUTPUT_DIR, 'ConnectorMetadataDefinitionV0.json');
async function bundleSchemas() {
try {
console.log('📦 Bundling JSON schemas...');
console.log(` Entry schema: ${ENTRY_SCHEMA}`);
console.log(` Output: ${BUNDLE_OUTPUT}`);
if (!fs.existsSync(YAML_DIR)) {
console.error(`❌ Error: The yaml directory does not exist: ${YAML_DIR}`);
process.exit(1);
}
if (!fs.existsSync(OUTPUT_DIR)) {
fs.mkdirSync(OUTPUT_DIR, { recursive: true });
}
const schema = await $RefParser.bundle(ENTRY_SCHEMA, {
dereference: {
circular: 'ignore' // Handle circular references gracefully
}
});
fs.writeFileSync(BUNDLE_OUTPUT, JSON.stringify(schema, null, 2));
console.log('✅ Successfully bundled schema to', BUNDLE_OUTPUT);
console.log(' This bundled schema can be used for IDE validation and other tools.');
} catch (error) {
console.error('❌ Error bundling schemas:', error.message);
process.exit(1);
}
}
bundleSchemas();

6
airbyte-ci/connectors/metadata_service/lib/bin/generate-python-classes.sh → airbyte-ci/connectors/metadata_service/lib/bin/generate-metadata-models.sh
View File

@@ -16,7 +16,7 @@ fi
rm -rf "$OUTPUT_DIR"/*.py
mkdir -p "$OUTPUT_DIR"
echo "# generated by generate-python-classes" > "$OUTPUT_DIR"/__init__.py
echo "# generated by generate-metadata-models" > "$OUTPUT_DIR"/__init__.py
for f in "$YAML_DIR"/*.yaml; do
filename_wo_ext=$(basename "$f" | cut -d . -f 1)
@@ -30,3 +30,7 @@ for f in "$YAML_DIR"/*.yaml; do
--enum-field-as-literal all \
--disable-timestamp
done
echo ""
echo "Generating bundled JSON schema..."
node bin/bundle-schemas.js

855
airbyte-ci/connectors/metadata_service/lib/metadata_service/models/generated/ConnectorMetadataDefinitionV0.json Normal file
View File

@@ -0,0 +1,855 @@
{
"$schema": "http://json-schema.org/draft-07/schema#",
"$id": "https://github.com/airbytehq/airbyte/airbyte-ci/connectors_ci/metadata_service/lib/models/src/ConnectorMetadataDefinitionV0.yml",
"title": "ConnectorMetadataDefinitionV0",
"description": "describes the metadata of a connector",
"type": "object",
"required": [
"metadataSpecVersion",
"data"
],
"additionalProperties": false,
"properties": {
"metadataSpecVersion": {
"type": "string"
},
"data": {
"type": "object",
"required": [
"name",
"definitionId",
"connectorType",
"dockerRepository",
"dockerImageTag",
"license",
"documentationUrl",
"githubIssueLabel",
"connectorSubtype",
"releaseStage"
],
"additionalProperties": false,
"properties": {
"name": {
"type": "string"
},
"icon": {
"type": "string"
},
"definitionId": {
"type": "string",
"format": "uuid"
},
"connectorBuildOptions": {
"$schema": "http://json-schema.org/draft-07/schema#",
"$id": "https://github.com/airbytehq/airbyte/airbyte-ci/connectors/metadata_service/lib/metadata_service/models/src/ConnectorBuildOptions.yaml",
"title": "ConnectorBuildOptions",
"description": "metadata specific to the build process.",
"type": "object",
"additionalProperties": false,
"properties": {
"baseImage": {
"type": "string"
}
}
},
"connectorTestSuitesOptions": {
"type": "array",
"items": {
"$schema": "http://json-schema.org/draft-07/schema#",
"$id": "https://github.com/airbytehq/airbyte/airbyte-ci/connectors/metadata_service/lib/metadata_service/models/src/ConnectorTestOptions.yaml",
"title": "ConnectorTestSuiteOptions",
"description": "Options for a specific connector test suite.",
"type": "object",
"required": [
"suite"
],
"additionalProperties": false,
"properties": {
"suite": {
"description": "Name of the configured test suite",
"type": "string",
"enum": [
"unitTests",
"integrationTests",
"acceptanceTests",
"liveTests"
]
},
"testSecrets": {
"description": "List of secrets required to run the test suite",
"type": "array",
"items": {
"$schema": "http://json-schema.org/draft-07/schema#",
"$id": "https://github.com/airbytehq/airbyte/airbyte-ci/connectors/metadata_service/lib/metadata_service/models/src/TestSecret.yaml",
"title": "Secret",
"description": "An object describing a secret's metadata",
"type": "object",
"required": [
"name",
"secretStore"
],
"additionalProperties": false,
"properties": {
"name": {
"type": "string",
"description": "The secret name in the secret store"
},
"fileName": {
"type": "string",
"description": "The name of the file to which the secret value would be persisted"
},
"secretStore": {
"$schema": "http://json-schema.org/draft-07/schema#",
"$id": "https://github.com/airbytehq/airbyte/airbyte-ci/connectors/metadata_service/lib/metadata_service/models/src/TestSecret.yaml",
"title": "SecretStore",
"description": "An object describing a secret store metadata",
"type": "object",
"required": [
"name",
"secretStore"
],
"additionalProperties": false,
"properties": {
"alias": {
"type": "string",
"description": "The alias of the secret store which can map to its actual secret address"
},
"type": {
"type": "string",
"description": "The type of the secret store",
"enum": [
"GSM"
]
}
}
}
}
}
},
"testConnections": {
"description": "List of sandbox cloud connections that tests can be run against",
"type": "array",
"items": {
"$schema": "http://json-schema.org/draft-07/schema#",
"$id": "https://github.com/airbytehq/airbyte/airbyte-ci/connectors/metadata_service/lib/metadata_service/models/src/TestConnections.yaml",
"title": "TestConnections",
"description": "List of sandbox cloud connections that tests can be run against",
"type": "object",
"required": [
"name",
"id"
],
"additionalProperties": false,
"properties": {
"name": {
"type": "string",
"description": "The connection name"
},
"id": {
"type": "string",
"description": "The connection ID"
}
}
}
}
}
}
},
"connectorType": {
"type": "string",
"enum": [
"destination",
"source"
]
},
"dockerRepository": {
"type": "string"
},
"dockerImageTag": {
"type": "string"
},
"supportsDbt": {
"type": "boolean"
},
"supportsNormalization": {
"type": "boolean"
},
"license": {
"type": "string"
},
"documentationUrl": {
"type": "string",
"format": "uri"
},
"githubIssueLabel": {
"type": "string"
},
"maxSecondsBetweenMessages": {
"description": "Maximum delay between 2 airbyte protocol messages, in second. The source will timeout if this delay is reached",
"type": "integer"
},
"releaseDate": {
"description": "The date when this connector was first released, in yyyy-mm-dd format.",
"type": "string",
"format": "date"
},
"protocolVersion": {
"type": "string",
"description": "the Airbyte Protocol version supported by the connector"
},
"erdUrl": {
"type": "string",
"description": "The URL where you can visualize the ERD"
},
"connectorSubtype": {
"type": "string",
"enum": [
"api",
"database",
"datalake",
"file",
"custom",
"message_queue",
"unknown",
"vectorstore"
]
},
"releaseStage": {
"$schema": "http://json-schema.org/draft-07/schema#",
"$id": "https://github.com/airbytehq/airbyte-platform/blob/main/airbyte-config/config-models/src/main/resources/types/ReleaseStage.yaml",
"title": "ReleaseStage",
"description": "enum that describes a connector's release stage",
"type": "string",
"enum": [
"alpha",
"beta",
"generally_available",
"custom"
]
},
"supportLevel": {
"$schema": "http://json-schema.org/draft-07/schema#",
"$id": "https://github.com/airbytehq/airbyte/airbyte-ci/connectors_ci/metadata_service/lib/models/src/SupportLevel.yaml",
"title": "SupportLevel",
"description": "enum that describes a connector's release stage",
"type": "string",
"enum": [
"community",
"certified",
"archived"
]
},
"tags": {
"type": "array",
"description": "An array of tags that describe the connector. E.g: language:python, keyword:rds, etc.",
"items": {
"type": "string"
},
"default": []
},
"registryOverrides": {
"anyOf": [
{
"type": "object",
"additionalProperties": false,
"properties": {
"oss": {
"anyOf": [
{
"$schema": "http://json-schema.org/draft-07/schema#",
"$id": "https://github.com/airbytehq/airbyte/airbyte-ci/connectors_ci/metadata_service/lib/models/src/RegistryOverrides.yml",
"title": "RegistryOverrides",
"description": "describes the overrides per registry of a connector",
"type": "object",
"additionalProperties": false,
"required": [
"enabled"
],
"properties": {
"enabled": {
"type": "boolean",
"default": false
},
"name": {
"type": "string"
},
"dockerRepository": {
"type": "string"
},
"dockerImageTag": {
"type": "string"
},
"supportsDbt": {
"type": "boolean"
},
"supportsNormalization": {
"type": "boolean"
},
"license": {
"type": "string"
},
"documentationUrl": {
"type": "string",
"format": "uri"
},
"connectorSubtype": {
"type": "string"
},
"allowedHosts": {
"$ref": "#/properties/data/properties/allowedHosts"
},
"normalizationConfig": {
"$ref": "#/properties/data/properties/normalizationConfig"
},
"suggestedStreams": {
"$ref": "#/properties/data/properties/suggestedStreams"
},
"resourceRequirements": {
"$ref": "#/properties/data/properties/resourceRequirements"
}
}
}
]
},
"cloud": {
"anyOf": [
{
"$ref": "#/properties/data/properties/registryOverrides/anyOf/0/properties/oss/anyOf/0"
}
]
}
}
}
]
},
"allowedHosts": {
"$schema": "http://json-schema.org/draft-07/schema#",
"$id": "https://github.com/airbytehq/airbyte/airbyte-ci/connectors/metadata_service/lib/metadata_service/models/src/AllowedHosts.yaml",
"title": "AllowedHosts",
"description": "A connector's allowed hosts. If present, the platform will limit communication to only hosts which are listed in `AllowedHosts.hosts`.",
"type": "object",
"additionalProperties": true,
"properties": {
"hosts": {
"type": "array",
"description": "An array of hosts that this connector can connect to. AllowedHosts not being present for the source or destination means that access to all hosts is allowed. An empty list here means that no network access is granted.",
"items": {
"type": "string"
}
}
}
},
"releases": {
"$schema": "http://json-schema.org/draft-07/schema#",
"$id": "https://github.com/airbytehq/airbyte/airbyte-ci/connectors/metadata_service/lib/metadata_service/models/src/ConnectorReleases.yaml",
"title": "ConnectorReleases",
"description": "Contains information about different types of releases for a connector.",
"type": "object",
"additionalProperties": false,
"properties": {
"rolloutConfiguration": {
"$schema": "http://json-schema.org/draft-07/schema#",
"$id": "https://github.com/airbytehq/airbyte/airbyte-ci/connectors/metadata_service/lib/metadata_service/models/src/RolloutConfiguration.yaml",
"title": "RolloutConfiguration",
"description": "configuration for the rollout of a connector",
"type": "object",
"additionalProperties": false,
"properties": {
"enableProgressiveRollout": {
"type": "boolean",
"default": false,
"description": "Whether to enable progressive rollout for the connector."
},
"initialPercentage": {
"type": "integer",
"minimum": 0,
"maximum": 100,
"default": 0,
"description": "The percentage of users that should receive the new version initially."
},
"maxPercentage": {
"type": "integer",
"minimum": 0,
"maximum": 100,
"default": 50,
"description": "The percentage of users who should receive the release candidate during the test phase before full rollout."
},
"advanceDelayMinutes": {
"type": "integer",
"minimum": 10,
"default": 10,
"description": "The number of minutes to wait before advancing the rollout percentage."
}
}
},
"breakingChanges": {
"$schema": "http://json-schema.org/draft-07/schema#",
"$id": "https://github.com/airbytehq/airbyte/airbyte-ci/connectors/metadata_service/lib/metadata_service/models/src/ConnectorBreakingChanges.yaml",
"title": "ConnectorBreakingChanges",
"description": "Each entry denotes a breaking change in a specific version of a connector that requires user action to upgrade.",
"type": "object",
"additionalProperties": false,
"minProperties": 1,
"patternProperties": {
"^\\d+\\.\\d+\\.\\d+$": {
"$ref": "#/properties/data/properties/releases/properties/breakingChanges/definitions/VersionBreakingChange"
}
},
"definitions": {
"VersionBreakingChange": {
"description": "Contains information about a breaking change, including the deadline to upgrade and a message detailing the change.",
"type": "object",
"additionalProperties": false,
"required": [
"upgradeDeadline",
"message"
],
"properties": {
"upgradeDeadline": {
"description": "The deadline by which to upgrade before the breaking change takes effect.",
"type": "string",
"format": "date"
},
"message": {
"description": "Descriptive message detailing the breaking change.",
"type": "string"
},
"deadlineAction": {
"description": "Action to do when the deadline is reached.",
"type": "string",
"enum": [
"auto_upgrade",
"disable"
]
},
"migrationDocumentationUrl": {
"description": "URL to documentation on how to migrate to the current version. Defaults to ${documentationUrl}-migrations#${version}",
"type": "string",
"format": "uri"
},
"scopedImpact": {
"description": "List of scopes that are impacted by the breaking change. If not specified, the breaking change cannot be scoped to reduce impact via the supported scope types.",
"type": "array",
"minItems": 1,
"items": {
"$ref": "#/properties/data/properties/releases/properties/breakingChanges/definitions/BreakingChangeScope"
}
}
}
},
"BreakingChangeScope": {
"description": "A scope that can be used to limit the impact of a breaking change.",
"type": "object",
"oneOf": [
{
"$ref": "#/properties/data/properties/releases/properties/breakingChanges/definitions/StreamBreakingChangeScope"
}
]
},
"StreamBreakingChangeScope": {
"description": "A scope that can be used to limit the impact of a breaking change to specific streams.",
"type": "object",
"additionalProperties": false,
"required": [
"scopeType",
"impactedScopes"
],
"properties": {
"scopeType": {
"type": "const",
"const": "stream"
},
"impactedScopes": {
"description": "List of streams that are impacted by the breaking change.",
"type": "array",
"minItems": 1,
"items": {
"type": "string"
}
}
}
}
}
},
"migrationDocumentationUrl": {
"description": "URL to documentation on how to migrate from the previous version to the current version. Defaults to ${documentationUrl}-migrations",
"type": "string",
"format": "uri"
}
}
},
"normalizationConfig": {
"$schema": "http://json-schema.org/draft-07/schema#",
"$id": "https://github.com/airbytehq/airbyte/airbyte-ci/connectors/metadata_service/lib/metadata_service/models/src/NormalizationDestinationDefinitionConfig.yaml",
"title": "NormalizationDestinationDefinitionConfig",
"description": "describes a normalization config for destination definition",
"type": "object",
"required": [
"normalizationRepository",
"normalizationTag",
"normalizationIntegrationType"
],
"additionalProperties": true,
"properties": {
"normalizationRepository": {
"type": "string",
"description": "a field indicating the name of the repository to be used for normalization. If the value of the flag is NULL - normalization is not used."
},
"normalizationTag": {
"type": "string",
"description": "a field indicating the tag of the docker repository to be used for normalization."
},
"normalizationIntegrationType": {
"type": "string",
"description": "a field indicating the type of integration dialect to use for normalization."
}
}
},
"suggestedStreams": {
"$schema": "http://json-schema.org/draft-07/schema#",
"$id": "https://github.com/airbytehq/airbyte/airbyte-ci/connectors/metadata_service/lib/metadata_service/models/src/SuggestedStreams.yaml",
"title": "SuggestedStreams",
"description": "A source's suggested streams. These will be suggested by default for new connections using this source. Otherwise, all streams will be selected. This is useful for when your source has a lot of streams, but the average user will only want a subset of them synced.",
"type": "object",
"additionalProperties": true,
"properties": {
"streams": {
"type": "array",
"description": "An array of streams that this connector suggests the average user will want. SuggestedStreams not being present for the source means that all streams are suggested. An empty list here means that no streams are suggested.",
"items": {
"type": "string"
}
}
}
},
"resourceRequirements": {
"$schema": "http://json-schema.org/draft-07/schema#",
"$id": "https://github.com/airbytehq/airbyte/airbyte-ci/connectors/metadata_service/lib/metadata_service/models/src/ActorDefinitionResourceRequirements.yaml",
"title": "ActorDefinitionResourceRequirements",
"description": "actor definition specific resource requirements",
"type": "object",
"additionalProperties": false,
"properties": {
"default": {
"description": "if set, these are the requirements that should be set for ALL jobs run for this actor definition.",
"$ref": "#/properties/data/properties/resourceRequirements/definitions/JobTypeResourceLimit/properties/resourceRequirements"
},
"jobSpecific": {
"type": "array",
"items": {
"$ref": "#/properties/data/properties/resourceRequirements/definitions/JobTypeResourceLimit"
}
}
},
"definitions": {
"JobTypeResourceLimit": {
"description": "sets resource requirements for a specific job type for an actor definition. these values override the default, if both are set.",
"type": "object",
"additionalProperties": false,
"required": [
"jobType",
"resourceRequirements"
],
"properties": {
"jobType": {
"$schema": "http://json-schema.org/draft-07/schema#",
"$id": "https://github.com/airbytehq/airbyte/airbyte-ci/connectors/metadata_service/lib/metadata_service/models/src/JobType.yaml",
"title": "JobType",
"description": "enum that describes the different types of jobs that the platform runs.",
"type": "string",
"enum": [
"get_spec",
"check_connection",
"discover_schema",
"sync",
"reset_connection",
"connection_updater",
"replicate"
]
},
"resourceRequirements": {
"$schema": "http://json-schema.org/draft-07/schema#",
"$id": "https://github.com/airbytehq/airbyte/airbyte-ci/connectors/metadata_service/lib/metadata_service/models/src/ResourceRequirements.yaml",
"title": "ResourceRequirements",
"description": "generic configuration for pod source requirements",
"type": "object",
"additionalProperties": false,
"properties": {
"cpu_request": {
"type": "string"
},
"cpu_limit": {
"type": "string"
},
"memory_request": {
"type": "string"
},
"memory_limit": {
"type": "string"
}
}
}
}
}
}
},
"ab_internal": {
"$schema": "http://json-schema.org/draft-07/schema#",
"$id": "https://github.com/airbytehq/airbyte/airbyte-ci/connectors_ci/metadata_service/lib/models/src/AirbyteInternal.yml",
"title": "AirbyteInternal",
"description": "Fields for internal use only",
"type": "object",
"additionalProperties": true,
"properties": {
"sl": {
"type": "integer",
"enum": [
0,
100,
200,
300
]
},
"ql": {
"type": "integer",
"enum": [
0,
100,
200,
300,
400,
500,
600
]
},
"isEnterprise": {
"type": "boolean",
"default": false
},
"requireVersionIncrementsInPullRequests": {
"type": "boolean",
"default": true,
"description": "When false, version increment checks will be skipped for this connector"
}
}
},
"remoteRegistries": {
"$schema": "http://json-schema.org/draft-07/schema#",
"$id": "https://github.com/airbytehq/airbyte/airbyte-ci/connectors_ci/metadata_service/lib/models/src/RemoteRegistries.yml",
"title": "RemoteRegistries",
"description": "describes how the connector is published to remote registries",
"type": "object",
"additionalProperties": false,
"properties": {
"pypi": {
"$ref": "#/properties/data/properties/remoteRegistries/definitions/PyPi"
}
},
"definitions": {
"PyPi": {
"title": "PyPi",
"description": "describes the PyPi publishing options",
"type": "object",
"additionalProperties": false,
"required": [
"enabled",
"packageName"
],
"properties": {
"enabled": {
"type": "boolean"
},
"packageName": {
"type": "string",
"description": "The name of the package on PyPi."
}
}
}
}
},
"supportsRefreshes": {
"type": "boolean",
"default": false
},
"generated": {
"$schema": "http://json-schema.org/draft-07/schema#",
"$id": "https://github.com/airbytehq/airbyte/airbyte-ci/connectors_ci/metadata_service/lib/models/src/GeneratedFields.yaml",
"title": "GeneratedFields",
"description": "Optional schema for fields generated at metadata upload time",
"type": "object",
"properties": {
"git": {
"$schema": "http://json-schema.org/draft-07/schema#",
"$id": "https://github.com/airbytehq/airbyte/airbyte-ci/connectors/metadata_service/lib/metadata_service/models/src/GitInfo.yaml",
"title": "GitInfo",
"description": "Information about the author of the last commit that modified this file. DO NOT DEFINE THIS FIELD MANUALLY. It will be overwritten by the CI.",
"type": "object",
"additionalProperties": false,
"properties": {
"commit_sha": {
"type": "string",
"description": "The git commit sha of the last commit that modified this file."
},
"commit_timestamp": {
"type": "string",
"format": "date-time",
"description": "The git commit timestamp of the last commit that modified this file."
},
"commit_author": {
"type": "string",
"description": "The git commit author of the last commit that modified this file."
},
"commit_author_email": {
"type": "string",
"description": "The git commit author email of the last commit that modified this file."
}
}
},
"source_file_info": {
"$schema": "http://json-schema.org/draft-07/schema#",
"$id": "https://github.com/airbytehq/airbyte/airbyte-ci/connectors_ci/metadata_service/lib/models/src/SourceFileInfo.yaml",
"title": "SourceFileInfo",
"description": "Information about the source file that generated the registry entry",
"type": "object",
"properties": {
"metadata_etag": {
"type": "string"
},
"metadata_file_path": {
"type": "string"
},
"metadata_bucket_name": {
"type": "string"
},
"metadata_last_modified": {
"type": "string"
},
"registry_entry_generated_at": {
"type": "string"
}
}
},
"metrics": {
"$schema": "http://json-schema.org/draft-07/schema#",
"$id": "https://github.com/airbytehq/airbyte/airbyte-ci/connectors_ci/metadata_service/lib/models/src/ConnectorMetrics.yaml",
"title": "ConnectorMetrics",
"description": "Information about the source file that generated the registry entry",
"type": "object",
"properties": {
"all": {
"type": "ConnectorMetric"
},
"cloud": {
"type": "ConnectorMetric"
},
"oss": {
"type": "ConnectorMetric"
}
},
"definitions": {
"ConnectorMetric": {
"type": "object",
"properties": {
"usage": {
"oneOf": [
{
"type": "string"
},
{
"type": "string",
"enum": [
"low",
"medium",
"high"
]
}
]
},
"sync_success_rate": {
"oneOf": [
{
"type": "string"
},
{
"type": "string",
"enum": [
"low",
"medium",
"high"
]
}
]
},
"connector_version": {
"type": "string"
}
},
"additionalProperties": true
}
}
},
"sbomUrl": {
"type": "string",
"description": "URL to the SBOM file"
}
}
},
"supportsFileTransfer": {
"type": "boolean",
"default": false
},
"supportsDataActivation": {
"type": "boolean",
"default": false
},
"connectorIPCOptions": {
"$schema": "http://json-schema.org/draft-07/schema#",
"$id": "https://github.com/airbytehq/airbyte/airbyte-ci/connectors/metadata_service/lib/metadata_service/models/src/ConnectorIPCOptions.yaml",
"title": "ConnectorIPCOptions",
"type": "object",
"required": [
"dataChannel"
],
"additionalProperties": false,
"properties": {
"dataChannel": {
"type": "object",
"required": [
"version",
"supportedSerialization",
"supportedTransport"
],
"additionalProperties": false,
"properties": {
"version": {
"type": "string"
},
"supportedSerialization": {
"type": "array",
"items": {
"type": "string",
"enum": [
"JSONL",
"PROTOBUF",
"FLATBUFFERS"
]
}
},
"supportedTransport": {
"type": "array",
"items": {
"type": "string",
"enum": [
"STDIO",
"SOCKET"
]
}
}
}
}
}
}
}
}
}
}

62
airbyte-ci/connectors/metadata_service/lib/package-lock.json generated Normal file
View File

@@ -0,0 +1,62 @@
{
"name": "metadata-service-schema-tools",
"version": "1.0.0",
"lockfileVersion": 3,
"requires": true,
"packages": {
"": {
"name": "metadata-service-schema-tools",
"version": "1.0.0",
"dependencies": {
"@apidevtools/json-schema-ref-parser": "^11.0.0"
}
},
"node_modules/@apidevtools/json-schema-ref-parser": {
"version": "11.9.3",
"resolved": "https://registry.npmjs.org/@apidevtools/json-schema-ref-parser/-/json-schema-ref-parser-11.9.3.tgz",
"integrity": "sha512-60vepv88RwcJtSHrD6MjIL6Ta3SOYbgfnkHb+ppAVK+o9mXprRtulx7VlRl3lN3bbvysAfCS7WMVfhUYemB0IQ==",
"license": "MIT",
"dependencies": {
"@jsdevtools/ono": "^7.1.3",
"@types/json-schema": "^7.0.15",
"js-yaml": "^4.1.0"
},
"engines": {
"node": ">= 16"
},
"funding": {
"url": "https://github.com/sponsors/philsturgeon"
}
},
"node_modules/@jsdevtools/ono": {
"version": "7.1.3",
"resolved": "https://registry.npmjs.org/@jsdevtools/ono/-/ono-7.1.3.tgz",
"integrity": "sha512-4JQNk+3mVzK3xh2rqd6RB4J46qUR19azEHBneZyTZM+c456qOrbbM/5xcR8huNCCcbVt7+UmizG6GuUvPvKUYg==",
"license": "MIT"
},
"node_modules/@types/json-schema": {
"version": "7.0.15",
"resolved": "https://registry.npmjs.org/@types/json-schema/-/json-schema-7.0.15.tgz",
"integrity": "sha512-5+fP8P8MFNC+AyZCDxrB2pkZFPGzqQWUzpSeuuVLvm8VMcorNYavBqoFcxK8bQz4Qsbn4oUEEem4wDLfcysGHA==",
"license": "MIT"
},
"node_modules/argparse": {
"version": "2.0.1",
"resolved": "https://registry.npmjs.org/argparse/-/argparse-2.0.1.tgz",
"integrity": "sha512-8+9WqebbFzpX9OR+Wa6O29asIogeRMzcGtAINdpMHHyAg10f05aSFVBbcEqGf/PXw1EjAZ+q2/bEBg3DvurK3Q==",
"license": "Python-2.0"
},
"node_modules/js-yaml": {
"version": "4.1.0",
"resolved": "https://registry.npmjs.org/js-yaml/-/js-yaml-4.1.0.tgz",
"integrity": "sha512-wpxZs9NoxZaJESJGIZTyDEaYpl0FKSA+FB9aJiyemKhMwkxQg63h4T1KJgUGHpTqPDNRcmmYLugrRjJlBtWvRA==",
"license": "MIT",
"dependencies": {
"argparse": "^2.0.1"
},
"bin": {
"js-yaml": "bin/js-yaml.js"
}
}
}
}

12
airbyte-ci/connectors/metadata_service/lib/package.json Normal file
View File

@@ -0,0 +1,12 @@
{
"name": "metadata-service-schema-tools",
"version": "1.0.0",
"description": "Schema bundling tools for Airbyte metadata service",
"private": true,
"scripts": {
"bundle-schemas": "node bin/bundle-schemas.js"
},
"dependencies": {
"@apidevtools/json-schema-ref-parser": "^11.0.0"
}
}

2
airbyte-ci/connectors/metadata_service/lib/pyproject.toml
View File

@@ -36,7 +36,7 @@ poethepoet = "^0.20.0"
metadata_service = "metadata_service.commands:metadata_service"
[tool.poe.tasks]
generate-models = { shell = "./bin/generate-python-classes.sh" }
generate-models = { shell = "npm ci --silent && ./bin/generate-metadata-models.sh" }
replicate-prod = "gsutil -m rsync -r -d gs://prod-airbyte-cloud-connector-metadata-service gs://$TARGET_BUCKET"
copy-connector-from-prod = "gsutil -m rsync -r -d gs://prod-airbyte-cloud-connector-metadata-service/metadata/$CONNECTOR/$VERSION gs://$TARGET_BUCKET/metadata/$CONNECTOR/$VERSION"
promote-connector-to-latest = "gsutil -m rsync -r -d gs://$TARGET_BUCKET/metadata/$CONNECTOR/$VERSION gs://$TARGET_BUCKET/metadata/$CONNECTOR/latest"