1
0
mirror of synced 2025-12-19 10:00:34 -05:00
Files
airbyte/airbyte-integrations/connectors/source-youtube-data/manifest.yaml
2025-12-19 14:50:04 +02:00

978 lines
25 KiB
YAML

version: 6.4.0
type: DeclarativeSource
description: >-
The YouTube Data API v3 is an API that provides access to YouTube data, such as videos, playlists, channels, comments and simple stats.
This is a simpler version of Youtube connector, if you need more detailed reports from your channel please check
the [Youtube Analytics Connector](https://docs.airbyte.com/integrations/sources/youtube-analytics)
check:
type: CheckStream
stream_names:
- channels
definitions:
streams:
video:
type: DeclarativeStream
name: video
primary_key:
- videoId
retriever:
type: SimpleRetriever
requester:
$ref: "#/definitions/base_requester"
path: videos
http_method: GET
request_parameters:
part: snippet,contentDetails,statistics
record_selector:
type: RecordSelector
extractor:
type: DpathExtractor
field_path:
- items
- "*"
- snippet
paginator:
type: DefaultPaginator
page_token_option:
type: RequestOption
inject_into: request_parameter
field_name: pageToken
pagination_strategy:
type: CursorPagination
cursor_value: "{{ response.get('nextPageToken', '') }}"
stop_condition: "{{ response.get('nextPageToken') is none }}"
partition_router:
type: SubstreamPartitionRouter
parent_stream_configs:
- type: ParentStreamConfig
parent_key: videoId
request_option:
type: RequestOption
inject_into: request_parameter
field_name: id
partition_field: id
stream:
$ref: "#/definitions/streams/videos"
transformations:
- type: AddFields
fields:
- path:
- datetime
value: "{{ now_utc() }}"
- type: AddFields
fields:
- path:
- videoId
value: "{{ stream_slice.id }}"
schema_loader:
type: InlineSchemaLoader
schema:
$ref: "#/schemas/video"
channels:
type: DeclarativeStream
name: channels
primary_key:
- id
retriever:
type: SimpleRetriever
requester:
$ref: "#/definitions/base_requester"
path: channels
http_method: GET
request_parameters:
part: snippet,contentDetails,statistics
record_selector:
type: RecordSelector
extractor:
type: DpathExtractor
field_path:
- items
- "*"
- snippet
paginator:
type: DefaultPaginator
page_token_option:
type: RequestOption
inject_into: request_parameter
field_name: pageToken
pagination_strategy:
type: CursorPagination
cursor_value: "{{ response.get('nextPageToken', '') }}"
stop_condition: "{{ response.get('nextPageToken') is none }}"
partition_router:
type: ListPartitionRouter
values: "{{ config.channel_ids }}"
cursor_field: channel_id
request_option:
type: RequestOption
inject_into: request_parameter
field_name: id
transformations:
- type: AddFields
fields:
- path:
- id
value: "{{ stream_partition.channel_id }}"
schema_loader:
type: InlineSchemaLoader
schema:
$ref: "#/schemas/channels"
comments:
type: DeclarativeStream
name: comments
retriever:
type: SimpleRetriever
requester:
$ref: "#/definitions/base_requester"
path: commentThreads
http_method: GET
request_parameters:
part: snippet,replies
record_selector:
type: RecordSelector
extractor:
type: DpathExtractor
field_path:
- items
- "*"
- snippet
paginator:
type: DefaultPaginator
page_token_option:
type: RequestOption
inject_into: request_parameter
field_name: pageToken
pagination_strategy:
type: CursorPagination
cursor_value: "{{ response.get('nextPageToken', '') }}"
stop_condition: "{{ response.get('nextPageToken') is none }}"
partition_router:
type: SubstreamPartitionRouter
parent_stream_configs:
- type: ParentStreamConfig
parent_key: videoId
request_option:
type: RequestOption
inject_into: request_parameter
field_name: videoId
partition_field: id
stream:
$ref: "#/definitions/streams/videos"
schema_loader:
type: InlineSchemaLoader
schema:
$ref: "#/schemas/comments"
videos:
type: DeclarativeStream
name: videos
retriever:
type: SimpleRetriever
requester:
$ref: "#/definitions/base_requester"
path: search
http_method: GET
request_parameters:
channelId: "{{ config.channel_ids }}"
record_selector:
type: RecordSelector
extractor:
type: DpathExtractor
field_path:
- items
- "*"
- id
paginator:
type: DefaultPaginator
page_token_option:
type: RequestOption
inject_into: request_parameter
field_name: pageToken
page_size_option:
type: RequestOption
inject_into: request_parameter
field_name: maxResults
pagination_strategy:
type: CursorPagination
page_size: 50
cursor_value: "{{ response.nextPageToken }}"
stop_condition: "{{ not response.get('nextPageToken') }}"
schema_loader:
type: InlineSchemaLoader
schema:
$ref: "#/schemas/videos"
channel_comments:
type: DeclarativeStream
name: channel_comments
retriever:
type: SimpleRetriever
requester:
$ref: "#/definitions/base_requester"
path: commentThreads
http_method: GET
request_parameters:
part: snippet,replies
allThreadsRelatedToChannelId: "{{ config.channel_ids }}"
record_selector:
type: RecordSelector
extractor:
type: DpathExtractor
field_path:
- items
- "*"
- snippet
paginator:
type: DefaultPaginator
page_token_option:
type: RequestOption
inject_into: request_parameter
field_name: pageToken
pagination_strategy:
type: CursorPagination
cursor_value: "{{ response.get('nextPageToken', '') }}"
stop_condition: "{{ response.get('nextPageToken') is none }}"
schema_loader:
type: InlineSchemaLoader
schema:
$ref: "#/schemas/channel_comments"
api_key_authenticator:
type: ApiKeyAuthenticator
api_token: '{{ config["credentials"]["api_key"] }}'
inject_into:
type: RequestOption
field_name: key
inject_into: request_parameter
oauth2_authenticator:
type: OAuthAuthenticator
client_id: '{{ config["credentials"]["client_id"] }}'
client_secret: '{{ config["credentials"]["client_secret"] }}'
token_refresh_endpoint: https://oauth2.googleapis.com/token
scopes:
- https://www.googleapis.com/auth/youtube.force-ssl
refresh_token: '{{ config["credentials"]["refresh_token"] }}'
refresh_request_body: {}
base_requester:
type: HttpRequester
url_base: https://www.googleapis.com/youtube/v3/
authenticator:
type: SelectiveAuthenticator
authenticator_selection_path: ["credentials", "auth_method"]
authenticators:
api_key: "#/definitions/api_key_authenticator"
oauth2.0: "#/definitions/oauth2_authenticator"
streams:
- $ref: "#/definitions/streams/video"
- $ref: "#/definitions/streams/channels"
- $ref: "#/definitions/streams/comments"
- $ref: "#/definitions/streams/videos"
- $ref: "#/definitions/streams/channel_comments"
spec:
type: Spec
connection_specification:
type: object
$schema: http://json-schema.org/draft-07/schema#
required:
- credentials
- channel_ids
properties:
credentials:
type: object
title: Authentication method
description: Authentication method
order: 0
oneOf:
- type: object
title: API Key
required:
- api_key
properties:
auth_method:
type: string
const: api_key
api_key:
type: string
order: 0
title: API Key
airbyte_secret: true
- type: object
title: Google OAuth 2.0
required:
- client_id
- client_secret
- refresh_token
properties:
auth_method:
type: string
const: oauth2.0
default: oauth2.0
enum:
- oauth2.0
client_id:
type: string
order: 0
title: Client ID
airbyte_secret: true
client_secret:
type: string
order: 1
title: Client Secret
airbyte_secret: true
refresh_token:
type: string
order: 2
title: Refresh Token
airbyte_secret: true
channel_ids:
type: array
title: Channel IDs
order: 1
additionalProperties: true
advanced_auth:
auth_flow_type: oauth2.0
predicate_key:
- credentials
- auth_method
predicate_value: oauth2.0
oauth_config_specification:
oauth_connector_input_specification:
scope: https://www.googleapis.com/auth/youtube.force-ssl
consent_url: https://accounts.google.com/o/oauth2/v2/auth?{{client_id_param}}&{{redirect_uri_param}}&response_type=code&{{scope_param}}&access_type=offline&{{state_param}}&include_granted_scopes=true&prompt=consent
access_token_url: https://oauth2.googleapis.com/token?{{client_id_param}}&{{client_secret_param}}&{{auth_code_param}}&grant_type=authorization_code&{{redirect_uri_param}}
extract_output: ["refresh_token"]
complete_oauth_output_specification:
type: object
additionalProperties: true
properties:
refresh_token:
type: string
path_in_connector_config:
- credentials
- refresh_token
complete_oauth_server_input_specification:
type: object
additionalProperties: true
properties:
client_id:
type: string
client_secret:
type: string
complete_oauth_server_output_specification:
type: object
additionalProperties: true
properties:
client_id:
type: string
path_in_connector_config:
- credentials
- client_id
client_secret:
type: string
path_in_connector_config:
- credentials
- client_secret
config_normalization_rules:
type: ConfigNormalizationRules
config_migrations:
- type: ConfigMigration
description: |
Remaps the existing config authentication format to a new format
OLD: {"api_key": ...}
NEW: {"credentials": {"api_key": ..., "auth_method": "api_key"}}
transformations:
- type: ConfigAddFields
fields:
- type: AddedFieldDefinition
path: ["credentials", "api_key"]
value: "{{ config['api_key'] }}"
condition: "{{ config.get('api_key', False) }}"
- type: ConfigAddFields
fields:
- type: AddedFieldDefinition
path: ["credentials", "auth_method"]
value: "api_key"
condition: "{{ config.get('api_key', False) }}"
metadata:
autoImportSchema:
video: true
channels: false
comments: true
videos: true
channel_comments: true
testedStreams:
video:
streamHash: a3f8831f5f8a0ca3efed0972cab191cc181d2b4e
hasResponse: true
responsesAreSuccessful: true
hasRecords: true
primaryKeysArePresent: true
primaryKeysAreUnique: true
channels:
streamHash: 343ca32364dc855bd2f38aa2eb633c05100b5fd1
hasResponse: true
responsesAreSuccessful: true
hasRecords: true
primaryKeysArePresent: true
primaryKeysAreUnique: true
comments:
streamHash: 8d2176a881a0bca19e00ec1d0cfe70a823e4de85
hasResponse: true
responsesAreSuccessful: true
hasRecords: true
primaryKeysArePresent: true
primaryKeysAreUnique: true
videos:
streamHash: 9f29c51ff5dbe597fbbd7ebfab42300eb29860f8
hasResponse: true
responsesAreSuccessful: true
hasRecords: true
primaryKeysArePresent: true
primaryKeysAreUnique: true
channel_comments:
streamHash: 958a9f915e04312572380361e2b97c730c37ad29
hasResponse: true
responsesAreSuccessful: true
hasRecords: true
primaryKeysArePresent: true
primaryKeysAreUnique: true
assist:
docsUrl: https://developers.google.com/youtube/v3/guides/implementation
schemas:
video:
type: object
$schema: http://json-schema.org/schema#
additionalProperties: true
properties:
description:
type:
- string
- "null"
categoryId:
type:
- string
- "null"
channelId:
type:
- string
- "null"
channelTitle:
type:
- string
- "null"
datetime:
type:
- string
- "null"
defaultAudioLanguage:
type:
- string
- "null"
defaultLanguage:
type:
- string
- "null"
liveBroadcastContent:
type:
- string
- "null"
localized:
type:
- object
- "null"
properties:
description:
type:
- string
- "null"
title:
type:
- string
- "null"
publishedAt:
type:
- string
- "null"
tags:
type:
- array
- "null"
items:
type:
- string
- "null"
thumbnails:
type:
- object
- "null"
properties:
default:
type:
- object
- "null"
properties:
height:
type:
- number
- "null"
url:
type:
- string
- "null"
width:
type:
- number
- "null"
high:
type:
- object
- "null"
properties:
height:
type:
- number
- "null"
url:
type:
- string
- "null"
width:
type:
- number
- "null"
maxres:
type:
- object
- "null"
properties:
height:
type:
- number
- "null"
url:
type:
- string
- "null"
width:
type:
- number
- "null"
medium:
type:
- object
- "null"
properties:
height:
type:
- number
- "null"
url:
type:
- string
- "null"
width:
type:
- number
- "null"
standard:
type:
- object
- "null"
properties:
height:
type:
- number
- "null"
url:
type:
- string
- "null"
width:
type:
- number
- "null"
title:
type:
- string
- "null"
videoId:
type: string
required:
- videoId
channels:
type: object
$schema: http://json-schema.org/schema#
additionalProperties: true
properties:
contentDetails:
type:
- object
- "null"
properties:
relatedPlaylists:
type:
- object
- "null"
properties:
likes:
type:
- string
- "null"
uploads:
type:
- string
- "null"
etag:
type:
- string
- "null"
id:
type: string
kind:
type:
- string
- "null"
snippet:
type:
- object
- "null"
properties:
description:
type:
- string
- "null"
country:
type:
- string
- "null"
customUrl:
type:
- string
- "null"
localized:
type:
- object
- "null"
properties:
description:
type:
- string
- "null"
title:
type:
- string
- "null"
publishedAt:
type:
- string
- "null"
thumbnails:
type:
- object
- "null"
properties:
default:
type:
- object
- "null"
properties:
height:
type:
- number
- "null"
url:
type:
- string
- "null"
width:
type:
- number
- "null"
high:
type:
- object
- "null"
properties:
height:
type:
- number
- "null"
url:
type:
- string
- "null"
width:
type:
- number
- "null"
medium:
type:
- object
- "null"
properties:
height:
type:
- number
- "null"
url:
type:
- string
- "null"
width:
type:
- number
- "null"
title:
type:
- string
- "null"
statistics:
type:
- object
- "null"
properties:
hiddenSubscriberCount:
type:
- boolean
- "null"
subscriberCount:
type:
- string
- "null"
videoCount:
type:
- string
- "null"
viewCount:
type:
- string
- "null"
required:
- id
comments:
type: object
$schema: http://json-schema.org/schema#
additionalProperties: true
properties:
canReply:
type:
- boolean
- "null"
channelId:
type:
- string
- "null"
isPublic:
type:
- boolean
- "null"
topLevelComment:
type:
- object
- "null"
properties:
etag:
type:
- string
- "null"
id:
type:
- string
- "null"
kind:
type:
- string
- "null"
snippet:
type:
- object
- "null"
properties:
authorChannelId:
type:
- object
- "null"
properties:
value:
type:
- string
- "null"
authorChannelUrl:
type:
- string
- "null"
authorDisplayName:
type:
- string
- "null"
authorProfileImageUrl:
type:
- string
- "null"
canRate:
type:
- boolean
- "null"
channelId:
type:
- string
- "null"
likeCount:
type:
- number
- "null"
publishedAt:
type:
- string
- "null"
textDisplay:
type:
- string
- "null"
textOriginal:
type:
- string
- "null"
updatedAt:
type:
- string
- "null"
videoId:
type:
- string
- "null"
viewerRating:
type:
- string
- "null"
totalReplyCount:
type:
- number
- "null"
videoId:
type:
- string
- "null"
videos:
type: object
$schema: http://json-schema.org/schema#
additionalProperties: true
properties:
kind:
type:
- string
- "null"
videoId:
type:
- string
- "null"
channel_comments:
type: object
$schema: http://json-schema.org/schema#
additionalProperties: true
properties:
canReply:
type:
- boolean
- "null"
channelId:
type:
- string
- "null"
isPublic:
type:
- boolean
- "null"
topLevelComment:
type:
- object
- "null"
properties:
etag:
type:
- string
- "null"
id:
type:
- string
- "null"
kind:
type:
- string
- "null"
snippet:
type:
- object
- "null"
properties:
authorChannelId:
type:
- object
- "null"
properties:
value:
type:
- string
- "null"
authorChannelUrl:
type:
- string
- "null"
authorDisplayName:
type:
- string
- "null"
authorProfileImageUrl:
type:
- string
- "null"
canRate:
type:
- boolean
- "null"
channelId:
type:
- string
- "null"
likeCount:
type:
- number
- "null"
publishedAt:
type:
- string
- "null"
textDisplay:
type:
- string
- "null"
textOriginal:
type:
- string
- "null"
updatedAt:
type:
- string
- "null"
videoId:
type:
- string
- "null"
viewerRating:
type:
- string
- "null"
totalReplyCount:
type:
- number
- "null"
videoId:
type:
- string
- "null"