version: 6.4.0

type: DeclarativeSource

description: >-
  The YouTube Data API v3 is an API that provides access to YouTube data, such as videos, playlists, channels, comments and simple stats.
  This is a simpler version of Youtube connector, if you need more detailed reports from your channel please check
  the [Youtube Analytics Connector](https://docs.airbyte.com/integrations/sources/youtube-analytics)

check:
  type: CheckStream
  stream_names:
    - channels

definitions:
  streams:
    video:
      type: DeclarativeStream
      name: video
      primary_key:
        - videoId
      retriever:
        type: SimpleRetriever
        requester:
          $ref: "#/definitions/base_requester"
          path: videos
          http_method: GET
          request_parameters:
            part: snippet,contentDetails,statistics
        record_selector:
          type: RecordSelector
          extractor:
            type: DpathExtractor
            field_path:
              - items
              - "*"
              - snippet
        paginator:
          type: DefaultPaginator
          page_token_option:
            type: RequestOption
            inject_into: request_parameter
            field_name: pageToken
          pagination_strategy:
            type: CursorPagination
            cursor_value: "{{ response.get('nextPageToken', '') }}"
            stop_condition: "{{ response.get('nextPageToken') is none }}"
        partition_router:
          type: SubstreamPartitionRouter
          parent_stream_configs:
            - type: ParentStreamConfig
              parent_key: videoId
              request_option:
                type: RequestOption
                inject_into: request_parameter
                field_name: id
              partition_field: id
              stream:
                $ref: "#/definitions/streams/videos"
      transformations:
        - type: AddFields
          fields:
            - path:
                - datetime
              value: "{{ now_utc() }}"
        - type: AddFields
          fields:
            - path:
                - videoId
              value: "{{ stream_slice.id }}"
      schema_loader:
        type: InlineSchemaLoader
        schema:
          $ref: "#/schemas/video"
    channels:
      type: DeclarativeStream
      name: channels
      primary_key:
        - id
      retriever:
        type: SimpleRetriever
        requester:
          $ref: "#/definitions/base_requester"
          path: channels
          http_method: GET
          request_parameters:
            part: snippet,contentDetails,statistics
        record_selector:
          type: RecordSelector
          extractor:
            type: DpathExtractor
            field_path:
              - items
              - "*"
              - snippet
        paginator:
          type: DefaultPaginator
          page_token_option:
            type: RequestOption
            inject_into: request_parameter
            field_name: pageToken
          pagination_strategy:
            type: CursorPagination
            cursor_value: "{{ response.get('nextPageToken', '') }}"
            stop_condition: "{{ response.get('nextPageToken') is none }}"
        partition_router:
          type: ListPartitionRouter
          values: "{{ config.channel_ids }}"
          cursor_field: channel_id
          request_option:
            type: RequestOption
            inject_into: request_parameter
            field_name: id
      transformations:
        - type: AddFields
          fields:
            - path:
                - id
              value: "{{ stream_partition.channel_id }}"
      schema_loader:
        type: InlineSchemaLoader
        schema:
          $ref: "#/schemas/channels"
    comments:
      type: DeclarativeStream
      name: comments
      retriever:
        type: SimpleRetriever
        requester:
          $ref: "#/definitions/base_requester"
          path: commentThreads
          http_method: GET
          request_parameters:
            part: snippet,replies
        record_selector:
          type: RecordSelector
          extractor:
            type: DpathExtractor
            field_path:
              - items
              - "*"
              - snippet
        paginator:
          type: DefaultPaginator
          page_token_option:
            type: RequestOption
            inject_into: request_parameter
            field_name: pageToken
          pagination_strategy:
            type: CursorPagination
            cursor_value: "{{ response.get('nextPageToken', '') }}"
            stop_condition: "{{ response.get('nextPageToken') is none }}"
        partition_router:
          type: SubstreamPartitionRouter
          parent_stream_configs:
            - type: ParentStreamConfig
              parent_key: videoId
              request_option:
                type: RequestOption
                inject_into: request_parameter
                field_name: videoId
              partition_field: id
              stream:
                $ref: "#/definitions/streams/videos"
      schema_loader:
        type: InlineSchemaLoader
        schema:
          $ref: "#/schemas/comments"
    videos:
      type: DeclarativeStream
      name: videos
      retriever:
        type: SimpleRetriever
        requester:
          $ref: "#/definitions/base_requester"
          path: search
          http_method: GET
          request_parameters:
            channelId: "{{ config.channel_ids }}"
        record_selector:
          type: RecordSelector
          extractor:
            type: DpathExtractor
            field_path:
              - items
              - "*"
              - id
        paginator:
          type: DefaultPaginator
          page_token_option:
            type: RequestOption
            inject_into: request_parameter
            field_name: pageToken
          page_size_option:
            type: RequestOption
            inject_into: request_parameter
            field_name: maxResults
          pagination_strategy:
            type: CursorPagination
            page_size: 50
            cursor_value: "{{ response.nextPageToken }}"
            stop_condition: "{{ not response.get('nextPageToken') }}"
      schema_loader:
        type: InlineSchemaLoader
        schema:
          $ref: "#/schemas/videos"
    channel_comments:
      type: DeclarativeStream
      name: channel_comments
      retriever:
        type: SimpleRetriever
        requester:
          $ref: "#/definitions/base_requester"
          path: commentThreads
          http_method: GET
          request_parameters:
            part: snippet,replies
            allThreadsRelatedToChannelId: "{{ config.channel_ids }}"
        record_selector:
          type: RecordSelector
          extractor:
            type: DpathExtractor
            field_path:
              - items
              - "*"
              - snippet
        paginator:
          type: DefaultPaginator
          page_token_option:
            type: RequestOption
            inject_into: request_parameter
            field_name: pageToken
          pagination_strategy:
            type: CursorPagination
            cursor_value: "{{ response.get('nextPageToken', '') }}"
            stop_condition: "{{ response.get('nextPageToken') is none }}"
      schema_loader:
        type: InlineSchemaLoader
        schema:
          $ref: "#/schemas/channel_comments"
  api_key_authenticator:
    type: ApiKeyAuthenticator
    api_token: '{{ config["credentials"]["api_key"] }}'
    inject_into:
      type: RequestOption
      field_name: key
      inject_into: request_parameter
  oauth2_authenticator:
    type: OAuthAuthenticator
    client_id: '{{ config["credentials"]["client_id"] }}'
    client_secret: '{{ config["credentials"]["client_secret"] }}'
    token_refresh_endpoint: https://oauth2.googleapis.com/token
    scopes:
      - https://www.googleapis.com/auth/youtube.force-ssl
    refresh_token: '{{ config["credentials"]["refresh_token"] }}'
    refresh_request_body: {}
  base_requester:
    type: HttpRequester
    url_base: https://www.googleapis.com/youtube/v3/
    authenticator:
      type: SelectiveAuthenticator
      authenticator_selection_path: ["credentials", "auth_method"]
      authenticators:
        api_key: "#/definitions/api_key_authenticator"
        oauth2.0: "#/definitions/oauth2_authenticator"

streams:
  - $ref: "#/definitions/streams/video"
  - $ref: "#/definitions/streams/channels"
  - $ref: "#/definitions/streams/comments"
  - $ref: "#/definitions/streams/videos"
  - $ref: "#/definitions/streams/channel_comments"

spec:
  type: Spec
  connection_specification:
    type: object
    $schema: http://json-schema.org/draft-07/schema#
    required:
      - credentials
      - channel_ids
    properties:
      credentials:
        type: object
        title: Authentication method
        description: Authentication method
        order: 0
        oneOf:
          - type: object
            title: API Key
            required:
              - api_key
            properties:
              auth_method:
                type: string
                const: api_key
              api_key:
                type: string
                order: 0
                title: API Key
                airbyte_secret: true
          - type: object
            title: Google OAuth 2.0
            required:
              - client_id
              - client_secret
              - refresh_token
            properties:
              auth_method:
                type: string
                const: oauth2.0
                default: oauth2.0
                enum:
                  - oauth2.0
              client_id:
                type: string
                order: 0
                title: Client ID
                airbyte_secret: true
              client_secret:
                type: string
                order: 1
                title: Client Secret
                airbyte_secret: true
              refresh_token:
                type: string
                order: 2
                title: Refresh Token
                airbyte_secret: true
      channel_ids:
        type: array
        title: Channel IDs
        order: 1
    additionalProperties: true
  advanced_auth:
    auth_flow_type: oauth2.0
    predicate_key:
      - credentials
      - auth_method
    predicate_value: oauth2.0
    oauth_config_specification:
      oauth_connector_input_specification:
        scope: https://www.googleapis.com/auth/youtube.force-ssl
        consent_url: https://accounts.google.com/o/oauth2/v2/auth?{{client_id_param}}&{{redirect_uri_param}}&response_type=code&{{scope_param}}&access_type=offline&{{state_param}}&include_granted_scopes=true&prompt=consent
        access_token_url: https://oauth2.googleapis.com/token?{{client_id_param}}&{{client_secret_param}}&{{auth_code_param}}&grant_type=authorization_code&{{redirect_uri_param}}
        extract_output: ["refresh_token"]
      complete_oauth_output_specification:
        type: object
        additionalProperties: true
        properties:
          refresh_token:
            type: string
            path_in_connector_config:
              - credentials
              - refresh_token
      complete_oauth_server_input_specification:
        type: object
        additionalProperties: true
        properties:
          client_id:
            type: string
          client_secret:
            type: string
      complete_oauth_server_output_specification:
        type: object
        additionalProperties: true
        properties:
          client_id:
            type: string
            path_in_connector_config:
              - credentials
              - client_id
          client_secret:
            type: string
            path_in_connector_config:
              - credentials
              - client_secret
  config_normalization_rules:
    type: ConfigNormalizationRules
    config_migrations:
      - type: ConfigMigration
        description: |
          Remaps the existing config authentication format to a new format
          OLD: {"api_key": ...}
          NEW: {"credentials": {"api_key": ..., "auth_method": "api_key"}}
        transformations:
          - type: ConfigAddFields
            fields:
              - type: AddedFieldDefinition
                path: ["credentials", "api_key"]
                value: "{{ config['api_key'] }}"
            condition: "{{ config.get('api_key', False) }}"
          - type: ConfigAddFields
            fields:
              - type: AddedFieldDefinition
                path: ["credentials", "auth_method"]
                value: "api_key"
            condition: "{{ config.get('api_key', False) }}"

metadata:
  autoImportSchema:
    video: true
    channels: false
    comments: true
    videos: true
    channel_comments: true
  testedStreams:
    video:
      streamHash: a3f8831f5f8a0ca3efed0972cab191cc181d2b4e
      hasResponse: true
      responsesAreSuccessful: true
      hasRecords: true
      primaryKeysArePresent: true
      primaryKeysAreUnique: true
    channels:
      streamHash: 343ca32364dc855bd2f38aa2eb633c05100b5fd1
      hasResponse: true
      responsesAreSuccessful: true
      hasRecords: true
      primaryKeysArePresent: true
      primaryKeysAreUnique: true
    comments:
      streamHash: 8d2176a881a0bca19e00ec1d0cfe70a823e4de85
      hasResponse: true
      responsesAreSuccessful: true
      hasRecords: true
      primaryKeysArePresent: true
      primaryKeysAreUnique: true
    videos:
      streamHash: 9f29c51ff5dbe597fbbd7ebfab42300eb29860f8
      hasResponse: true
      responsesAreSuccessful: true
      hasRecords: true
      primaryKeysArePresent: true
      primaryKeysAreUnique: true
    channel_comments:
      streamHash: 958a9f915e04312572380361e2b97c730c37ad29
      hasResponse: true
      responsesAreSuccessful: true
      hasRecords: true
      primaryKeysArePresent: true
      primaryKeysAreUnique: true
  assist:
    docsUrl: https://developers.google.com/youtube/v3/guides/implementation

schemas:
  video:
    type: object
    $schema: http://json-schema.org/schema#
    additionalProperties: true
    properties:
      description:
        type:
          - string
          - "null"
      categoryId:
        type:
          - string
          - "null"
      channelId:
        type:
          - string
          - "null"
      channelTitle:
        type:
          - string
          - "null"
      datetime:
        type:
          - string
          - "null"
      defaultAudioLanguage:
        type:
          - string
          - "null"
      defaultLanguage:
        type:
          - string
          - "null"
      liveBroadcastContent:
        type:
          - string
          - "null"
      localized:
        type:
          - object
          - "null"
        properties:
          description:
            type:
              - string
              - "null"
          title:
            type:
              - string
              - "null"
      publishedAt:
        type:
          - string
          - "null"
      tags:
        type:
          - array
          - "null"
        items:
          type:
            - string
            - "null"
      thumbnails:
        type:
          - object
          - "null"
        properties:
          default:
            type:
              - object
              - "null"
            properties:
              height:
                type:
                  - number
                  - "null"
              url:
                type:
                  - string
                  - "null"
              width:
                type:
                  - number
                  - "null"
          high:
            type:
              - object
              - "null"
            properties:
              height:
                type:
                  - number
                  - "null"
              url:
                type:
                  - string
                  - "null"
              width:
                type:
                  - number
                  - "null"
          maxres:
            type:
              - object
              - "null"
            properties:
              height:
                type:
                  - number
                  - "null"
              url:
                type:
                  - string
                  - "null"
              width:
                type:
                  - number
                  - "null"
          medium:
            type:
              - object
              - "null"
            properties:
              height:
                type:
                  - number
                  - "null"
              url:
                type:
                  - string
                  - "null"
              width:
                type:
                  - number
                  - "null"
          standard:
            type:
              - object
              - "null"
            properties:
              height:
                type:
                  - number
                  - "null"
              url:
                type:
                  - string
                  - "null"
              width:
                type:
                  - number
                  - "null"
      title:
        type:
          - string
          - "null"
      videoId:
        type: string
    required:
      - videoId
  channels:
    type: object
    $schema: http://json-schema.org/schema#
    additionalProperties: true
    properties:
      contentDetails:
        type:
          - object
          - "null"
        properties:
          relatedPlaylists:
            type:
              - object
              - "null"
            properties:
              likes:
                type:
                  - string
                  - "null"
              uploads:
                type:
                  - string
                  - "null"
      etag:
        type:
          - string
          - "null"
      id:
        type: string
      kind:
        type:
          - string
          - "null"
      snippet:
        type:
          - object
          - "null"
        properties:
          description:
            type:
              - string
              - "null"
          country:
            type:
              - string
              - "null"
          customUrl:
            type:
              - string
              - "null"
          localized:
            type:
              - object
              - "null"
            properties:
              description:
                type:
                  - string
                  - "null"
              title:
                type:
                  - string
                  - "null"
          publishedAt:
            type:
              - string
              - "null"
          thumbnails:
            type:
              - object
              - "null"
            properties:
              default:
                type:
                  - object
                  - "null"
                properties:
                  height:
                    type:
                      - number
                      - "null"
                  url:
                    type:
                      - string
                      - "null"
                  width:
                    type:
                      - number
                      - "null"
              high:
                type:
                  - object
                  - "null"
                properties:
                  height:
                    type:
                      - number
                      - "null"
                  url:
                    type:
                      - string
                      - "null"
                  width:
                    type:
                      - number
                      - "null"
              medium:
                type:
                  - object
                  - "null"
                properties:
                  height:
                    type:
                      - number
                      - "null"
                  url:
                    type:
                      - string
                      - "null"
                  width:
                    type:
                      - number
                      - "null"
          title:
            type:
              - string
              - "null"
      statistics:
        type:
          - object
          - "null"
        properties:
          hiddenSubscriberCount:
            type:
              - boolean
              - "null"
          subscriberCount:
            type:
              - string
              - "null"
          videoCount:
            type:
              - string
              - "null"
          viewCount:
            type:
              - string
              - "null"
    required:
      - id
  comments:
    type: object
    $schema: http://json-schema.org/schema#
    additionalProperties: true
    properties:
      canReply:
        type:
          - boolean
          - "null"
      channelId:
        type:
          - string
          - "null"
      isPublic:
        type:
          - boolean
          - "null"
      topLevelComment:
        type:
          - object
          - "null"
        properties:
          etag:
            type:
              - string
              - "null"
          id:
            type:
              - string
              - "null"
          kind:
            type:
              - string
              - "null"
          snippet:
            type:
              - object
              - "null"
            properties:
              authorChannelId:
                type:
                  - object
                  - "null"
                properties:
                  value:
                    type:
                      - string
                      - "null"
              authorChannelUrl:
                type:
                  - string
                  - "null"
              authorDisplayName:
                type:
                  - string
                  - "null"
              authorProfileImageUrl:
                type:
                  - string
                  - "null"
              canRate:
                type:
                  - boolean
                  - "null"
              channelId:
                type:
                  - string
                  - "null"
              likeCount:
                type:
                  - number
                  - "null"
              publishedAt:
                type:
                  - string
                  - "null"
              textDisplay:
                type:
                  - string
                  - "null"
              textOriginal:
                type:
                  - string
                  - "null"
              updatedAt:
                type:
                  - string
                  - "null"
              videoId:
                type:
                  - string
                  - "null"
              viewerRating:
                type:
                  - string
                  - "null"
      totalReplyCount:
        type:
          - number
          - "null"
      videoId:
        type:
          - string
          - "null"
  videos:
    type: object
    $schema: http://json-schema.org/schema#
    additionalProperties: true
    properties:
      kind:
        type:
          - string
          - "null"
      videoId:
        type:
          - string
          - "null"
  channel_comments:
    type: object
    $schema: http://json-schema.org/schema#
    additionalProperties: true
    properties:
      canReply:
        type:
          - boolean
          - "null"
      channelId:
        type:
          - string
          - "null"
      isPublic:
        type:
          - boolean
          - "null"
      topLevelComment:
        type:
          - object
          - "null"
        properties:
          etag:
            type:
              - string
              - "null"
          id:
            type:
              - string
              - "null"
          kind:
            type:
              - string
              - "null"
          snippet:
            type:
              - object
              - "null"
            properties:
              authorChannelId:
                type:
                  - object
                  - "null"
                properties:
                  value:
                    type:
                      - string
                      - "null"
              authorChannelUrl:
                type:
                  - string
                  - "null"
              authorDisplayName:
                type:
                  - string
                  - "null"
              authorProfileImageUrl:
                type:
                  - string
                  - "null"
              canRate:
                type:
                  - boolean
                  - "null"
              channelId:
                type:
                  - string
                  - "null"
              likeCount:
                type:
                  - number
                  - "null"
              publishedAt:
                type:
                  - string
                  - "null"
              textDisplay:
                type:
                  - string
                  - "null"
              textOriginal:
                type:
                  - string
                  - "null"
              updatedAt:
                type:
                  - string
                  - "null"
              videoId:
                type:
                  - string
                  - "null"
              viewerRating:
                type:
                  - string
                  - "null"
      totalReplyCount:
        type:
          - number
          - "null"
      videoId:
        type:
          - string
          - "null"