airbyte/airbyte-integrations/connectors/source-jira/unit_tests/conftest.py

#
# Copyright (c) 2023 Airbyte, Inc., all rights reserved.
#

import json
import os
import sys
from pathlib import Path

import responses
from pytest import fixture
from responses import matchers

from airbyte_cdk.sources.declarative.yaml_declarative_source import YamlDeclarativeSource
from airbyte_cdk.sources.streams import Stream
from airbyte_cdk.test.catalog_builder import CatalogBuilder
from airbyte_cdk.test.state_builder import StateBuilder


pytest_plugins = ["airbyte_cdk.test.utils.manifest_only_fixtures"]

ENV_REQUEST_CACHE_PATH = "REQUEST_CACHE_PATH"
os.environ["REQUEST_CACHE_PATH"] = ENV_REQUEST_CACHE_PATH


def _get_manifest_path() -> Path:
    source_declarative_manifest_path = Path("/airbyte/integration_code/source_declarative_manifest")
    if source_declarative_manifest_path.exists():
        return source_declarative_manifest_path
    return Path(__file__).parent.parent


_SOURCE_FOLDER_PATH = _get_manifest_path()
_YAML_FILE_PATH = _SOURCE_FOLDER_PATH / "manifest.yaml"

sys.path.append(str(_SOURCE_FOLDER_PATH))  # to allow loading custom components


def get_source(config, state=None) -> YamlDeclarativeSource:
    """
    Create a YamlDeclarativeSource instance for testing.

    This is the main entry point for running your connector in tests.
    """
    catalog = CatalogBuilder().build()
    state = StateBuilder().build() if not state else state
    return YamlDeclarativeSource(path_to_yaml=str(_YAML_FILE_PATH), catalog=catalog, config=config, state=state)


def delete_cache_files(cache_directory):
    directory_path = Path(cache_directory)
    if directory_path.exists() and directory_path.is_dir():
        for file_path in directory_path.glob("*.sqlite"):
            file_path.unlink()


@fixture(autouse=True)
def clear_cache_before_each_test():
    # The problem: Once the first request is cached, we will keep getting the cached result no matter what setup we prepared for a particular test.
    # Solution: We must delete the cache before each test because for the same URL, we want to define multiple responses and status codes.
    delete_cache_files(os.getenv(ENV_REQUEST_CACHE_PATH))
    yield


@fixture
def config():
    return {
        "api_token": "token",
        "domain": "domain",
        "email": "email@email.com",
        "start_date": "2021-01-01T00:00:00Z",
        "projects": ["Project1"],
        "enable_experimental_streams": True,
    }


def load_file(fn):
    return open(os.path.join(os.path.dirname(__file__), "responses", fn)).read()


@fixture
def application_roles_response():
    return json.loads(load_file("application_role.json"))


@fixture
def boards_response():
    return json.loads(load_file("board.json"))


@fixture
def dashboards_response():
    return json.loads(load_file("dashboard.json"))


@fixture
def filters_response():
    return json.loads(load_file("filter.json"))


@fixture
def groups_response():
    return json.loads(load_file("groups.json"))


@fixture
def issue_fields_response():
    return json.loads(load_file("issue_fields.json"))


@fixture
def issues_field_configurations_response():
    return json.loads(load_file("issues_field_configurations.json"))


@fixture
def issues_link_types_response():
    return json.loads(load_file("issues_link_types.json"))


@fixture
def issues_navigator_settings_response():
    return json.loads(load_file("issues_navigator_settings.json"))


@fixture
def issue_notification_schemas_response():
    return json.loads(load_file("issue_notification_schemas.json"))


@fixture
def issue_properties_response():
    return json.loads(load_file("issue_properties.json"))


@fixture
def issue_resolutions_response():
    return json.loads(load_file("issue_resolutions.json"))


@fixture
def issue_security_schemes_response():
    return json.loads(load_file("issue_security_schemes.json"))


@fixture
def issue_type_schemes_response():
    return json.loads(load_file("issue_type.json"))


@fixture
def jira_settings_response():
    return json.loads(load_file("jira_settings.json"))


@fixture
def board_issues_response():
    return json.loads(load_file("board_issues.json"))


@fixture
def filter_sharing_response():
    return json.loads(load_file("filter_sharing.json"))


@fixture
def projects_response():
    return json.loads(load_file("projects.json"))


@fixture
def projects_avatars_response():
    return json.loads(load_file("projects_avatars.json"))


@fixture
def projects_categories_response():
    return json.loads(load_file("projects_categories.json"))


@fixture
def screens_response():
    return json.loads(load_file("screens.json"))


@fixture
def screen_tabs_response():
    return json.loads(load_file("screen_tabs.json"))


@fixture
def screen_tab_fields_response():
    return json.loads(load_file("screen_tab_fields.json"))


@fixture
def sprints_response():
    return json.loads(load_file("sprints.json"))


@fixture
def sprints_issues_response():
    return json.loads(load_file("sprint_issues.json"))


@fixture
def time_tracking_response():
    return json.loads(load_file("time_tracking.json"))


@fixture
def users_response():
    return json.loads(load_file("users.json"))


@fixture
def users_groups_detailed_response():
    return json.loads(load_file("users_groups_detailed.json"))


@fixture
def workflows_response():
    return json.loads(load_file("workflows.json"))


@fixture
def workflow_schemas_response():
    return json.loads(load_file("workflow_schemas.json"))


@fixture
def workflow_statuses_response():
    return json.loads(load_file("workflow_statuses.json"))


@fixture
def workflow_status_categories_response():
    return json.loads(load_file("workflow_status_categories.json"))


@fixture
def avatars_response():
    return json.loads(load_file("avatars.json"))


@fixture
def issues_response():
    return json.loads(load_file("issues.json"))


@fixture
def issue_comments_response():
    return json.loads(load_file("issue_comments.json"))


@fixture
def issue_custom_field_contexts_response():
    return json.loads(load_file("issue_custom_field_contexts.json"))


@fixture
def issue_custom_field_options_response():
    return json.loads(load_file("issue_custom_field_options.json"))


@fixture
def issue_property_keys_response():
    return json.loads(load_file("issue_property_keys.json"))


@fixture
def project_permissions_response():
    return json.loads(load_file("project_permissions.json"))


@fixture
def project_email_response():
    return json.loads(load_file("project_email.json"))


@fixture
def project_components_response():
    return json.loads(load_file("project_components.json"))


@fixture
def permissions_response():
    return json.loads(load_file("permissions.json"))


@fixture
def labels_response():
    return json.loads(load_file("labels.json"))


@fixture
def issue_worklogs_response():
    return json.loads(load_file("issue_worklogs.json"))


@fixture
def issue_watchers_response():
    return json.loads(load_file("issue_watchers.json"))


@fixture
def issue_votes_response():
    return json.loads(load_file("issue_votes.json"))


@fixture
def issue_remote_links_response():
    return json.loads(load_file("issue_remote_links.json"))


@fixture
def projects_versions_response():
    return json.loads(load_file("projects_versions.json"))


@fixture
def mock_projects_responses(config, projects_response):
    responses.add(
        responses.GET,
        f"https://{config['domain']}/rest/api/3/project/search?maxResults=50&expand=description%2Clead&status=live&status=archived&status=deleted",
        json=projects_response,
    )


@fixture
def mock_non_deleted_projects_responses(config, projects_response):
    responses.add(
        responses.GET,
        f"https://{config['domain']}/rest/api/3/project/search?maxResults=50&expand=description%2Clead&status=live&status=archived",
        json=projects_response,
    )


@fixture
def mock_projects_responses_additional_project(config, projects_response):
    projects_response["values"] += [{"id": "3", "key": "Project3"}, {"id": "4", "key": "Project4"}]
    responses.add(
        responses.GET,
        f"https://{config['domain']}/rest/api/3/project/search?maxResults=50&expand=description%2Clead&status=live&status=archived&status=deleted",
        json=projects_response,
    )


@fixture
def mock_issues_responses_with_date_filter(config, issues_response):
    responses.add(
        responses.GET,
        f"https://{config['domain']}/rest/api/3/search/jql",
        match=[
            matchers.query_param_matcher(
                {
                    "maxResults": 50,
                    "fields": "*all",
                    "jql": "updated >= 1609459200000 and project in (1) ORDER BY updated asc",
                    "expand": "renderedFields,transitions,changelog",
                }
            )
        ],
        json=issues_response,
    )
    responses.add(
        responses.GET,
        f"https://{config['domain']}/rest/api/3/search/jql",
        match=[
            matchers.query_param_matcher(
                {
                    "maxResults": 50,
                    "fields": "*all",
                    "jql": "updated >= 1609459200000 and project in (2) ORDER BY updated asc",
                    "expand": "renderedFields,transitions,changelog",
                }
            )
        ],
        json={},
    )
    responses.add(
        responses.GET,
        f"https://{config['domain']}/rest/api/3/search/jql",
        match=[
            matchers.query_param_matcher(
                {
                    "maxResults": 50,
                    "fields": "*all",
                    "jql": "updated >= 1609459200000 and project in (3) ORDER BY updated asc",
                    "expand": "renderedFields,transitions,changelog",
                }
            )
        ],
        json={"errorMessages": ["The value '3' does not exist for the field 'project'."]},
        status=400,
    )
    responses.add(
        responses.GET,
        f"https://{config['domain']}/rest/api/3/search/jql",
        match=[
            matchers.query_param_matcher(
                {
                    "maxResults": 50,
                    "fields": "*all",
                    "jql": "updated >= 1609459200000 and project in (4) ORDER BY updated asc",
                    "expand": "renderedFields,transitions,changelog",
                }
            )
        ],
        json={
            "issues": [
                {
                    "key": "TESTKEY13-2",
                    "fields": {
                        "project": {
                            "id": "10016",
                            "key": "TESTKEY13",
                        },
                        "created": "2022-06-09T16:29:31.871-0700",
                        "updated": "2022-12-08T02:22:18.889-0800",
                    },
                }
            ]
        },
    )


@fixture
def mock_project_emails(config, project_email_response):
    responses.add(
        responses.GET,
        f"https://{config['domain']}/rest/api/3/project/1/email",
        json=project_email_response,
    )
    responses.add(
        responses.GET,
        f"https://{config['domain']}/rest/api/3/project/2/email",
        json=project_email_response,
    )
    responses.add(
        responses.GET,
        f"https://{config['domain']}/rest/api/3/project/3/email",
        json={"errorMessages": ["No access to emails for project 3"]},
        status=403,
    )
    responses.add(
        responses.GET,
        f"https://{config['domain']}/rest/api/3/project/4/email",
        json=project_email_response,
    )


@fixture
def mock_issue_watchers_responses(config, issue_watchers_response):
    responses.add(
        responses.GET,
        f"https://{config['domain']}/rest/api/3/issue/TESTKEY13-1/watchers",
        json=issue_watchers_response,
    )
    responses.add(
        responses.GET,
        f"https://{config['domain']}/rest/api/3/issue/TESTKEY13-2/watchers",
        json={"errorMessages": ["Not found watchers for issue TESTKEY13-2"]},
        status=404,
    )


@fixture
def mock_issue_custom_field_contexts_response(config, issue_custom_field_contexts_response):
    responses.add(
        responses.GET,
        f"https://{config['domain']}/rest/api/3/field/issuetype/context?maxResults=50",
        json=issue_custom_field_contexts_response,
    )
    responses.add(
        responses.GET,
        f"https://{config['domain']}/rest/api/3/field/issuetype2/context?maxResults=50",
        json={},
    )
    responses.add(
        responses.GET,
        f"https://{config['domain']}/rest/api/3/field/issuetype3/context?maxResults=50",
        json={},
    )


@fixture
def mock_issue_custom_field_contexts_response_error(config, issue_custom_field_contexts_response):
    responses.add(
        responses.GET,
        f"https://{config['domain']}/rest/api/3/field/issuetype/context?maxResults=50",
        json=issue_custom_field_contexts_response,
    )
    responses.add(
        responses.GET,
        f"https://{config['domain']}/rest/api/3/field/issuetype2/context?maxResults=50",
        json={"errorMessages": ["Not found issue custom field context for issue fields issuetype2"]},
        status=404,
    )
    responses.add(responses.GET, f"https://{config['domain']}/rest/api/3/field/issuetype3/context?maxResults=50", json={})


@fixture
def mock_issue_custom_field_options_response(config, issue_custom_field_options_response):
    responses.add(
        responses.GET,
        f"https://{config['domain']}/rest/api/3/field/issuetype/context/10130/option?maxResults=50",
        json=issue_custom_field_options_response,
    )
    responses.add(
        responses.GET,
        f"https://{config['domain']}/rest/api/3/field/issuetype/context/10129/option?maxResults=50",
        json={"errorMessages": ["Not found issue custom field options for issue fields issuetype3"]},
        status=404,
    )


@fixture
def mock_fields_response(config, issue_fields_response):
    responses.add(
        responses.GET,
        f"https://{config['domain']}/rest/api/3/field",
        json=issue_fields_response,
    )


@fixture
def mock_users_response(config, users_response):
    responses.add(
        responses.GET,
        f"https://{config['domain']}/rest/api/3/users/search?maxResults=50",
        json=users_response,
    )


@fixture
def mock_board_response(config, boards_response):
    responses.add(
        responses.GET,
        f"https://{config['domain']}/rest/agile/1.0/board?maxResults=50",
        json=boards_response,
    )


@fixture
def mock_screen_response(config, screens_response):
    responses.add(
        responses.GET,
        f"https://{config['domain']}/rest/api/3/screens?maxResults=50",
        json=screens_response,
    )


@fixture
def mock_filter_response(config, filters_response):
    responses.add(
        responses.GET,
        f"https://{config['domain']}/rest/api/3/filter/search?maxResults=50&expand=description%2Cowner%2Cjql%2CviewUrl%2CsearchUrl%2Cfavourite%2CfavouritedCount%2CsharePermissions%2CisWritable%2Csubscriptions",
        json=filters_response,
    )


@fixture
def mock_sprints_response(config, sprints_response):
    responses.add(
        responses.GET,
        f"https://{config['domain']}/rest/agile/1.0/board/1/sprint?maxResults=50",
        json=sprints_response,
    )
    responses.add(
        responses.GET,
        f"https://{config['domain']}/rest/agile/1.0/board/2/sprint?maxResults=50",
        json=sprints_response,
    )
    responses.add(
        responses.GET,
        f"https://{config['domain']}/rest/agile/1.0/board/3/sprint?maxResults=50",
        json=sprints_response,
    )


def find_stream(stream_name, config):
    for stream in YamlDeclarativeSource(config=config, catalog=None, state=None, path_to_yaml=str(_YAML_FILE_PATH)).streams(config=config):
        if stream.name == stream_name:
            return stream
    raise ValueError(f"Stream {stream_name} not found")


def read_full_refresh(stream_instance: Stream):
    yield from stream_instance.read_only_records()