jprdonnelly/airbyte
1
0
Fork 0
mirror of synced 2025-12-25 02:09:19 -05:00
Code Issues Packages Projects Releases Wiki Activity

feat(source-microsoft-sharepoint): Advanced Oauth (#54200)

Browse Source
This commit is contained in:
Aldo Gonzalez
2025-02-27 11:07:10 -06:00
committed by GitHub
parent 82b9d9fda8
commit e325464cd3
7 changed files with 220 additions and 6 deletions
Download Patch File Download Diff File Expand all files Collapse all files

18
airbyte-integrations/connectors/source-microsoft-sharepoint/integration_tests/spec.json
View File

@@ -531,13 +531,29 @@
}
}
},
"oauth_connector_input_specification": {
"consent_url": "https://login.microsoftonline.com/{{tenant_id}}/oauth2/v2.0/authorize?{{client_id_param}}&{{redirect_uri_param}}&{{state_param}}&{{scope_param}}&response_type=code",
"access_token_url": "https://login.microsoftonline.com/{{tenant_id}}/oauth2/v2.0/token",
"scope": "offline_access Files.Read.All Sites.Read.All",
"access_token_headers": {
"Content-Type": "application/x-www-form-urlencoded"
},
"access_token_params": {
"code": "{{ auth_code_value }}",
"client_id": "{{ client_id_value }}",
"redirect_uri": "{{ redirect_uri_value }}",
"client_secret": "{{ client_secret_value }}",
"grant_type": "authorization_code"
}
},
"complete_oauth_output_specification": {
"type": "object",
"additionalProperties": false,
"properties": {
"refresh_token": {
"type": "string",
"path_in_connector_config": ["credentials", "refresh_token"]
"path_in_connector_config": ["credentials", "refresh_token"],
"path_in_oauth_response": ["refresh_token"]
}
}
},

2
airbyte-integrations/connectors/source-microsoft-sharepoint/metadata.yaml
View File

@@ -20,7 +20,7 @@ data:
connectorSubtype: file
connectorType: source
definitionId: 59353119-f0f2-4e5a-a8ba-15d887bc34f6
dockerImageTag: 0.6.1
dockerImageTag: 0.7.0
dockerRepository: airbyte/source-microsoft-sharepoint
githubIssueLabel: source-microsoft-sharepoint
icon: microsoft-sharepoint.svg

2
airbyte-integrations/connectors/source-microsoft-sharepoint/pyproject.toml
View File

@@ -3,7 +3,7 @@ requires = [ "poetry-core>=1.0.0",]
build-backend = "poetry.core.masonry.api"
[tool.poetry]
version = "0.6.1"
version = "0.7.0"
name = "source-microsoft-sharepoint"
description = "Source implementation for Microsoft SharePoint."
authors = [ "Airbyte <contact@airbyte.io>",]

51
airbyte-integrations/connectors/source-microsoft-sharepoint/source_microsoft_sharepoint/source.py
View File

@@ -6,14 +6,17 @@
from typing import Any, Mapping, Optional
from airbyte_cdk import AdvancedAuth, ConfiguredAirbyteCatalog, ConnectorSpecification, OAuthConfigSpecification, TState
from airbyte_cdk.models import AuthFlowType
from airbyte_cdk.models import AuthFlowType, OauthConnectorInputSpecification
from airbyte_cdk.sources.file_based.file_based_source import FileBasedSource
from airbyte_cdk.sources.file_based.stream.cursor.default_file_based_cursor import DefaultFileBasedCursor
from source_microsoft_sharepoint.spec import SourceMicrosoftSharePointSpec
from source_microsoft_sharepoint.stream_reader import SourceMicrosoftSharePointStreamReader
from source_microsoft_sharepoint.utils import PlaceholderUrlBuilder
class SourceMicrosoftSharePoint(FileBasedSource):
SCOPES = ["offline_access", "Files.Read.All"]
def __init__(self, catalog: Optional[ConfiguredAirbyteCatalog], config: Optional[Mapping[str, Any]], state: Optional[TState]):
super().__init__(
stream_reader=SourceMicrosoftSharePointStreamReader(),
@@ -28,6 +31,43 @@ class SourceMicrosoftSharePoint(FileBasedSource):
"""
Returns the specification describing what fields can be configured by a user when setting up a file-based source.
"""
consent_url = (
PlaceholderUrlBuilder()
.set_scheme("https")
.set_host("login.microsoftonline.com")
.set_path("/{{tenant_id}}/oauth2/v2.0/authorize")
.add_key_value_placeholder_param("client_id")
.add_key_value_placeholder_param("redirect_uri")
.add_key_value_placeholder_param("state")
.add_key_value_placeholder_param("scope")
.add_literal_param("response_type=code")
.build()
)
access_token_url = (
PlaceholderUrlBuilder()
.set_scheme("https")
.set_host("login.microsoftonline.com")
.set_path("/{{tenant_id}}/oauth2/v2.0/token")
.build()
)
scopes = " ".join(SourceMicrosoftSharePoint.SCOPES)
oauth_connector_input_specification = OauthConnectorInputSpecification(
consent_url=consent_url,
access_token_url=access_token_url,
access_token_headers={
"Content-Type": "application/x-www-form-urlencoded",
},
access_token_params={
"code": "{{ auth_code_value }}",
"client_id": "{{ client_id_value }}",
"redirect_uri": "{{ redirect_uri_value }}",
"client_secret": "{{ client_secret_value }}",
"grant_type": "authorization_code",
},
scope=scopes,
)
return ConnectorSpecification(
documentationUrl=self.spec_class.documentation_url(),
@@ -37,10 +77,17 @@ class SourceMicrosoftSharePoint(FileBasedSource):
predicate_key=["credentials", "auth_type"],
predicate_value="Client",
oauth_config_specification=OAuthConfigSpecification(
oauth_connector_input_specification=oauth_connector_input_specification,
complete_oauth_output_specification={
"type": "object",
"additionalProperties": False,
"properties": {"refresh_token": {"type": "string", "path_in_connector_config": ["credentials", "refresh_token"]}},
"properties": {
"refresh_token": {
"type": "string",
"path_in_connector_config": ["credentials", "refresh_token"],
"path_in_oauth_response": ["refresh_token"],
}
},
},
complete_oauth_server_input_specification={
"type": "object",

57
airbyte-integrations/connectors/source-microsoft-sharepoint/source_microsoft_sharepoint/utils.py
View File

@@ -93,3 +93,60 @@ def execute_query_with_retry(obj, max_retries=5, initial_retry_after=5, max_retr
message = f"Maximum number of retries of {max_retries} exceeded for execute_query."
raise AirbyteTracedException(message, message, failure_type=FailureType.system_error)
class PlaceholderUrlBuilder:
"""
A basic builder that constructs a URL with placeholder parameters like:
{{client_id_param}}
{{redirect_uri_param}}
etc.
These placeholders will be replaced later during Oauth flow.
"""
def __init__(self):
self._scheme = "https"
self._host = ""
self._path = ""
self._segments = [] # Each segment will become part of the query string
def set_scheme(self, scheme: str):
self._scheme = scheme
return self
def set_host(self, host: str):
self._host = host
return self
def set_path(self, path: str):
# If you want a leading slash, you can incorporate it here or let caller handle it
self._path = path
return self
def add_key_value_placeholder_param(self, param_name: str):
"""
Example:
add_key_value_placeholder_param("client_id")
=> produces "{{client_id_param}}" in the final query string.
"""
placeholder = f"{{{{{param_name}_param}}}}" # e.g. "{{client_id_param}}"
self._segments.append(placeholder)
return self
def add_literal_param(self, literal: str):
"""
If you want to add a static string like "response_type=code"
or "access_type=offline", you can do so here.
"""
self._segments.append(literal)
return self
def build(self) -> str:
"""
Joins the query segments with '&' and forms the complete URL.
Example final output might look like:
https://accounts.google.com/o/oauth2/v2/auth?{{client_id_param}}&{{redirect_uri_param}}&response_type=code
"""
query_string = "&".join(self._segments)
query_string = "?" + query_string if query_string else ""
return f"{self._scheme}://{self._host}{self._path}{query_string}"

95
airbyte-integrations/connectors/source-microsoft-sharepoint/unit_tests/test_utils.py
View File

@@ -4,9 +4,10 @@
from datetime import datetime, timedelta
from http import HTTPStatus
from unittest.mock import Mock, patch
from urllib.parse import parse_qs, urlparse
import pytest
from source_microsoft_sharepoint.utils import execute_query_with_retry, filter_http_urls
from source_microsoft_sharepoint.utils import PlaceholderUrlBuilder, execute_query_with_retry, filter_http_urls
from airbyte_cdk import AirbyteTracedException
@@ -91,3 +92,95 @@ def test_filter_http_urls():
assert len(filtered_files) == 2
mock_logger.error.assert_called_once_with("Cannot open file file3.txt. The URL returned by SharePoint is not secure.")
@pytest.mark.parametrize(
"steps, expected_url",
[
(
# steps is a list of (method_name, argument)
[
("set_scheme", "https"),
("set_host", "accounts.google.com"),
("set_path", "/o/oauth2/v2/auth"),
("add_key_value_placeholder_param", "client_id"),
("add_key_value_placeholder_param", "redirect_uri"),
("add_literal_param", "response_type=code"),
("add_key_value_placeholder_param", "scope"),
("add_literal_param", "access_type=offline"),
("add_key_value_placeholder_param", "state"),
("add_literal_param", "include_granted_scopes=true"),
("add_literal_param", "prompt=consent"),
],
# And this is the expected URL for these steps
"https://accounts.google.com/o/oauth2/v2/auth?{{client_id_param}}&{{redirect_uri_param}}&response_type=code&{{scope_param}}&access_type=offline&{{state_param}}&include_granted_scopes=true&prompt=consent",
),
(
# steps is a list of (method_name, argument)
[
("set_scheme", "https"),
("set_host", "login.microsoftonline.com"),
("set_path", "/TENANT_ID/oauth2/v2.0/authorize"),
("add_key_value_placeholder_param", "client_id"),
("add_key_value_placeholder_param", "redirect_uri"),
("add_key_value_placeholder_param", "state"),
("add_key_value_placeholder_param", "scope"),
("add_literal_param", "response_type=code"),
],
# And this is the expected URL for these steps
"https://login.microsoftonline.com/TENANT_ID/oauth2/v2.0/authorize?{{client_id_param}}&{{redirect_uri_param}}&{{state_param}}&{{scope_param}}&response_type=code",
),
(
[
("set_scheme", "https"),
("set_host", "oauth2.googleapis.com"),
("set_path", "/token"),
("add_key_value_placeholder_param", "client_id"),
("add_key_value_placeholder_param", "client_secret"),
("add_key_value_placeholder_param", "auth_code"),
("add_key_value_placeholder_param", "redirect_uri"),
("add_literal_param", "grant_type=authorization_code"),
],
"https://oauth2.googleapis.com/token?{{client_id_param}}&{{client_secret_param}}&{{auth_code_param}}&{{redirect_uri_param}}&grant_type=authorization_code",
),
(
[
("set_scheme", "https"),
("set_host", "login.microsoftonline.com"),
("set_path", "/TENANT_ID/oauth2/v2.0/token"),
("add_key_value_placeholder_param", "client_id"),
("add_key_value_placeholder_param", "auth_code"),
("add_key_value_placeholder_param", "redirect_uri"),
("add_key_value_placeholder_param", "client_secret"),
("add_literal_param", "grant_type=authorization_code"),
],
"https://login.microsoftonline.com/TENANT_ID/oauth2/v2.0/token?{{client_id_param}}&{{auth_code_param}}&{{redirect_uri_param}}&{{client_secret_param}}&grant_type=authorization_code",
),
],
)
def test_url_builder_for_key_pair_value_pair(steps, expected_url):
"""
Demonstrates building a URL in a specified order,
using a list of (method_name, argument) tuples.
"""
builder = PlaceholderUrlBuilder()
# We'll call each builder method in the order given by steps
for method_name, arg in steps:
if method_name == "set_scheme":
builder.set_scheme(arg)
elif method_name == "set_host":
builder.set_host(arg)
elif method_name == "set_path":
builder.set_path(arg)
elif method_name == "add_key_value_placeholder_param":
builder.add_key_value_placeholder_param(arg)
elif method_name == "add_literal_param":
builder.add_literal_param(arg)
else:
raise ValueError(f"Unknown method_name: {method_name}")
# Finally, build the URL and compare to expected
url = builder.build()
assert url == expected_url, f"Expected {expected_url}, but got {url}"