🎉 CI secrets: migration to GSM (#7528)
* fix 404 responses for the ticket_comments stream * add unit test * add unit test * add oauth2 access token * Update airbyte-integrations/connectors/source-zendesk-support/source_zendesk_support/streams.py Co-authored-by: George Claireaux <george@claireaux.co.uk> * switching among auth methods * update spec file * update CI secrets logic * update CI secrets logic * remove debug data * add a debug message * fix json convertation * fix json convertation * support one secret by several connectors * Update tools/bin/ci_credentials.sh Co-authored-by: Sherif A. Nada <snadalive@gmail.com> * Update tools/bin/ci_credentials.sh Co-authored-by: LiRen Tu <tuliren@gmail.com> * update function names * update docs * reset failed changes * update json set value * update secrets json key * Update tools/bin/ci_credentials.sh * add JSON validation * Update docs/connector-development/README.md Co-authored-by: Sherif A. Nada <snadalive@gmail.com> * update doc * Update README.md * rename the function write_standard_creds Co-authored-by: Maksym Pavlenok <maksym.pavlenok@globallogic.com> Co-authored-by: George Claireaux <george@claireaux.co.uk> Co-authored-by: Sherif A. Nada <snadalive@gmail.com> Co-authored-by: LiRen Tu <tuliren@gmail.com>
This commit is contained in:
Maksym Pavlenok
138
.github/workflows/publish-command.yml
vendored
138
.github/workflows/publish-command.yml
vendored
@@ -70,142 +70,10 @@ jobs:
|
||||
- name: Install Pyenv
|
||||
run: python3 -m pip install virtualenv==16.7.9 --user
|
||||
- name: Write Integration Test Credentials # TODO DRY this with test-command.yml
|
||||
run: ./tools/bin/ci_credentials.sh
|
||||
run: ./tools/bin/ci_credentials.sh ${{ github.event.inputs.connector }}
|
||||
env:
|
||||
AMAZON_SELLER_PARTNER_TEST_CREDS: ${{ secrets.AMAZON_SELLER_PARTNER_TEST_CREDS }}
|
||||
AMAZON_ADS_TEST_CREDS: ${{ secrets.AMAZON_ADS_TEST_CREDS }}
|
||||
AMPLITUDE_INTEGRATION_TEST_CREDS: ${{ secrets.AMPLITUDE_INTEGRATION_TEST_CREDS }}
|
||||
AWS_S3_INTEGRATION_TEST_CREDS: ${{ secrets.AWS_S3_INTEGRATION_TEST_CREDS }}
|
||||
AWS_REDSHIFT_INTEGRATION_TEST_CREDS: ${{ secrets.AWS_REDSHIFT_INTEGRATION_TEST_CREDS }}
|
||||
AWS_ORACLE_INTEGRATION_TEST_CREDS: ${{ secrets.AWS_ORACLE_INTEGRATION_TEST_CREDS }}
|
||||
SOURCE_AWS_CLOUDTRAIL_CREDS: ${{ secrets.SOURCE_AWS_CLOUDTRAIL_CREDS }}
|
||||
AZURE_STORAGE_INTEGRATION_TEST_CREDS: ${{ secrets.AZURE_STORAGE_INTEGRATION_TEST_CREDS }}
|
||||
BIGQUERY_INTEGRATION_TEST_CREDS: ${{ secrets.BIGQUERY_INTEGRATION_TEST_CREDS }}
|
||||
BIGQUERY_DENORMALIZED_INTEGRATION_TEST_CREDS: ${{ secrets.BIGQUERY_DENORMALIZED_INTEGRATION_TEST_CREDS }}
|
||||
SOURCE_BING_ADS_CREDS: ${{ secrets.SOURCE_BING_ADS_CREDS }}
|
||||
BIGQUERY_TEST_CREDS: ${{ secrets.BIGQUERY_TEST_CREDS }}
|
||||
BRAINTREE_TEST_CREDS: ${{ secrets.BRAINTREE_TEST_CREDS }}
|
||||
CART_TEST_CREDS: ${{ secrets.CART_TEST_CREDS }}
|
||||
CHARGEBEE_INTEGRATION_TEST_CREDS: ${{ secrets.CHARGEBEE_INTEGRATION_TEST_CREDS }}
|
||||
DESTINATION_POSTGRES_SSH_KEY_TEST_CREDS: ${{ secrets.DESTINATION_POSTGRES_SSH_KEY_TEST_CREDS }}
|
||||
DESTINATION_POSTGRES_SSH_PWD_TEST_CREDS: ${{ secrets.DESTINATION_POSTGRES_SSH_PWD_TEST_CREDS }}
|
||||
DESTINATION_PUBSUB_TEST_CREDS: ${{ secrets.DESTINATION_PUBSUB_TEST_CREDS }}
|
||||
DESTINATION_KEEN_TEST_CREDS: ${{ secrets.DESTINATION_KEEN_TEST_CREDS }}
|
||||
DESTINATION_KVDB_TEST_CREDS: ${{ secrets.DESTINATION_KVDB_TEST_CREDS }}
|
||||
DRIFT_INTEGRATION_TEST_CREDS: ${{ secrets.DRIFT_INTEGRATION_TEST_CREDS }}
|
||||
SOURCE_DIXA_TEST_CREDS: ${{ secrets.SOURCE_DIXA_TEST_CREDS }}
|
||||
EXCHANGE_RATES_TEST_CREDS: ${{ secrets.EXCHANGE_RATES_TEST_CREDS }}
|
||||
FACEBOOK_MARKETING_TEST_INTEGRATION_CREDS: ${{ secrets.FACEBOOK_MARKETING_TEST_INTEGRATION_CREDS }}
|
||||
FACEBOOK_PAGES_INTEGRATION_TEST_CREDS: ${{ secrets.FACEBOOK_PAGES_INTEGRATION_TEST_CREDS }}
|
||||
FILE_SECURE_HTTPS_TEST_CREDS: ${{ secrets.FILE_SECURE_HTTPS_TEST_CREDS }}
|
||||
FRESHDESK_TEST_CREDS: ${{ secrets.FRESHDESK_TEST_CREDS }}
|
||||
GITLAB_INTEGRATION_TEST_CREDS: ${{ secrets.GITLAB_INTEGRATION_TEST_CREDS }}
|
||||
GH_NATIVE_INTEGRATION_TEST_CREDS: ${{ secrets.GH_NATIVE_INTEGRATION_TEST_CREDS }}
|
||||
GOOGLE_ADS_TEST_CREDS: ${{ secrets.GOOGLE_ADS_TEST_CREDS }}
|
||||
GOOGLE_ANALYTICS_V4_TEST_CREDS: ${{ secrets.GOOGLE_ANALYTICS_V4_TEST_CREDS }}
|
||||
GOOGLE_ANALYTICS_V4_TEST_CREDS_SRV_ACC: ${{ secrets.GOOGLE_ANALYTICS_V4_TEST_CREDS_SRV_ACC }}
|
||||
GOOGLE_ANALYTICS_V4_TEST_CREDS_OLD: ${{ secrets.GOOGLE_ANALYTICS_V4_TEST_CREDS_OLD }}
|
||||
GOOGLE_CLOUD_STORAGE_TEST_CREDS: ${{ secrets.GOOGLE_CLOUD_STORAGE_TEST_CREDS }}
|
||||
GOOGLE_DIRECTORY_TEST_CREDS: ${{ secrets.GOOGLE_DIRECTORY_TEST_CREDS }}
|
||||
GOOGLE_DIRECTORY_TEST_CREDS_OAUTH: ${{ secrets.GOOGLE_DIRECTORY_TEST_CREDS_OAUTH }}
|
||||
GOOGLE_SEARCH_CONSOLE_CDK_TEST_CREDS: ${{ secrets.GOOGLE_SEARCH_CONSOLE_CDK_TEST_CREDS }}
|
||||
GOOGLE_SEARCH_CONSOLE_CDK_TEST_CREDS_SRV_ACC: ${{ secrets.GOOGLE_SEARCH_CONSOLE_CDK_TEST_CREDS_SRV_ACC }}
|
||||
GOOGLE_SHEETS_TESTS_CREDS: ${{ secrets.GOOGLE_SHEETS_TESTS_CREDS }}
|
||||
GOOGLE_SHEETS_TESTS_CREDS_SRV_ACC: ${{ secrets.GOOGLE_SHEETS_TESTS_CREDS_SRV_ACC }}
|
||||
GOOGLE_SHEETS_TESTS_CREDS_OLD: ${{ secrets.GOOGLE_SHEETS_TESTS_CREDS_OLD }}
|
||||
GOOGLE_WORKSPACE_ADMIN_REPORTS_TEST_CREDS: ${{ secrets.GOOGLE_WORKSPACE_ADMIN_REPORTS_TEST_CREDS }}
|
||||
GREENHOUSE_TEST_CREDS: ${{ secrets.GREENHOUSE_TEST_CREDS }}
|
||||
GREENHOUSE_TEST_CREDS_LIMITED: ${{ secrets.GREENHOUSE_TEST_CREDS_LIMITED }}
|
||||
HARVEST_INTEGRATION_TESTS_CREDS: ${{ secrets.HARVEST_INTEGRATION_TESTS_CREDS }}
|
||||
HUBSPOT_INTEGRATION_TESTS_CREDS: ${{ secrets.HUBSPOT_INTEGRATION_TESTS_CREDS }}
|
||||
HUBSPOT_INTEGRATION_TESTS_CREDS_OAUTH: ${{ secrets.HUBSPOT_INTEGRATION_TESTS_CREDS_OAUTH }}
|
||||
INSTAGRAM_INTEGRATION_TESTS_CREDS: ${{ secrets.INSTAGRAM_INTEGRATION_TESTS_CREDS }}
|
||||
INTERCOM_INTEGRATION_TEST_CREDS: ${{ secrets.INTERCOM_INTEGRATION_TEST_CREDS }}
|
||||
INTERCOM_INTEGRATION_OAUTH_TEST_CREDS: ${{ secrets.INTERCOM_INTEGRATION_OAUTH_TEST_CREDS }}
|
||||
ITERABLE_INTEGRATION_TEST_CREDS: ${{ secrets.ITERABLE_INTEGRATION_TEST_CREDS }}
|
||||
JIRA_INTEGRATION_TEST_CREDS: ${{ secrets.JIRA_INTEGRATION_TEST_CREDS }}
|
||||
KLAVIYO_TEST_CREDS: ${{ secrets.KLAVIYO_TEST_CREDS }}
|
||||
LEVER_HIRING_INTEGRATION_TEST_CREDS: ${{ secrets.LEVER_HIRING_INTEGRATION_TEST_CREDS }}
|
||||
LOOKER_INTEGRATION_TEST_CREDS: ${{ secrets.LOOKER_INTEGRATION_TEST_CREDS }}
|
||||
MAILCHIMP_TEST_CREDS: ${{ secrets.MAILCHIMP_TEST_CREDS }}
|
||||
MICROSOFT_TEAMS_TEST_CREDS: ${{ secrets.MICROSOFT_TEAMS_TEST_CREDS }}
|
||||
MIXPANEL_INTEGRATION_TEST_CREDS: ${{ secrets.MIXPANEL_INTEGRATION_TEST_CREDS }}
|
||||
MSSQL_RDS_TEST_CREDS: ${{ secrets.MSSQL_RDS_TEST_CREDS }}
|
||||
PAYPAL_TRANSACTION_CREDS: ${{ secrets.SOURCE_PAYPAL_TRANSACTION_CREDS }}
|
||||
PINTEREST_TEST_CREDS: ${{ secrets.PINTEREST_TEST_CREDS }}
|
||||
POSTGRES_SSH_KEY_TEST_CREDS: ${{ secrets.POSTGRES_SSH_KEY_TEST_CREDS }}
|
||||
POSTGRES_SSH_PWD_TEST_CREDS: ${{ secrets.POSTGRES_SSH_PWD_TEST_CREDS }}
|
||||
MYSQL_SSH_KEY_TEST_CREDS: ${{ secrets.MYSQL_SSH_KEY_TEST_CREDS }}
|
||||
MYSQL_SSH_PWD_TEST_CREDS: ${{ secrets.MYSQL_SSH_PWD_TEST_CREDS }}
|
||||
POSTHOG_TEST_CREDS: ${{ secrets.POSTHOG_TEST_CREDS }}
|
||||
PIPEDRIVE_INTEGRATION_TESTS_CREDS: ${{ secrets.PIPEDRIVE_INTEGRATION_TESTS_CREDS }}
|
||||
PIPEDRIVE_INTEGRATION_TESTS_CREDS_OAUTH: ${{ secrets.PIPEDRIVE_INTEGRATION_TESTS_CREDS_OAUTH }}
|
||||
PIPEDRIVE_INTEGRATION_TESTS_CREDS_OLD: ${{ secrets.PIPEDRIVE_INTEGRATION_TESTS_CREDS_OLD }}
|
||||
RECHARGE_INTEGRATION_TEST_CREDS: ${{ secrets.RECHARGE_INTEGRATION_TEST_CREDS }}
|
||||
QUICKBOOKS_TEST_CREDS: ${{ secrets.QUICKBOOKS_TEST_CREDS }}
|
||||
SALESFORCE_BULK_INTEGRATION_TESTS_CREDS: ${{ secrets.SALESFORCE_BULK_INTEGRATION_TESTS_CREDS }}
|
||||
SALESFORCE_INTEGRATION_TESTS_CREDS: ${{ secrets.SALESFORCE_INTEGRATION_TESTS_CREDS }}
|
||||
SENDGRID_INTEGRATION_TEST_CREDS: ${{ secrets.SENDGRID_INTEGRATION_TEST_CREDS }}
|
||||
SHOPIFY_INTEGRATION_TEST_CREDS: ${{ secrets.SHOPIFY_INTEGRATION_TEST_CREDS }}
|
||||
SHOPIFY_INTEGRATION_TEST_OAUTH_CREDS: ${{ secrets.SHOPIFY_INTEGRATION_TEST_OAUTH_CREDS }}
|
||||
SOURCE_ASANA_TEST_CREDS: ${{ secrets.SOURCE_ASANA_TEST_CREDS }}
|
||||
SOURCE_OKTA_TEST_CREDS: ${{ secrets.SOURCE_OKTA_TEST_CREDS }}
|
||||
SOURCE_SLACK_TEST_CREDS: ${{ secrets.SOURCE_SLACK_TEST_CREDS }}
|
||||
SOURCE_SLACK_OAUTH_TEST_CREDS: ${{ secrets.SOURCE_SLACK_OAUTH_TEST_CREDS }}
|
||||
SOURCE_US_CENSUS_TEST_CREDS: ${{ secrets.SOURCE_US_CENSUS_TEST_CREDS }}
|
||||
SMARTSHEETS_TEST_CREDS: ${{ secrets.SMARTSHEETS_TEST_CREDS }}
|
||||
SOURCE_SNAPCHAT_MARKETING_CREDS: ${{ secrets.SOURCE_SNAPCHAT_MARKETING_CREDS }}
|
||||
SNOWFLAKE_INTEGRATION_TEST_CREDS: ${{ secrets.SNOWFLAKE_INTEGRATION_TEST_CREDS }}
|
||||
SNOWFLAKE_S3_COPY_INTEGRATION_TEST_CREDS: ${{ secrets.SNOWFLAKE_S3_COPY_INTEGRATION_TEST_CREDS }}
|
||||
SNOWFLAKE_GCS_COPY_INTEGRATION_TEST_CREDS: ${{ secrets.SNOWFLAKE_GCS_COPY_INTEGRATION_TEST_CREDS }}
|
||||
SOURCE_SQUARE_CREDS: ${{ secrets.SOURCE_SQUARE_CREDS }}
|
||||
SOURCE_MARKETO_TEST_CREDS: ${{ secrets.SOURCE_MARKETO_TEST_CREDS }}
|
||||
SOURCE_RECURLY_INTEGRATION_TEST_CREDS: ${{ secrets.SOURCE_RECURLY_INTEGRATION_TEST_CREDS }}
|
||||
SOURCE_S3_TEST_CREDS: ${{ secrets.SOURCE_S3_TEST_CREDS }}
|
||||
SOURCE_S3_PARQUET_CREDS: ${{ secrets.SOURCE_S3_PARQUET_CREDS }}
|
||||
SOURCE_SHORTIO_TEST_CREDS: ${{ secrets.SOURCE_SHORTIO_TEST_CREDS }}
|
||||
SOURCE_STRIPE_CREDS: ${{ secrets.SOURCE_STRIPE_CREDS }}
|
||||
STRIPE_INTEGRATION_CONNECTED_ACCOUNT_TEST_CREDS: ${{ secrets.STRIPE_INTEGRATION_CONNECTED_ACCOUNT_TEST_CREDS }}
|
||||
SURVEYMONKEY_TEST_CREDS: ${{ secrets.SURVEYMONKEY_TEST_CREDS }}
|
||||
TEMPO_INTEGRATION_TEST_CREDS: ${{ secrets.TEMPO_INTEGRATION_TEST_CREDS }}
|
||||
TRELLO_TEST_CREDS: ${{ secrets.TRELLO_TEST_CREDS }}
|
||||
TWILIO_TEST_CREDS: ${{ secrets.TWILIO_TEST_CREDS }}
|
||||
SOURCE_TYPEFORM_CREDS: ${{ secrets.SOURCE_TYPEFORM_CREDS }}
|
||||
ZENDESK_CHAT_INTEGRATION_TEST_CREDS: ${{ secrets.ZENDESK_CHAT_INTEGRATION_TEST_CREDS }}
|
||||
ZENDESK_SUNSHINE_TEST_CREDS: ${{ secrets.ZENDESK_SUNSHINE_TEST_CREDS }}
|
||||
ZENDESK_TALK_TEST_CREDS: ${{ secrets.ZENDESK_TALK_TEST_CREDS }}
|
||||
ZENDESK_SUPPORT_TEST_CREDS: ${{ secrets.ZENDESK_SUPPORT_TEST_CREDS }}
|
||||
ZENDESK_SUPPORT_OAUTH_TEST_CREDS: ${{ secrets.ZENDESK_SUPPORT_OAUTH_TEST_CREDS }}
|
||||
ZOOM_INTEGRATION_TEST_CREDS: ${{ secrets.ZOOM_INTEGRATION_TEST_CREDS }}
|
||||
PLAID_INTEGRATION_TEST_CREDS: ${{ secrets.PLAID_INTEGRATION_TEST_CREDS }}
|
||||
DESTINATION_S3_INTEGRATION_TEST_CREDS: ${{ secrets.DESTINATION_S3_INTEGRATION_TEST_CREDS }}
|
||||
DESTINATION_AZURE_BLOB_CREDS: ${{ secrets.DESTINATION_AZURE_BLOB_CREDS }}
|
||||
DESTINATION_GCS_CREDS: ${{ secrets.DESTINATION_GCS_CREDS }}
|
||||
APIFY_INTEGRATION_TEST_CREDS: ${{ secrets.APIFY_INTEGRATION_TEST_CREDS }}
|
||||
DESTINATION_DYNAMODB_TEST_CREDS: ${{ secrets.DESTINATION_DYNAMODB_TEST_CREDS }}
|
||||
SOURCE_ZUORA_TEST_CREDS: ${{ secrets.SOURCE_ZUORA_TEST_CREDS }}
|
||||
SOURCE_CLOSE_COM_CREDS: ${{ secrets.SOURCE_CLOSE_COM_CREDS }}
|
||||
SOURCE_BAMBOO_HR_CREDS: ${{ secrets.SOURCE_BAMBOO_HR_CREDS }}
|
||||
SOURCE_LINKEDIN_ADS_TEST_CREDS: ${{ secrets.SOURCE_LINKEDIN_ADS_TEST_CREDS }}
|
||||
SOURCE_BIGCOMMERCE_CREDS: ${{ secrets.SOURCE_BIGCOMMERCE_CREDS }}
|
||||
SOURCE_TIKTOK_MARKETING_TEST_CREDS: ${{ secrets.SOURCE_TIKTOK_MARKETING_TEST_CREDS }}
|
||||
SOURCE_TIKTOK_MARKETING_PROD_TEST_CREDS: ${{ secrets.SOURCE_TIKTOK_MARKETING_PROD_TEST_CREDS }}
|
||||
DESTINATION_DATABRICKS_CREDS: ${{ secrets.DESTINATION_DATABRICKS_CREDS }}
|
||||
MONGODB_TEST_CREDS: ${{ secrets.MONGODB_TEST_CREDS }}
|
||||
SOURCE_ONESIGNAL_TEST_CREDS: ${{ secrets.SOURCE_ONESIGNAL_TEST_CREDS }}
|
||||
SOURCE_SALESLOFT_TEST_CREDS: ${{ secrets.SOURCE_SALESLOFT_TEST_CREDS }}
|
||||
SOURCE_CONFLUENCE_TEST_CREDS: ${{ secrets.SOURCE_CONFLUENCE_TEST_CREDS }}
|
||||
SOURCE_AMAZON_SQS_TEST_CREDS: ${{ secrets.SOURCE_AMAZON_SQS_TEST_CREDS }}
|
||||
SOURCE_FRESHSERVICE_TEST_CREDS: ${{ secrets.SOURCE_FRESHSERVICE_TEST_CREDS }}
|
||||
SOURCE_LEMLIST_TEST_CREDS: ${{ secrets.SOURCE_LEMLIST_TEST_CREDS }}
|
||||
SOURCE_STRAVA_TEST_CREDS: ${{ secrets.SOURCE_STRAVA_TEST_CREDS }}
|
||||
SOURCE_PAYSTACK_TEST_CREDS: ${{ secrets.SOURCE_PAYSTACK_TEST_CREDS }}
|
||||
SOURCE_DELIGHTED_TEST_CREDS: ${{ secrets.SOURCE_DELIGHTED_TEST_CREDS }}
|
||||
SOURCE_RETENTLY_TEST_CREDS: ${{ secrets.SOURCE_RETENTLY_TEST_CREDS }}
|
||||
SOURCE_SENTRY_TEST_CREDS: ${{ secrets.SOURCE_SENTRY_TEST_CREDS }}
|
||||
SOURCE_FRESHSALES_TEST_CREDS: ${{ secrets.SOURCE_FRESHSALES_TEST_CREDS }}
|
||||
SOURCE_MONDAY_TEST_CREDS: ${{ secrets.SOURCE_MONDAY_TEST_CREDS }}
|
||||
SOURCE_COMMERCETOOLS_TEST_CREDS: ${{ secrets.SOURCE_COMMERCETOOLS_TEST_CREDS }}
|
||||
GITHUB_PROVIDED_SECRETS_JSON: ${{ toJson(secrets) }}
|
||||
GCP_GSM_CREDENTIALS: ${{ secrets.GCP_GSM_CREDENTIALS }}
|
||||
- run: |
|
||||
echo "$SPEC_CACHE_SERVICE_ACCOUNT_KEY" > spec_cache_key_file.json && docker login -u airbytebot -p ${DOCKER_PASSWORD}
|
||||
./tools/integrations/manage.sh publish airbyte-integrations/${{ github.event.inputs.connector }} ${{ github.event.inputs.run-tests }} --publish_spec_to_cache
|
||||
|
||||
138
.github/workflows/test-command.yml
vendored
138
.github/workflows/test-command.yml
vendored
@@ -65,142 +65,10 @@ jobs:
|
||||
- name: Install Pyenv
|
||||
run: python3 -m pip install virtualenv==16.7.9 --user
|
||||
- name: Write Integration Test Credentials
|
||||
run: ./tools/bin/ci_credentials.sh
|
||||
run: ./tools/bin/ci_credentials.sh ${{ github.event.inputs.connector }}
|
||||
env:
|
||||
AMAZON_SELLER_PARTNER_TEST_CREDS: ${{ secrets.AMAZON_SELLER_PARTNER_TEST_CREDS }}
|
||||
AMAZON_ADS_TEST_CREDS: ${{ secrets.AMAZON_ADS_TEST_CREDS }}
|
||||
AMPLITUDE_INTEGRATION_TEST_CREDS: ${{ secrets.AMPLITUDE_INTEGRATION_TEST_CREDS }}
|
||||
AWS_S3_INTEGRATION_TEST_CREDS: ${{ secrets.AWS_S3_INTEGRATION_TEST_CREDS }}
|
||||
AWS_ORACLE_INTEGRATION_TEST_CREDS: ${{ secrets.AWS_ORACLE_INTEGRATION_TEST_CREDS }}
|
||||
SOURCE_AWS_CLOUDTRAIL_CREDS: ${{ secrets.SOURCE_AWS_CLOUDTRAIL_CREDS }}
|
||||
AWS_REDSHIFT_INTEGRATION_TEST_CREDS: ${{ secrets.AWS_REDSHIFT_INTEGRATION_TEST_CREDS }}
|
||||
AZURE_STORAGE_INTEGRATION_TEST_CREDS: ${{ secrets.AZURE_STORAGE_INTEGRATION_TEST_CREDS }}
|
||||
BIGQUERY_INTEGRATION_TEST_CREDS: ${{ secrets.BIGQUERY_INTEGRATION_TEST_CREDS }}
|
||||
BIGQUERY_DENORMALIZED_INTEGRATION_TEST_CREDS: ${{ secrets.BIGQUERY_DENORMALIZED_INTEGRATION_TEST_CREDS }}
|
||||
SOURCE_BING_ADS_CREDS: ${{ secrets.SOURCE_BING_ADS_CREDS }}
|
||||
BIGQUERY_TEST_CREDS: ${{ secrets.BIGQUERY_TEST_CREDS }}
|
||||
BRAINTREE_TEST_CREDS: ${{ secrets.BRAINTREE_TEST_CREDS }}
|
||||
CART_TEST_CREDS: ${{ secrets.CART_TEST_CREDS }}
|
||||
CHARGEBEE_INTEGRATION_TEST_CREDS: ${{ secrets.CHARGEBEE_INTEGRATION_TEST_CREDS }}
|
||||
DESTINATION_POSTGRES_SSH_KEY_TEST_CREDS: ${{ secrets.DESTINATION_POSTGRES_SSH_KEY_TEST_CREDS }}
|
||||
DESTINATION_POSTGRES_SSH_PWD_TEST_CREDS: ${{ secrets.DESTINATION_POSTGRES_SSH_PWD_TEST_CREDS }}
|
||||
DESTINATION_PUBSUB_TEST_CREDS: ${{ secrets.DESTINATION_PUBSUB_TEST_CREDS }}
|
||||
DESTINATION_KEEN_TEST_CREDS: ${{ secrets.DESTINATION_KEEN_TEST_CREDS }}
|
||||
DESTINATION_KVDB_TEST_CREDS: ${{ secrets.DESTINATION_KVDB_TEST_CREDS }}
|
||||
DRIFT_INTEGRATION_TEST_CREDS: ${{ secrets.DRIFT_INTEGRATION_TEST_CREDS }}
|
||||
SOURCE_DIXA_TEST_CREDS: ${{ secrets.SOURCE_DIXA_TEST_CREDS }}
|
||||
EXCHANGE_RATES_TEST_CREDS: ${{ secrets.EXCHANGE_RATES_TEST_CREDS }}
|
||||
FACEBOOK_MARKETING_TEST_INTEGRATION_CREDS: ${{ secrets.FACEBOOK_MARKETING_TEST_INTEGRATION_CREDS }}
|
||||
FACEBOOK_PAGES_INTEGRATION_TEST_CREDS: ${{ secrets.FACEBOOK_PAGES_INTEGRATION_TEST_CREDS }}
|
||||
FILE_SECURE_HTTPS_TEST_CREDS: ${{ secrets.FILE_SECURE_HTTPS_TEST_CREDS }}
|
||||
FRESHDESK_TEST_CREDS: ${{ secrets.FRESHDESK_TEST_CREDS }}
|
||||
GITLAB_INTEGRATION_TEST_CREDS: ${{ secrets.GITLAB_INTEGRATION_TEST_CREDS }}
|
||||
GH_NATIVE_INTEGRATION_TEST_CREDS: ${{ secrets.GH_NATIVE_INTEGRATION_TEST_CREDS }}
|
||||
GOOGLE_ADS_TEST_CREDS: ${{ secrets.GOOGLE_ADS_TEST_CREDS }}
|
||||
GOOGLE_ANALYTICS_V4_TEST_CREDS: ${{ secrets.GOOGLE_ANALYTICS_V4_TEST_CREDS }}
|
||||
GOOGLE_ANALYTICS_V4_TEST_CREDS_SRV_ACC: ${{ secrets.GOOGLE_ANALYTICS_V4_TEST_CREDS_SRV_ACC }}
|
||||
GOOGLE_ANALYTICS_V4_TEST_CREDS_OLD: ${{ secrets.GOOGLE_ANALYTICS_V4_TEST_CREDS_OLD }}
|
||||
GOOGLE_CLOUD_STORAGE_TEST_CREDS: ${{ secrets.GOOGLE_CLOUD_STORAGE_TEST_CREDS }}
|
||||
GOOGLE_DIRECTORY_TEST_CREDS: ${{ secrets.GOOGLE_DIRECTORY_TEST_CREDS }}
|
||||
GOOGLE_DIRECTORY_TEST_CREDS_OAUTH: ${{ secrets.GOOGLE_DIRECTORY_TEST_CREDS_OAUTH }}
|
||||
GOOGLE_SEARCH_CONSOLE_CDK_TEST_CREDS: ${{ secrets.GOOGLE_SEARCH_CONSOLE_CDK_TEST_CREDS }}
|
||||
GOOGLE_SEARCH_CONSOLE_CDK_TEST_CREDS_SRV_ACC: ${{ secrets.GOOGLE_SEARCH_CONSOLE_CDK_TEST_CREDS_SRV_ACC }}
|
||||
GOOGLE_SHEETS_TESTS_CREDS: ${{ secrets.GOOGLE_SHEETS_TESTS_CREDS }}
|
||||
GOOGLE_SHEETS_TESTS_CREDS_SRV_ACC: ${{ secrets.GOOGLE_SHEETS_TESTS_CREDS_SRV_ACC }}
|
||||
GOOGLE_SHEETS_TESTS_CREDS_OLD: ${{ secrets.GOOGLE_SHEETS_TESTS_CREDS_OLD }}
|
||||
GOOGLE_WORKSPACE_ADMIN_REPORTS_TEST_CREDS: ${{ secrets.GOOGLE_WORKSPACE_ADMIN_REPORTS_TEST_CREDS }}
|
||||
GREENHOUSE_TEST_CREDS: ${{ secrets.GREENHOUSE_TEST_CREDS }}
|
||||
GREENHOUSE_TEST_CREDS_LIMITED: ${{ secrets.GREENHOUSE_TEST_CREDS_LIMITED }}
|
||||
HARVEST_INTEGRATION_TESTS_CREDS: ${{ secrets.HARVEST_INTEGRATION_TESTS_CREDS }}
|
||||
HUBSPOT_INTEGRATION_TESTS_CREDS: ${{ secrets.HUBSPOT_INTEGRATION_TESTS_CREDS }}
|
||||
HUBSPOT_INTEGRATION_TESTS_CREDS_OAUTH: ${{ secrets.HUBSPOT_INTEGRATION_TESTS_CREDS_OAUTH }}
|
||||
INSTAGRAM_INTEGRATION_TESTS_CREDS: ${{ secrets.INSTAGRAM_INTEGRATION_TESTS_CREDS }}
|
||||
INTERCOM_INTEGRATION_TEST_CREDS: ${{ secrets.INTERCOM_INTEGRATION_TEST_CREDS }}
|
||||
INTERCOM_INTEGRATION_OAUTH_TEST_CREDS: ${{ secrets.INTERCOM_INTEGRATION_OAUTH_TEST_CREDS }}
|
||||
ITERABLE_INTEGRATION_TEST_CREDS: ${{ secrets.ITERABLE_INTEGRATION_TEST_CREDS }}
|
||||
JIRA_INTEGRATION_TEST_CREDS: ${{ secrets.JIRA_INTEGRATION_TEST_CREDS }}
|
||||
KLAVIYO_TEST_CREDS: ${{ secrets.KLAVIYO_TEST_CREDS }}
|
||||
SOURCE_ASANA_TEST_CREDS: ${{ secrets.SOURCE_ASANA_TEST_CREDS }}
|
||||
LEVER_HIRING_INTEGRATION_TEST_CREDS: ${{ secrets.LEVER_HIRING_INTEGRATION_TEST_CREDS }}
|
||||
LOOKER_INTEGRATION_TEST_CREDS: ${{ secrets.LOOKER_INTEGRATION_TEST_CREDS }}
|
||||
MAILCHIMP_TEST_CREDS: ${{ secrets.MAILCHIMP_TEST_CREDS }}
|
||||
MICROSOFT_TEAMS_TEST_CREDS: ${{ secrets.MICROSOFT_TEAMS_TEST_CREDS }}
|
||||
MIXPANEL_INTEGRATION_TEST_CREDS: ${{ secrets.MIXPANEL_INTEGRATION_TEST_CREDS }}
|
||||
MSSQL_RDS_TEST_CREDS: ${{ secrets.MSSQL_RDS_TEST_CREDS }}
|
||||
PAYPAL_TRANSACTION_CREDS: ${{ secrets.SOURCE_PAYPAL_TRANSACTION_CREDS }}
|
||||
PINTEREST_TEST_CREDS: ${{ secrets.PINTEREST_TEST_CREDS }}
|
||||
POSTGRES_SSH_KEY_TEST_CREDS: ${{ secrets.POSTGRES_SSH_KEY_TEST_CREDS }}
|
||||
POSTGRES_SSH_PWD_TEST_CREDS: ${{ secrets.POSTGRES_SSH_PWD_TEST_CREDS }}
|
||||
MYSQL_SSH_KEY_TEST_CREDS: ${{ secrets.MYSQL_SSH_KEY_TEST_CREDS }}
|
||||
MYSQL_SSH_PWD_TEST_CREDS: ${{ secrets.MYSQL_SSH_PWD_TEST_CREDS }}
|
||||
POSTHOG_TEST_CREDS: ${{ secrets.POSTHOG_TEST_CREDS }}
|
||||
PIPEDRIVE_INTEGRATION_TESTS_CREDS: ${{ secrets.PIPEDRIVE_INTEGRATION_TESTS_CREDS }}
|
||||
PIPEDRIVE_INTEGRATION_TESTS_CREDS_OAUTH: ${{ secrets.PIPEDRIVE_INTEGRATION_TESTS_CREDS_OAUTH }}
|
||||
PIPEDRIVE_INTEGRATION_TESTS_CREDS_OLD: ${{ secrets.PIPEDRIVE_INTEGRATION_TESTS_CREDS_OLD }}
|
||||
RECHARGE_INTEGRATION_TEST_CREDS: ${{ secrets.RECHARGE_INTEGRATION_TEST_CREDS }}
|
||||
QUICKBOOKS_TEST_CREDS: ${{ secrets.QUICKBOOKS_TEST_CREDS }}
|
||||
SALESFORCE_BULK_INTEGRATION_TESTS_CREDS: ${{ secrets.SALESFORCE_BULK_INTEGRATION_TESTS_CREDS }}
|
||||
SALESFORCE_INTEGRATION_TESTS_CREDS: ${{ secrets.SALESFORCE_INTEGRATION_TESTS_CREDS }}
|
||||
SENDGRID_INTEGRATION_TEST_CREDS: ${{ secrets.SENDGRID_INTEGRATION_TEST_CREDS }}
|
||||
SHOPIFY_INTEGRATION_TEST_CREDS: ${{ secrets.SHOPIFY_INTEGRATION_TEST_CREDS }}
|
||||
SHOPIFY_INTEGRATION_TEST_OAUTH_CREDS: ${{ secrets.SHOPIFY_INTEGRATION_TEST_OAUTH_CREDS }}
|
||||
SOURCE_OKTA_TEST_CREDS: ${{ secrets.SOURCE_OKTA_TEST_CREDS }}
|
||||
SOURCE_SLACK_TEST_CREDS: ${{ secrets.SOURCE_SLACK_TEST_CREDS }}
|
||||
SOURCE_SLACK_OAUTH_TEST_CREDS: ${{ secrets.SOURCE_SLACK_OAUTH_TEST_CREDS }}
|
||||
SOURCE_US_CENSUS_TEST_CREDS: ${{ secrets.SOURCE_US_CENSUS_TEST_CREDS }}
|
||||
SMARTSHEETS_TEST_CREDS: ${{ secrets.SMARTSHEETS_TEST_CREDS }}
|
||||
SOURCE_SNAPCHAT_MARKETING_CREDS: ${{ secrets.SOURCE_SNAPCHAT_MARKETING_CREDS }}
|
||||
SNOWFLAKE_INTEGRATION_TEST_CREDS: ${{ secrets.SNOWFLAKE_INTEGRATION_TEST_CREDS }}
|
||||
SNOWFLAKE_S3_COPY_INTEGRATION_TEST_CREDS: ${{ secrets.SNOWFLAKE_S3_COPY_INTEGRATION_TEST_CREDS }}
|
||||
SNOWFLAKE_GCS_COPY_INTEGRATION_TEST_CREDS: ${{ secrets.SNOWFLAKE_GCS_COPY_INTEGRATION_TEST_CREDS }}
|
||||
SOURCE_SQUARE_CREDS: ${{ secrets.SOURCE_SQUARE_CREDS }}
|
||||
SOURCE_MARKETO_TEST_CREDS: ${{ secrets.SOURCE_MARKETO_TEST_CREDS }}
|
||||
SOURCE_RECURLY_INTEGRATION_TEST_CREDS: ${{ secrets.SOURCE_RECURLY_INTEGRATION_TEST_CREDS }}
|
||||
SOURCE_S3_TEST_CREDS: ${{ secrets.SOURCE_S3_TEST_CREDS }}
|
||||
SOURCE_S3_PARQUET_CREDS: ${{ secrets.SOURCE_S3_PARQUET_CREDS }}
|
||||
SOURCE_SHORTIO_TEST_CREDS: ${{ secrets.SOURCE_SHORTIO_TEST_CREDS }}
|
||||
SOURCE_STRIPE_CREDS: ${{ secrets.SOURCE_STRIPE_CREDS }}
|
||||
STRIPE_INTEGRATION_CONNECTED_ACCOUNT_TEST_CREDS: ${{ secrets.STRIPE_INTEGRATION_CONNECTED_ACCOUNT_TEST_CREDS }}
|
||||
SURVEYMONKEY_TEST_CREDS: ${{ secrets.SURVEYMONKEY_TEST_CREDS }}
|
||||
TEMPO_INTEGRATION_TEST_CREDS: ${{ secrets.TEMPO_INTEGRATION_TEST_CREDS }}
|
||||
TRELLO_TEST_CREDS: ${{ secrets.TRELLO_TEST_CREDS }}
|
||||
TWILIO_TEST_CREDS: ${{ secrets.TWILIO_TEST_CREDS }}
|
||||
SOURCE_TYPEFORM_CREDS: ${{ secrets.SOURCE_TYPEFORM_CREDS }}
|
||||
ZENDESK_CHAT_INTEGRATION_TEST_CREDS: ${{ secrets.ZENDESK_CHAT_INTEGRATION_TEST_CREDS }}
|
||||
ZENDESK_SUNSHINE_TEST_CREDS: ${{ secrets.ZENDESK_SUNSHINE_TEST_CREDS }}
|
||||
ZENDESK_TALK_TEST_CREDS: ${{ secrets.ZENDESK_TALK_TEST_CREDS }}
|
||||
ZENDESK_SUPPORT_TEST_CREDS: ${{ secrets.ZENDESK_SUPPORT_TEST_CREDS }}
|
||||
ZENDESK_SUPPORT_OAUTH_TEST_CREDS: ${{ secrets.ZENDESK_SUPPORT_OAUTH_TEST_CREDS }}
|
||||
ZOOM_INTEGRATION_TEST_CREDS: ${{ secrets.ZOOM_INTEGRATION_TEST_CREDS }}
|
||||
PLAID_INTEGRATION_TEST_CREDS: ${{ secrets.PLAID_INTEGRATION_TEST_CREDS }}
|
||||
DESTINATION_S3_INTEGRATION_TEST_CREDS: ${{ secrets.DESTINATION_S3_INTEGRATION_TEST_CREDS }}
|
||||
DESTINATION_AZURE_BLOB_CREDS: ${{ secrets.DESTINATION_AZURE_BLOB_CREDS }}
|
||||
DESTINATION_GCS_CREDS: ${{ secrets.DESTINATION_GCS_CREDS }}
|
||||
DESTINATION_DYNAMODB_TEST_CREDS: ${{ secrets.DESTINATION_DYNAMODB_TEST_CREDS }}
|
||||
APIFY_INTEGRATION_TEST_CREDS: ${{ secrets.APIFY_INTEGRATION_TEST_CREDS }}
|
||||
SOURCE_ZUORA_TEST_CREDS: ${{ secrets.SOURCE_ZUORA_TEST_CREDS }}
|
||||
SOURCE_CLOSE_COM_CREDS: ${{ secrets.SOURCE_CLOSE_COM_CREDS }}
|
||||
SOURCE_BAMBOO_HR_CREDS: ${{ secrets.SOURCE_BAMBOO_HR_CREDS }}
|
||||
SOURCE_LINKEDIN_ADS_TEST_CREDS: ${{ secrets.SOURCE_LINKEDIN_ADS_TEST_CREDS }}
|
||||
SOURCE_BIGCOMMERCE_CREDS: ${{ secrets.SOURCE_BIGCOMMERCE_CREDS }}
|
||||
SOURCE_TIKTOK_MARKETING_TEST_CREDS: ${{ secrets.SOURCE_TIKTOK_MARKETING_TEST_CREDS }}
|
||||
SOURCE_TIKTOK_MARKETING_PROD_TEST_CREDS: ${{ secrets.SOURCE_TIKTOK_MARKETING_PROD_TEST_CREDS }}
|
||||
DESTINATION_DATABRICKS_CREDS: ${{ secrets.DESTINATION_DATABRICKS_CREDS }}
|
||||
MONGODB_TEST_CREDS: ${{ secrets.MONGODB_TEST_CREDS }}
|
||||
SOURCE_ONESIGNAL_TEST_CREDS: ${{ secrets.SOURCE_ONESIGNAL_TEST_CREDS }}
|
||||
SOURCE_SALESLOFT_TEST_CREDS: ${{ secrets.SOURCE_SALESLOFT_TEST_CREDS }}
|
||||
SOURCE_CONFLUENCE_TEST_CREDS: ${{ secrets.SOURCE_CONFLUENCE_TEST_CREDS }}
|
||||
SOURCE_AMAZON_SQS_TEST_CREDS: ${{ secrets.SOURCE_AMAZON_SQS_TEST_CREDS }}
|
||||
SOURCE_FRESHSERVICE_TEST_CREDS: ${{ secrets.SOURCE_FRESHSERVICE_TEST_CREDS }}
|
||||
SOURCE_LEMLIST_TEST_CREDS: ${{ secrets.SOURCE_LEMLIST_TEST_CREDS }}
|
||||
SOURCE_STRAVA_TEST_CREDS: ${{ secrets.SOURCE_STRAVA_TEST_CREDS }}
|
||||
SOURCE_PAYSTACK_TEST_CREDS: ${{ secrets.SOURCE_PAYSTACK_TEST_CREDS }}
|
||||
SOURCE_DELIGHTED_TEST_CREDS: ${{ secrets.SOURCE_DELIGHTED_TEST_CREDS }}
|
||||
SOURCE_RETENTLY_TEST_CREDS: ${{ secrets.SOURCE_RETENTLY_TEST_CREDS }}
|
||||
SOURCE_SENTRY_TEST_CREDS: ${{ secrets.SOURCE_SENTRY_TEST_CREDS }}
|
||||
SOURCE_FRESHSALES_TEST_CREDS: ${{ secrets.SOURCE_FRESHSALES_TEST_CREDS }}
|
||||
SOURCE_MONDAY_TEST_CREDS: ${{ secrets.SOURCE_MONDAY_TEST_CREDS }}
|
||||
SOURCE_COMMERCETOOLS_TEST_CREDS: ${{ secrets.SOURCE_COMMERCETOOLS_TEST_CREDS }}
|
||||
GITHUB_PROVIDED_SECRETS_JSON: ${{ toJson(secrets) }}
|
||||
GCP_GSM_CREDENTIALS: ${{ secrets.GCP_GSM_CREDENTIALS }}
|
||||
- run: |
|
||||
./tools/bin/ci_integration_test.sh ${{ github.event.inputs.connector }}
|
||||
name: test ${{ github.event.inputs.connector }}
|
||||
|
||||
@@ -130,10 +130,23 @@ Once you've finished iterating on the changes to a connector as specified in its
|
||||
## Using credentials in CI
|
||||
|
||||
In order to run integration tests in CI, you'll often need to inject credentials into CI. There are a few steps for doing this:
|
||||
1. **Place the credentials into Google Secret Manager(GSM)**: Airbyte uses a project 'Google Secret Manager' service as the source of truth for all CI secrets. Place the credentials **exactly as they should be used by the connector** into a GSM secret [here](https://console.cloud.google.com/security/secret-manager?referrer=search&orgonly=true&project=dataline-integration-testing&supportedpurview=organizationId) i.e.: it should basically be a copy paste of the `config.json` passed into a connector via the `--config` flag. We use the following naming pattern: `SECRET_<capital source OR destination name>_CREDS` e.g: `SECRET_SOURCE-S3_CREDS` or `SECRET_DESTINATION-SNOWFLAKE_CREDS`.
|
||||
2. **Add the GSM secret's labels**:
|
||||
* `connector` (required) -- unique connector's name or set of connectors' names with '_' as delimiter i.e.: `connector=source-s3`, `connector=destination-snowflake`
|
||||
* `filename` (optional) -- custom target secret file. Unfortunately Google doesn't use '.' into labels' values and so Airbyte CI scripts will add '.json' to the end automatically. By default secrets will be saved to `./secrets/config.json` i.e: `filename=config_auth` => `secrets/config_auth.json`
|
||||
3. That should be it.
|
||||
|
||||
1. **Place the credentials into Lastpass**: Airbyte uses a shared Lastpass account as the source of truth for all secrets. Place the credentials **exactly as they should be used by the connector** into a secure note i.e: it should basically be a copy paste of the `config.json` passed into a connector via the `--config` flag. We use the following naming pattern: `<source OR destination> <name> creds` e.g: `source google adwords creds` or `destination snowflake creds`.
|
||||
2. **Add the credentials to Github Secrets**: To inject credentials into a CI workflow, the first step is to add it to Github Secrets, specifically within the ["more-secrets" environment](https://github.com/airbytehq/airbyte/settings/environments/276695501/edit). Admin access to the Airbyte repo is required to do this. All Airbyte engineers have admin access and should be able to do this themselves. External contributors or contractors will need to request this from their team lead or project manager who should have admin access. Follow the same naming pattern as all the other secrets e.g: if you are placing credentials for source google adwords, name the secret `SOURCE_GOOGLE_ADWORDS_CREDS`. After doing this step, the secret will be available in the relevant Github workflows using the workflow secrets syntax.
|
||||
3. **Inject the credentials into test and publish CI workflows**: edit the files `.github/workflows/publish-command.yml` and `.github/workflows/test-command.yml` to inject the secret into the CI run. This will make these secrets available to the `/test` and `/publish` commands.
|
||||
4. **During CI, write the secret from env variables to the connector directory**: edit `tools/bin/ci_credentials.sh` to write the secret into the `secrets/` directory of the relevant connector.
|
||||
5. That should be it.
|
||||
#### Access CI secrets on GSM
|
||||
Access to GSM storage is limited to Airbyte employees. To give an employee permissions to the project:
|
||||
1. Go to the permissions' [page](https://console.cloud.google.com/iam-admin/iam?project=dataline-integration-testing)
|
||||
2. Add a new principal to `dataline-integration-testing`:
|
||||
- input their login email
|
||||
- select the role `Development_CI_Secrets`
|
||||
3. Save
|
||||
|
||||
#### How to migrate to the new secrets' logic:
|
||||
1. Create all necessary secrets according to the instructions above
|
||||
2. Remove all lines with old connector's Github secrets from this file: tools/bin/ci_credentials.sh
|
||||
3. Remove all old secrets from Github repository secrets.
|
||||
4. That should be it.
|
||||
|
||||
|
||||
@@ -1,16 +1,54 @@
|
||||
#!/usr/bin/env bash
|
||||
|
||||
. tools/lib/lib.sh
|
||||
. tools/lib/gcp-token.sh
|
||||
|
||||
set -e
|
||||
|
||||
function write_standard_creds() {
|
||||
# all secrets will be loaded if the second argument is not present
|
||||
CONNECTOR_FULLNAME=${1:-all}
|
||||
CONNECTOR_NAME=`echo ${CONNECTOR_FULLNAME} | rev | cut -d'/' -f1 | rev`
|
||||
|
||||
GSM_SCOPES="https://www.googleapis.com/auth/cloud-platform"
|
||||
|
||||
# If a secret is available in both Github and GSM, then the GSM secret is used, otherwise Github.
|
||||
|
||||
declare -A SECRET_MAP
|
||||
|
||||
|
||||
function read_secrets() {
|
||||
local connector_name=$1
|
||||
local creds=$2
|
||||
local cred_filename=${3:-config.json}
|
||||
local secrets_provider_name=${4:-github}
|
||||
|
||||
[ -z "$connector_name" ] && error "Empty connector name"
|
||||
[ -z "$creds" ] && error "Creds not set for $connector_name"
|
||||
[ -z "$creds" ] && echo "!!!!!Creds not set for the connector $connector_name from ${secrets_provider_name}"
|
||||
|
||||
if [[ $CONNECTOR_NAME != "all" && ${connector_name} != ${CONNECTOR_NAME} ]]; then
|
||||
return 0
|
||||
fi
|
||||
local key="${connector_name}#${cred_filename}"
|
||||
[[ -z "${creds}" ]] && error "Empty credential for the connector '${key} from ${secrets_provider_name}"
|
||||
|
||||
if [ -v SECRET_MAP[${key}] ]; then
|
||||
echo "The connector '${key}' was added before"
|
||||
return 0
|
||||
fi
|
||||
echo "register the secret ${key} from ${secrets_provider_name}"
|
||||
SECRET_MAP[${key}]="${creds}"
|
||||
return 0
|
||||
}
|
||||
|
||||
function write_secret_to_disk() {
|
||||
local connector_name=$1
|
||||
local cred_filename=$2
|
||||
local creds=$3
|
||||
if jq -e . >/dev/null 2>&1 <<< "${creds}"; then
|
||||
echo "Parsed JSON for '${connector_name}' => ${cred_filename} successfully"
|
||||
else
|
||||
error "Failed to parse JSON for '${connector_name}' => ${cred_filename}"
|
||||
fi
|
||||
|
||||
if [ "$connector_name" = "base-normalization" ]; then
|
||||
local secrets_dir="airbyte-integrations/bases/${connector_name}/secrets"
|
||||
@@ -18,153 +56,246 @@ function write_standard_creds() {
|
||||
local secrets_dir="airbyte-integrations/connectors/${connector_name}/secrets"
|
||||
fi
|
||||
mkdir -p "$secrets_dir"
|
||||
echo "Saved a secret => ${secrets_dir}/${cred_filename}"
|
||||
echo "$creds" > "${secrets_dir}/${cred_filename}"
|
||||
}
|
||||
|
||||
function write_all_secrets() {
|
||||
for key in "${!SECRET_MAP[@]}"; do
|
||||
local connector_name=$(echo ${key} | cut -d'#' -f1)
|
||||
local cred_filename=$(echo ${key} | cut -d'#' -f2)
|
||||
local creds=${SECRET_MAP[${key}]}
|
||||
write_secret_to_disk ${connector_name} ${cred_filename} "${creds}"
|
||||
|
||||
done
|
||||
return 0
|
||||
}
|
||||
|
||||
|
||||
function export_github_secrets(){
|
||||
# We expect that all secrets injected from github are available in an env variable `SECRETS_JSON`
|
||||
local pairs=`echo ${GITHUB_PROVIDED_SECRETS_JSON} | jq -c 'keys[] as $k | {"name": $k, "value": .[$k]} | @base64'`
|
||||
while read row; do
|
||||
pair=$(echo "${row}" | tr -d '"' | base64 -d)
|
||||
local key=$(echo ${pair} | jq -r .name)
|
||||
local value=$(echo ${pair} | jq -r .value)
|
||||
if [[ "$key" == *"_CREDS"* ]]; then
|
||||
declare -gxr "${key}"="$(echo ${value})"
|
||||
fi
|
||||
done <<< ${pairs}
|
||||
unset GITHUB_PROVIDED_SECRETS_JSON
|
||||
}
|
||||
|
||||
function export_gsm_secrets(){
|
||||
local config_file=`mktemp`
|
||||
echo "${GCP_GSM_CREDENTIALS}" > ${config_file}
|
||||
local access_token=$(get_gcp_access_token "${config_file}" "${GSM_SCOPES}")
|
||||
local project_id=$(parse_project_id "${config_file}")
|
||||
rm ${config_file}
|
||||
|
||||
# docs: https://cloud.google.com/secret-manager/docs/filtering#api
|
||||
local filter="name:SECRET_"
|
||||
[[ ${CONNECTOR_NAME} != "all" ]] && filter="${filter} AND labels.connector:${CONNECTOR_NAME}"
|
||||
local uri="https://secretmanager.googleapis.com/v1/projects/${project_id}/secrets"
|
||||
local next_token=''
|
||||
while true; do
|
||||
local data=$(curl -s --get --fail "${uri}" \
|
||||
--data-urlencode "filter=${filter}" \
|
||||
--data-urlencode "pageToken=${next_token}" \
|
||||
--header "authorization: Bearer ${access_token}" \
|
||||
--header "content-type: application/json" \
|
||||
--header "x-goog-user-project: ${project_id}")
|
||||
[[ -z ${data} ]] && error "Can't load secret for connector ${CONNECTOR_NAME}"
|
||||
# GSM returns an empty JSON object if secrets are not found.
|
||||
# It breaks JSON parsing by the 'jq' utility. The simplest fix is response normalization
|
||||
[[ ${data} == "{}" ]] && data='{"secrets": []}'
|
||||
|
||||
for row in $(echo "${data}" | jq -r '.secrets[] | @base64'); do
|
||||
local secret_info=$(echo ${row} | base64 --decode)
|
||||
local secret_name=$(echo ${secret_info}| jq -r .name)
|
||||
local label_filename=$(echo ${secret_info}| jq -r '.labels.filename // "config"')
|
||||
local label_connectors=$(echo ${secret_info}| jq -r '.labels.connector // ""')
|
||||
|
||||
# skip secrets without the label "connector"
|
||||
[[ -z ${label_connectors} ]] && continue
|
||||
if [[ "$label_connectors" != *"${CONNECTOR_NAME}"* ]]; then
|
||||
echo "Not found ${CONNECTOR_NAME} info into the label 'connector' of the secret ${secret_name}"
|
||||
continue
|
||||
fi
|
||||
|
||||
# all secret file names should be finished with ".json"
|
||||
# but '.' cant be used in google, so we append it
|
||||
local filename="${label_filename}.json"
|
||||
echo "found the Google secret of ${label_connectors}: ${secret_name} => ${filename}"
|
||||
local secret_uri="https://secretmanager.googleapis.com/v1/${secret_name}/versions/latest:access"
|
||||
local secret_data=$(curl -s --get --fail "${secret_uri}" \
|
||||
--header "authorization: Bearer ${access_token}" \
|
||||
--header "content-type: application/json" \
|
||||
--header "x-goog-user-project: ${project_id}")
|
||||
[[ -z ${secret_data} ]] && error "Can't load secrets' list"
|
||||
|
||||
secret_data=$(echo ${secret_data} | jq -r '.payload.data // ""' | base64 -d)
|
||||
read_secrets "${CONNECTOR_NAME}" "${secret_data}" "${filename}" "gsm"
|
||||
done
|
||||
next_token=`echo ${data} | jq -r '.nextPageToken // ""'`
|
||||
[[ -z ${next_token} ]] && break
|
||||
done
|
||||
return 0
|
||||
}
|
||||
|
||||
export_gsm_secrets
|
||||
export_github_secrets
|
||||
|
||||
|
||||
|
||||
# Please maintain this organisation and alphabetise.
|
||||
write_standard_creds destination-bigquery "$BIGQUERY_INTEGRATION_TEST_CREDS" "credentials.json"
|
||||
write_standard_creds destination-bigquery-denormalized "$BIGQUERY_DENORMALIZED_INTEGRATION_TEST_CREDS" "credentials.json"
|
||||
write_standard_creds destination-databricks "$DESTINATION_DATABRICKS_CREDS"
|
||||
write_standard_creds destination-gcs "$DESTINATION_GCS_CREDS"
|
||||
write_standard_creds destination-kvdb "$DESTINATION_KVDB_TEST_CREDS"
|
||||
write_standard_creds destination-keen "$DESTINATION_KEEN_TEST_CREDS"
|
||||
read_secrets destination-bigquery "$BIGQUERY_INTEGRATION_TEST_CREDS" "credentials.json"
|
||||
read_secrets destination-bigquery-denormalized "$BIGQUERY_DENORMALIZED_INTEGRATION_TEST_CREDS" "credentials.json"
|
||||
read_secrets destination-databricks "$DESTINATION_DATABRICKS_CREDS"
|
||||
read_secrets destination-gcs "$DESTINATION_GCS_CREDS"
|
||||
read_secrets destination-kvdb "$DESTINATION_KVDB_TEST_CREDS"
|
||||
read_secrets destination-keen "$DESTINATION_KEEN_TEST_CREDS"
|
||||
|
||||
write_standard_creds destination-postgres "$DESTINATION_PUBSUB_TEST_CREDS" "credentials.json"
|
||||
write_standard_creds destination-mongodb-strict-encrypt "$MONGODB_TEST_CREDS" "credentials.json"
|
||||
write_standard_creds destination-mysql "$MYSQL_SSH_KEY_TEST_CREDS" "ssh-key-config.json"
|
||||
write_standard_creds destination-mysql "$MYSQL_SSH_PWD_TEST_CREDS" "ssh-pwd-config.json"
|
||||
write_standard_creds destination-pubsub "$DESTINATION_PUBSUB_TEST_CREDS" "credentials.json"
|
||||
write_standard_creds destination-redshift "$AWS_REDSHIFT_INTEGRATION_TEST_CREDS"
|
||||
write_standard_creds destination-dynamodb "$DESTINATION_DYNAMODB_TEST_CREDS"
|
||||
write_standard_creds destination-oracle "$AWS_ORACLE_INTEGRATION_TEST_CREDS"
|
||||
write_standard_creds destination-s3 "$DESTINATION_S3_INTEGRATION_TEST_CREDS"
|
||||
write_standard_creds destination-azure-blob-storage "$DESTINATION_AZURE_BLOB_CREDS"
|
||||
write_standard_creds destination-snowflake "$SNOWFLAKE_GCS_COPY_INTEGRATION_TEST_CREDS" "copy_gcs_config.json"
|
||||
write_standard_creds destination-snowflake "$SNOWFLAKE_S3_COPY_INTEGRATION_TEST_CREDS" "copy_s3_config.json"
|
||||
write_standard_creds destination-snowflake "$SNOWFLAKE_INTEGRATION_TEST_CREDS" "insert_config.json"
|
||||
read_secrets destination-postgres "$DESTINATION_PUBSUB_TEST_CREDS" "credentials.json"
|
||||
read_secrets destination-mongodb-strict-encrypt "$MONGODB_TEST_CREDS" "credentials.json"
|
||||
read_secrets destination-mysql "$MYSQL_SSH_KEY_TEST_CREDS" "ssh-key-config.json"
|
||||
read_secrets destination-mysql "$MYSQL_SSH_PWD_TEST_CREDS" "ssh-pwd-config.json"
|
||||
read_secrets destination-pubsub "$DESTINATION_PUBSUB_TEST_CREDS" "credentials.json"
|
||||
read_secrets destination-redshift "$AWS_REDSHIFT_INTEGRATION_TEST_CREDS"
|
||||
read_secrets destination-dynamodb "$DESTINATION_DYNAMODB_TEST_CREDS"
|
||||
read_secrets destination-oracle "$AWS_ORACLE_INTEGRATION_TEST_CREDS"
|
||||
read_secrets destination-s3 "$DESTINATION_S3_INTEGRATION_TEST_CREDS"
|
||||
read_secrets destination-azure-blob-storage "$DESTINATION_AZURE_BLOB_CREDS"
|
||||
read_secrets destination-snowflake "$SNOWFLAKE_GCS_COPY_INTEGRATION_TEST_CREDS" "copy_gcs_config.json"
|
||||
read_secrets destination-snowflake "$SNOWFLAKE_S3_COPY_INTEGRATION_TEST_CREDS" "copy_s3_config.json"
|
||||
read_secrets destination-snowflake "$SNOWFLAKE_INTEGRATION_TEST_CREDS" "insert_config.json"
|
||||
|
||||
write_standard_creds base-normalization "$BIGQUERY_INTEGRATION_TEST_CREDS" "bigquery.json"
|
||||
write_standard_creds base-normalization "$SNOWFLAKE_INTEGRATION_TEST_CREDS" "snowflake.json"
|
||||
write_standard_creds base-normalization "$AWS_REDSHIFT_INTEGRATION_TEST_CREDS" "redshift.json"
|
||||
write_standard_creds base-normalization "$AWS_ORACLE_INTEGRATION_TEST_CREDS" "oracle.json"
|
||||
read_secrets base-normalization "$BIGQUERY_INTEGRATION_TEST_CREDS" "bigquery.json"
|
||||
read_secrets base-normalization "$SNOWFLAKE_INTEGRATION_TEST_CREDS" "snowflake.json"
|
||||
read_secrets base-normalization "$AWS_REDSHIFT_INTEGRATION_TEST_CREDS" "redshift.json"
|
||||
read_secrets base-normalization "$AWS_ORACLE_INTEGRATION_TEST_CREDS" "oracle.json"
|
||||
|
||||
read_secrets source-amazon-seller-partner "$AMAZON_SELLER_PARTNER_TEST_CREDS"
|
||||
read_secrets source-amazon-sqs "$SOURCE_AMAZON_SQS_TEST_CREDS"
|
||||
read_secrets source-amplitude "$AMPLITUDE_INTEGRATION_TEST_CREDS"
|
||||
read_secrets source-apify-dataset "$APIFY_INTEGRATION_TEST_CREDS"
|
||||
read_secrets source-amazon-ads "$AMAZON_ADS_TEST_CREDS"
|
||||
read_secrets source-amplitude "$AMPLITUDE_INTEGRATION_TEST_CREDS"
|
||||
read_secrets source-asana "$SOURCE_ASANA_TEST_CREDS"
|
||||
read_secrets source-aws-cloudtrail "$SOURCE_AWS_CLOUDTRAIL_CREDS"
|
||||
read_secrets source-bamboo-hr "$SOURCE_BAMBOO_HR_CREDS"
|
||||
read_secrets source-bigcommerce "$SOURCE_BIGCOMMERCE_CREDS"
|
||||
read_secrets source-bigquery "$BIGQUERY_TEST_CREDS" "credentials.json"
|
||||
read_secrets source-bing-ads "$SOURCE_BING_ADS_CREDS"
|
||||
read_secrets source-braintree "$BRAINTREE_TEST_CREDS"
|
||||
read_secrets source-cart "$CART_TEST_CREDS"
|
||||
read_secrets source-chargebee "$CHARGEBEE_INTEGRATION_TEST_CREDS"
|
||||
read_secrets source-close-com "$SOURCE_CLOSE_COM_CREDS"
|
||||
read_secrets source-commercetools "$SOURCE_COMMERCETOOLS_TEST_CREDS"
|
||||
read_secrets source-confluence "$SOURCE_CONFLUENCE_TEST_CREDS"
|
||||
read_secrets source-delighted "$SOURCE_DELIGHTED_TEST_CREDS"
|
||||
read_secrets source-drift "$DRIFT_INTEGRATION_TEST_CREDS"
|
||||
read_secrets source-dixa "$SOURCE_DIXA_TEST_CREDS"
|
||||
read_secrets source-exchange-rates "$EXCHANGE_RATES_TEST_CREDS"
|
||||
read_secrets source-file "$GOOGLE_CLOUD_STORAGE_TEST_CREDS" "gcs.json"
|
||||
read_secrets source-file "$AWS_S3_INTEGRATION_TEST_CREDS" "aws.json"
|
||||
read_secrets source-file "$AZURE_STORAGE_INTEGRATION_TEST_CREDS" "azblob.json"
|
||||
read_secrets source-file "$FILE_SECURE_HTTPS_TEST_CREDS"
|
||||
read_secrets source-file-secure "$FILE_SECURE_HTTPS_TEST_CREDS"
|
||||
read_secrets source-freshdesk "$FRESHDESK_TEST_CREDS"
|
||||
read_secrets source-freshsales "$SOURCE_FRESHSALES_TEST_CREDS"
|
||||
read_secrets source-freshservice "$SOURCE_FRESHSERVICE_TEST_CREDS"
|
||||
read_secrets source-facebook-marketing "$FACEBOOK_MARKETING_TEST_INTEGRATION_CREDS"
|
||||
read_secrets source-facebook-pages "$FACEBOOK_PAGES_INTEGRATION_TEST_CREDS"
|
||||
read_secrets source-gitlab "$GITLAB_INTEGRATION_TEST_CREDS"
|
||||
read_secrets source-github "$GH_NATIVE_INTEGRATION_TEST_CREDS"
|
||||
read_secrets source-google-ads "$GOOGLE_ADS_TEST_CREDS"
|
||||
read_secrets source-google-analytics-v4 "$GOOGLE_ANALYTICS_V4_TEST_CREDS"
|
||||
read_secrets source-google-analytics-v4 "$GOOGLE_ANALYTICS_V4_TEST_CREDS_SRV_ACC" "service_config.json"
|
||||
read_secrets source-google-analytics-v4 "$GOOGLE_ANALYTICS_V4_TEST_CREDS_OLD" "old_config.json"
|
||||
read_secrets source-google-directory "$GOOGLE_DIRECTORY_TEST_CREDS"
|
||||
read_secrets source-google-directory "$GOOGLE_DIRECTORY_TEST_CREDS_OAUTH" "config_oauth.json"
|
||||
read_secrets source-google-search-console "$GOOGLE_SEARCH_CONSOLE_CDK_TEST_CREDS"
|
||||
read_secrets source-google-search-console "$GOOGLE_SEARCH_CONSOLE_CDK_TEST_CREDS_SRV_ACC" "service_account_config.json"
|
||||
read_secrets source-google-sheets "$GOOGLE_SHEETS_TESTS_CREDS"
|
||||
read_secrets source-google-sheets "$GOOGLE_SHEETS_TESTS_CREDS_SRV_ACC" "service_config.json"
|
||||
read_secrets source-google-sheets "$GOOGLE_SHEETS_TESTS_CREDS_OLD" "old_config.json"
|
||||
read_secrets source-google-workspace-admin-reports "$GOOGLE_WORKSPACE_ADMIN_REPORTS_TEST_CREDS"
|
||||
read_secrets source-greenhouse "$GREENHOUSE_TEST_CREDS"
|
||||
read_secrets source-greenhouse "$GREENHOUSE_TEST_CREDS_LIMITED" "config_users_only.json"
|
||||
read_secrets source-harvest "$HARVEST_INTEGRATION_TESTS_CREDS"
|
||||
read_secrets source-hubspot "$HUBSPOT_INTEGRATION_TESTS_CREDS"
|
||||
read_secrets source-hubspot "$HUBSPOT_INTEGRATION_TESTS_CREDS_OAUTH" "config_oauth.json"
|
||||
read_secrets source-instagram "$INSTAGRAM_INTEGRATION_TESTS_CREDS"
|
||||
read_secrets source-intercom "$INTERCOM_INTEGRATION_TEST_CREDS"
|
||||
read_secrets source-intercom "$INTERCOM_INTEGRATION_OAUTH_TEST_CREDS" "config_apikey.json"
|
||||
read_secrets source-iterable "$ITERABLE_INTEGRATION_TEST_CREDS"
|
||||
read_secrets source-jira "$JIRA_INTEGRATION_TEST_CREDS"
|
||||
read_secrets source-klaviyo "$KLAVIYO_TEST_CREDS"
|
||||
read_secrets source-lemlist "$SOURCE_LEMLIST_TEST_CREDS"
|
||||
read_secrets source-lever-hiring "$LEVER_HIRING_INTEGRATION_TEST_CREDS"
|
||||
read_secrets source-looker "$LOOKER_INTEGRATION_TEST_CREDS"
|
||||
read_secrets source-linkedin-ads "$SOURCE_LINKEDIN_ADS_TEST_CREDS"
|
||||
read_secrets source-mailchimp "$MAILCHIMP_TEST_CREDS"
|
||||
read_secrets source-marketo "$SOURCE_MARKETO_TEST_CREDS"
|
||||
read_secrets source-microsoft-teams "$MICROSOFT_TEAMS_TEST_CREDS"
|
||||
read_secrets source-mixpanel "$MIXPANEL_INTEGRATION_TEST_CREDS"
|
||||
read_secrets source-monday "$SOURCE_MONDAY_TEST_CREDS"
|
||||
read_secrets source-mongodb-strict-encrypt "$MONGODB_TEST_CREDS" "credentials.json"
|
||||
read_secrets source-mongodb-v2 "$MONGODB_TEST_CREDS" "credentials.json"
|
||||
read_secrets source-mssql "$MSSQL_RDS_TEST_CREDS"
|
||||
read_secrets source-okta "$SOURCE_OKTA_TEST_CREDS"
|
||||
read_secrets source-onesignal "$SOURCE_ONESIGNAL_TEST_CREDS"
|
||||
read_secrets source-plaid "$PLAID_INTEGRATION_TEST_CREDS"
|
||||
read_secrets source-paypal-transaction "$PAYPAL_TRANSACTION_CREDS"
|
||||
read_secrets source-pinterest "$PINTEREST_TEST_CREDS"
|
||||
read_secrets source-mysql "$MYSQL_SSH_KEY_TEST_CREDS" "ssh-key-config.json"
|
||||
read_secrets source-mysql "$MYSQL_SSH_PWD_TEST_CREDS" "ssh-pwd-config.json"
|
||||
read_secrets source-posthog "$POSTHOG_TEST_CREDS"
|
||||
read_secrets source-pipedrive "$PIPEDRIVE_INTEGRATION_TESTS_CREDS" "config.json"
|
||||
read_secrets source-pipedrive "$PIPEDRIVE_INTEGRATION_TESTS_CREDS_OAUTH" "oauth_config.json"
|
||||
read_secrets source-pipedrive "$PIPEDRIVE_INTEGRATION_TESTS_CREDS_OLD" "old_config.json"
|
||||
read_secrets source-quickbooks-singer "$QUICKBOOKS_TEST_CREDS"
|
||||
read_secrets source-recharge "$RECHARGE_INTEGRATION_TEST_CREDS"
|
||||
read_secrets source-recurly "$SOURCE_RECURLY_INTEGRATION_TEST_CREDS"
|
||||
read_secrets source-redshift "$AWS_REDSHIFT_INTEGRATION_TEST_CREDS"
|
||||
read_secrets source-retently "$SOURCE_RETENTLY_TEST_CREDS"
|
||||
read_secrets source-s3 "$SOURCE_S3_TEST_CREDS"
|
||||
read_secrets source-s3 "$SOURCE_S3_PARQUET_CREDS" "parquet_config.json"
|
||||
read_secrets source-salesforce "$SALESFORCE_BULK_INTEGRATION_TESTS_CREDS" "config_bulk.json"
|
||||
read_secrets source-salesforce "$SALESFORCE_INTEGRATION_TESTS_CREDS"
|
||||
read_secrets source-salesloft "$SOURCE_SALESLOFT_TEST_CREDS"
|
||||
read_secrets source-sendgrid "$SENDGRID_INTEGRATION_TEST_CREDS"
|
||||
read_secrets source-shopify "$SHOPIFY_INTEGRATION_TEST_CREDS"
|
||||
read_secrets source-shopify "$SHOPIFY_INTEGRATION_TEST_OAUTH_CREDS" "config_oauth.json"
|
||||
read_secrets source-shortio "$SOURCE_SHORTIO_TEST_CREDS"
|
||||
read_secrets source-slack "$SOURCE_SLACK_TEST_CREDS"
|
||||
read_secrets source-slack "$SOURCE_SLACK_OAUTH_TEST_CREDS" "config_oauth.json"
|
||||
read_secrets source-smartsheets "$SMARTSHEETS_TEST_CREDS"
|
||||
read_secrets source-snapchat-marketing "$SOURCE_SNAPCHAT_MARKETING_CREDS"
|
||||
read_secrets source-snowflake "$SNOWFLAKE_INTEGRATION_TEST_CREDS" "config.json"
|
||||
read_secrets source-square "$SOURCE_SQUARE_CREDS"
|
||||
read_secrets source-strava "$SOURCE_STRAVA_TEST_CREDS"
|
||||
read_secrets source-paystack "$SOURCE_PAYSTACK_TEST_CREDS"
|
||||
read_secrets source-sentry "$SOURCE_SENTRY_TEST_CREDS"
|
||||
read_secrets source-stripe "$SOURCE_STRIPE_CREDS"
|
||||
read_secrets source-stripe "$STRIPE_INTEGRATION_CONNECTED_ACCOUNT_TEST_CREDS" "connected_account_config.json"
|
||||
read_secrets source-surveymonkey "$SURVEYMONKEY_TEST_CREDS"
|
||||
read_secrets source-tempo "$TEMPO_INTEGRATION_TEST_CREDS"
|
||||
read_secrets source-tiktok-marketing "$SOURCE_TIKTOK_MARKETING_TEST_CREDS"
|
||||
read_secrets source-tiktok-marketing "$SOURCE_TIKTOK_MARKETING_PROD_TEST_CREDS" "prod_config.json"
|
||||
read_secrets source-trello "$TRELLO_TEST_CREDS"
|
||||
read_secrets source-twilio "$TWILIO_TEST_CREDS"
|
||||
read_secrets source-typeform "$SOURCE_TYPEFORM_CREDS"
|
||||
read_secrets source-us-census "$SOURCE_US_CENSUS_TEST_CREDS"
|
||||
read_secrets source-zendesk-chat "$ZENDESK_CHAT_INTEGRATION_TEST_CREDS"
|
||||
read_secrets source-zendesk-sunshine "$ZENDESK_SUNSHINE_TEST_CREDS"
|
||||
read_secrets source-zendesk-support "$ZENDESK_SUPPORT_TEST_CREDS"
|
||||
read_secrets source-zendesk-support "$ZENDESK_SUPPORT_OAUTH_TEST_CREDS" "config_oauth.json"
|
||||
read_secrets source-zendesk-talk "$ZENDESK_TALK_TEST_CREDS"
|
||||
read_secrets source-zoom-singer "$ZOOM_INTEGRATION_TEST_CREDS"
|
||||
read_secrets source-zuora "$SOURCE_ZUORA_TEST_CREDS"
|
||||
|
||||
write_all_secrets
|
||||
exit $?
|
||||
|
||||
write_standard_creds source-amazon-seller-partner "$AMAZON_SELLER_PARTNER_TEST_CREDS"
|
||||
write_standard_creds source-amazon-sqs "$SOURCE_AMAZON_SQS_TEST_CREDS"
|
||||
write_standard_creds source-amplitude "$AMPLITUDE_INTEGRATION_TEST_CREDS"
|
||||
write_standard_creds source-apify-dataset "$APIFY_INTEGRATION_TEST_CREDS"
|
||||
write_standard_creds source-amazon-ads "$AMAZON_ADS_TEST_CREDS"
|
||||
write_standard_creds source-amplitude "$AMPLITUDE_INTEGRATION_TEST_CREDS"
|
||||
write_standard_creds source-asana "$SOURCE_ASANA_TEST_CREDS"
|
||||
write_standard_creds source-aws-cloudtrail "$SOURCE_AWS_CLOUDTRAIL_CREDS"
|
||||
write_standard_creds source-bamboo-hr "$SOURCE_BAMBOO_HR_CREDS"
|
||||
write_standard_creds source-bigcommerce "$SOURCE_BIGCOMMERCE_CREDS"
|
||||
write_standard_creds source-bigquery "$BIGQUERY_TEST_CREDS" "credentials.json"
|
||||
write_standard_creds source-bing-ads "$SOURCE_BING_ADS_CREDS"
|
||||
write_standard_creds source-braintree "$BRAINTREE_TEST_CREDS"
|
||||
write_standard_creds source-cart "$CART_TEST_CREDS"
|
||||
write_standard_creds source-chargebee "$CHARGEBEE_INTEGRATION_TEST_CREDS"
|
||||
write_standard_creds source-close-com "$SOURCE_CLOSE_COM_CREDS"
|
||||
write_standard_creds source-commercetools "$SOURCE_COMMERCETOOLS_TEST_CREDS"
|
||||
write_standard_creds source-confluence "$SOURCE_CONFLUENCE_TEST_CREDS"
|
||||
write_standard_creds source-delighted "$SOURCE_DELIGHTED_TEST_CREDS"
|
||||
write_standard_creds source-drift "$DRIFT_INTEGRATION_TEST_CREDS"
|
||||
write_standard_creds source-dixa "$SOURCE_DIXA_TEST_CREDS"
|
||||
write_standard_creds source-exchange-rates "$EXCHANGE_RATES_TEST_CREDS"
|
||||
write_standard_creds source-file "$GOOGLE_CLOUD_STORAGE_TEST_CREDS" "gcs.json"
|
||||
write_standard_creds source-file "$AWS_S3_INTEGRATION_TEST_CREDS" "aws.json"
|
||||
write_standard_creds source-file "$AZURE_STORAGE_INTEGRATION_TEST_CREDS" "azblob.json"
|
||||
write_standard_creds source-file "$FILE_SECURE_HTTPS_TEST_CREDS"
|
||||
write_standard_creds source-file-secure "$FILE_SECURE_HTTPS_TEST_CREDS"
|
||||
write_standard_creds source-freshdesk "$FRESHDESK_TEST_CREDS"
|
||||
write_standard_creds source-freshsales "$SOURCE_FRESHSALES_TEST_CREDS"
|
||||
write_standard_creds source-freshservice "$SOURCE_FRESHSERVICE_TEST_CREDS"
|
||||
write_standard_creds source-facebook-marketing "$FACEBOOK_MARKETING_TEST_INTEGRATION_CREDS"
|
||||
write_standard_creds source-facebook-pages "$FACEBOOK_PAGES_INTEGRATION_TEST_CREDS"
|
||||
write_standard_creds source-gitlab "$GITLAB_INTEGRATION_TEST_CREDS"
|
||||
write_standard_creds source-github "$GH_NATIVE_INTEGRATION_TEST_CREDS"
|
||||
write_standard_creds source-google-ads "$GOOGLE_ADS_TEST_CREDS"
|
||||
write_standard_creds source-google-analytics-v4 "$GOOGLE_ANALYTICS_V4_TEST_CREDS"
|
||||
write_standard_creds source-google-analytics-v4 "$GOOGLE_ANALYTICS_V4_TEST_CREDS_SRV_ACC" "service_config.json"
|
||||
write_standard_creds source-google-analytics-v4 "$GOOGLE_ANALYTICS_V4_TEST_CREDS_OLD" "old_config.json"
|
||||
write_standard_creds source-google-directory "$GOOGLE_DIRECTORY_TEST_CREDS"
|
||||
write_standard_creds source-google-directory "$GOOGLE_DIRECTORY_TEST_CREDS_OAUTH" "config_oauth.json"
|
||||
write_standard_creds source-google-search-console "$GOOGLE_SEARCH_CONSOLE_CDK_TEST_CREDS"
|
||||
write_standard_creds source-google-search-console "$GOOGLE_SEARCH_CONSOLE_CDK_TEST_CREDS_SRV_ACC" "service_account_config.json"
|
||||
write_standard_creds source-google-sheets "$GOOGLE_SHEETS_TESTS_CREDS"
|
||||
write_standard_creds source-google-sheets "$GOOGLE_SHEETS_TESTS_CREDS_SRV_ACC" "service_config.json"
|
||||
write_standard_creds source-google-sheets "$GOOGLE_SHEETS_TESTS_CREDS_OLD" "old_config.json"
|
||||
write_standard_creds source-google-workspace-admin-reports "$GOOGLE_WORKSPACE_ADMIN_REPORTS_TEST_CREDS"
|
||||
write_standard_creds source-greenhouse "$GREENHOUSE_TEST_CREDS"
|
||||
write_standard_creds source-greenhouse "$GREENHOUSE_TEST_CREDS_LIMITED" "config_users_only.json"
|
||||
write_standard_creds source-harvest "$HARVEST_INTEGRATION_TESTS_CREDS"
|
||||
write_standard_creds source-hubspot "$HUBSPOT_INTEGRATION_TESTS_CREDS"
|
||||
write_standard_creds source-hubspot "$HUBSPOT_INTEGRATION_TESTS_CREDS_OAUTH" "config_oauth.json"
|
||||
write_standard_creds source-instagram "$INSTAGRAM_INTEGRATION_TESTS_CREDS"
|
||||
write_standard_creds source-intercom "$INTERCOM_INTEGRATION_TEST_CREDS"
|
||||
write_standard_creds source-intercom "$INTERCOM_INTEGRATION_OAUTH_TEST_CREDS" "config_apikey.json"
|
||||
write_standard_creds source-iterable "$ITERABLE_INTEGRATION_TEST_CREDS"
|
||||
write_standard_creds source-jira "$JIRA_INTEGRATION_TEST_CREDS"
|
||||
write_standard_creds source-klaviyo "$KLAVIYO_TEST_CREDS"
|
||||
write_standard_creds source-lemlist "$SOURCE_LEMLIST_TEST_CREDS"
|
||||
write_standard_creds source-lever-hiring "$LEVER_HIRING_INTEGRATION_TEST_CREDS"
|
||||
write_standard_creds source-looker "$LOOKER_INTEGRATION_TEST_CREDS"
|
||||
write_standard_creds source-linkedin-ads "$SOURCE_LINKEDIN_ADS_TEST_CREDS"
|
||||
write_standard_creds source-mailchimp "$MAILCHIMP_TEST_CREDS"
|
||||
write_standard_creds source-marketo "$SOURCE_MARKETO_TEST_CREDS"
|
||||
write_standard_creds source-microsoft-teams "$MICROSOFT_TEAMS_TEST_CREDS"
|
||||
write_standard_creds source-mixpanel "$MIXPANEL_INTEGRATION_TEST_CREDS"
|
||||
write_standard_creds source-monday "$SOURCE_MONDAY_TEST_CREDS"
|
||||
write_standard_creds source-mongodb-strict-encrypt "$MONGODB_TEST_CREDS" "credentials.json"
|
||||
write_standard_creds source-mongodb-v2 "$MONGODB_TEST_CREDS" "credentials.json"
|
||||
write_standard_creds source-mssql "$MSSQL_RDS_TEST_CREDS"
|
||||
write_standard_creds source-okta "$SOURCE_OKTA_TEST_CREDS"
|
||||
write_standard_creds source-onesignal "$SOURCE_ONESIGNAL_TEST_CREDS"
|
||||
write_standard_creds source-plaid "$PLAID_INTEGRATION_TEST_CREDS"
|
||||
write_standard_creds source-paypal-transaction "$PAYPAL_TRANSACTION_CREDS"
|
||||
write_standard_creds source-pinterest "$PINTEREST_TEST_CREDS"
|
||||
write_standard_creds source-mysql "$MYSQL_SSH_KEY_TEST_CREDS" "ssh-key-config.json"
|
||||
write_standard_creds source-mysql "$MYSQL_SSH_PWD_TEST_CREDS" "ssh-pwd-config.json"
|
||||
write_standard_creds source-posthog "$POSTHOG_TEST_CREDS"
|
||||
write_standard_creds source-pipedrive "$PIPEDRIVE_INTEGRATION_TESTS_CREDS" "config.json"
|
||||
write_standard_creds source-pipedrive "$PIPEDRIVE_INTEGRATION_TESTS_CREDS_OAUTH" "oauth_config.json"
|
||||
write_standard_creds source-pipedrive "$PIPEDRIVE_INTEGRATION_TESTS_CREDS_OLD" "old_config.json"
|
||||
write_standard_creds source-quickbooks-singer "$QUICKBOOKS_TEST_CREDS"
|
||||
write_standard_creds source-recharge "$RECHARGE_INTEGRATION_TEST_CREDS"
|
||||
write_standard_creds source-recurly "$SOURCE_RECURLY_INTEGRATION_TEST_CREDS"
|
||||
write_standard_creds source-redshift "$AWS_REDSHIFT_INTEGRATION_TEST_CREDS"
|
||||
write_standard_creds source-retently "$SOURCE_RETENTLY_TEST_CREDS"
|
||||
write_standard_creds source-s3 "$SOURCE_S3_TEST_CREDS"
|
||||
write_standard_creds source-s3 "$SOURCE_S3_PARQUET_CREDS" "parquet_config.json"
|
||||
write_standard_creds source-salesforce "$SALESFORCE_BULK_INTEGRATION_TESTS_CREDS" "config_bulk.json"
|
||||
write_standard_creds source-salesforce "$SALESFORCE_INTEGRATION_TESTS_CREDS"
|
||||
write_standard_creds source-salesloft "$SOURCE_SALESLOFT_TEST_CREDS"
|
||||
write_standard_creds source-sendgrid "$SENDGRID_INTEGRATION_TEST_CREDS"
|
||||
write_standard_creds source-shopify "$SHOPIFY_INTEGRATION_TEST_CREDS"
|
||||
write_standard_creds source-shopify "$SHOPIFY_INTEGRATION_TEST_OAUTH_CREDS" "config_oauth.json"
|
||||
write_standard_creds source-shortio "$SOURCE_SHORTIO_TEST_CREDS"
|
||||
write_standard_creds source-slack "$SOURCE_SLACK_TEST_CREDS"
|
||||
write_standard_creds source-slack "$SOURCE_SLACK_OAUTH_TEST_CREDS" "config_oauth.json"
|
||||
write_standard_creds source-smartsheets "$SMARTSHEETS_TEST_CREDS"
|
||||
write_standard_creds source-snapchat-marketing "$SOURCE_SNAPCHAT_MARKETING_CREDS"
|
||||
write_standard_creds source-snowflake "$SNOWFLAKE_INTEGRATION_TEST_CREDS" "config.json"
|
||||
write_standard_creds source-square "$SOURCE_SQUARE_CREDS"
|
||||
write_standard_creds source-strava "$SOURCE_STRAVA_TEST_CREDS"
|
||||
write_standard_creds source-paystack "$SOURCE_PAYSTACK_TEST_CREDS"
|
||||
write_standard_creds source-sentry "$SOURCE_SENTRY_TEST_CREDS"
|
||||
write_standard_creds source-stripe "$SOURCE_STRIPE_CREDS"
|
||||
write_standard_creds source-stripe "$STRIPE_INTEGRATION_CONNECTED_ACCOUNT_TEST_CREDS" "connected_account_config.json"
|
||||
write_standard_creds source-surveymonkey "$SURVEYMONKEY_TEST_CREDS"
|
||||
write_standard_creds source-tempo "$TEMPO_INTEGRATION_TEST_CREDS"
|
||||
write_standard_creds source-tiktok-marketing "$SOURCE_TIKTOK_MARKETING_TEST_CREDS"
|
||||
write_standard_creds source-tiktok-marketing "$SOURCE_TIKTOK_MARKETING_PROD_TEST_CREDS" "prod_config.json"
|
||||
write_standard_creds source-trello "$TRELLO_TEST_CREDS"
|
||||
write_standard_creds source-twilio "$TWILIO_TEST_CREDS"
|
||||
write_standard_creds source-typeform "$SOURCE_TYPEFORM_CREDS"
|
||||
write_standard_creds source-us-census "$SOURCE_US_CENSUS_TEST_CREDS"
|
||||
write_standard_creds source-zendesk-chat "$ZENDESK_CHAT_INTEGRATION_TEST_CREDS"
|
||||
write_standard_creds source-zendesk-sunshine "$ZENDESK_SUNSHINE_TEST_CREDS"
|
||||
write_standard_creds source-zendesk-support "$ZENDESK_SUPPORT_TEST_CREDS"
|
||||
write_standard_creds source-zendesk-support "$ZENDESK_SUPPORT_OAUTH_TEST_CREDS" "config_oauth.json"
|
||||
write_standard_creds source-zendesk-talk "$ZENDESK_TALK_TEST_CREDS"
|
||||
write_standard_creds source-zoom-singer "$ZOOM_INTEGRATION_TEST_CREDS"
|
||||
write_standard_creds source-zuora "$SOURCE_ZUORA_TEST_CREDS"
|
||||
|
||||
69
tools/lib/gcp-token.sh
Normal file
69
tools/lib/gcp-token.sh
Normal file
@@ -0,0 +1,69 @@
|
||||
#!/usr/bin/env bash
|
||||
# Test script to access/generate secrets in Secret Manager
|
||||
|
||||
# PROJECT="engineering-devops"
|
||||
# SCOPE="https://www.googleapis.com/auth/cloud-platform"
|
||||
# SERVICE_ACCOUNT_FILE=secret-manager.json
|
||||
# SECRET=my-secret
|
||||
TOKEN_TTL=3600
|
||||
|
||||
|
||||
_var2base64() {
|
||||
printf "$1" | _urlencode_base64
|
||||
}
|
||||
|
||||
_urlencode_base64() {
|
||||
base64 | tr '/+' '_-' | tr -d '=\n'
|
||||
}
|
||||
|
||||
function _parse_token_uri(){
|
||||
local config_file=$1
|
||||
local token_uri=$(jq -r .token_uri ${config_file})
|
||||
echo "${token_uri}"
|
||||
}
|
||||
|
||||
function _generate_jwt() {
|
||||
# Generate JWT token by a service account json file and scopes
|
||||
local config_file=$1
|
||||
local scopes=$2
|
||||
|
||||
local now="$(date +%s)"
|
||||
local expiration_time=$((${now} + ${TOKEN_TTL}))
|
||||
# parse a file with credentials
|
||||
local private_key=$(jq -r .private_key ${config_file})
|
||||
local client_email=$(jq -r .client_email ${config_file})
|
||||
local token_uri=$(_parse_token_uri "${config_file}")
|
||||
|
||||
local claim=$(echo "{
|
||||
\"iat\": ${now},
|
||||
\"iss\": \"${client_email}\",
|
||||
\"scope\": \"$scopes\",
|
||||
\"aud\": \"${token_uri}\",
|
||||
\"exp\":${expiration_time}
|
||||
}" | jq -c)
|
||||
local headers='{"typ":"JWT","alg":"RS256"}'
|
||||
local body="$(_var2base64 "$headers").$(_var2base64 "$claim")"
|
||||
local signature=$(openssl dgst -sha256 -sign <(echo "$private_key") <(printf "$body") | _urlencode_base64)
|
||||
echo "$body.$signature"
|
||||
}
|
||||
|
||||
function parse_project_id(){
|
||||
# find a project_id into config file
|
||||
local config_file=$1
|
||||
local project_id=$(jq -r .project_id ${config_file})
|
||||
echo "${project_id}"
|
||||
}
|
||||
|
||||
function get_gcp_access_token() {
|
||||
# Generate an access token by a service account json file and scopes
|
||||
local config_file="$1"
|
||||
local scopes="$2"
|
||||
local jwt=`_generate_jwt "${config_file}" "$scopes"`
|
||||
local token_uri=$(_parse_token_uri "${config_file}")
|
||||
local data=$(curl -s -X POST ${token_uri} \
|
||||
--data-urlencode "assertion=${jwt}" \
|
||||
--data-urlencode 'grant_type=urn:ietf:params:oauth:grant-type:jwt-bearer'
|
||||
)
|
||||
echo $data | jq -r .access_token
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user