From 6465aae0dc1fabd9315d18d5544bfbb6cec7ffce Mon Sep 17 00:00:00 2001 From: Oleksandr Sheheda Date: Thu, 22 Sep 2022 21:45:06 +0300 Subject: [PATCH] [16617] Source BigQuery: added user agent header (#16902) * [16617] Source BigQuery: added user agent header * [16617] Source BigQuery: added user agent header * [16617] Source BigQuery: added user agent header updated image tag, added changelog * [16617] Source BigQuery: added user agent header fixed NPE * auto-bump connector version [ci skip] Co-authored-by: Octavia Squidington III --- .../main/resources/seed/source_definitions.yaml | 2 +- .../src/main/resources/seed/source_specs.yaml | 2 +- .../airbyte/db/bigquery/BigQueryDatabase.java | 17 +++++++++++++++++ .../connectors/source-bigquery/Dockerfile | 2 +- docs/integrations/sources/bigquery.md | 1 + 5 files changed, 21 insertions(+), 3 deletions(-) diff --git a/airbyte-config/init/src/main/resources/seed/source_definitions.yaml b/airbyte-config/init/src/main/resources/seed/source_definitions.yaml index e1d6fa76893..5cb7ad34f82 100644 --- a/airbyte-config/init/src/main/resources/seed/source_definitions.yaml +++ b/airbyte-config/init/src/main/resources/seed/source_definitions.yaml @@ -112,7 +112,7 @@ - name: BigQuery sourceDefinitionId: bfd1ddf8-ae8a-4620-b1d7-55597d2ba08c dockerRepository: airbyte/source-bigquery - dockerImageTag: 0.2.1 + dockerImageTag: 0.2.2 documentationUrl: https://docs.airbyte.io/integrations/sources/bigquery icon: bigquery.svg sourceType: database diff --git a/airbyte-config/init/src/main/resources/seed/source_specs.yaml b/airbyte-config/init/src/main/resources/seed/source_specs.yaml index ab41e713442..f6e19121500 100644 --- a/airbyte-config/init/src/main/resources/seed/source_specs.yaml +++ b/airbyte-config/init/src/main/resources/seed/source_specs.yaml @@ -1268,7 +1268,7 @@ supportsNormalization: false supportsDBT: false supported_destination_sync_modes: [] -- dockerImage: "airbyte/source-bigquery:0.2.1" +- dockerImage: "airbyte/source-bigquery:0.2.2" spec: documentationUrl: "https://docs.airbyte.io/integrations/sources/bigquery" connectionSpecification: diff --git a/airbyte-db/db-lib/src/main/java/io/airbyte/db/bigquery/BigQueryDatabase.java b/airbyte-db/db-lib/src/main/java/io/airbyte/db/bigquery/BigQueryDatabase.java index 8ee6d27b04d..90238f9ebe3 100644 --- a/airbyte-db/db-lib/src/main/java/io/airbyte/db/bigquery/BigQueryDatabase.java +++ b/airbyte-db/db-lib/src/main/java/io/airbyte/db/bigquery/BigQueryDatabase.java @@ -5,6 +5,7 @@ package io.airbyte.db.bigquery; import static java.util.Objects.isNull; +import static org.apache.commons.lang3.StringUtils.EMPTY; import com.fasterxml.jackson.databind.JsonNode; import com.google.api.gax.retrying.RetrySettings; @@ -20,7 +21,9 @@ import com.google.cloud.bigquery.QueryParameterValue; import com.google.cloud.bigquery.StandardSQLTypeName; import com.google.cloud.bigquery.Table; import com.google.common.base.Charsets; +import com.google.common.collect.ImmutableMap; import com.google.common.collect.Streams; +import io.airbyte.config.WorkerEnvConstants; import io.airbyte.db.SqlDatabase; import java.io.ByteArrayInputStream; import java.io.IOException; @@ -29,9 +32,11 @@ import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; import java.util.List; +import java.util.Optional; import java.util.UUID; import java.util.stream.Collectors; import java.util.stream.Stream; +import org.apache.commons.lang3.StringUtils; import org.apache.commons.lang3.tuple.ImmutablePair; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -40,6 +45,7 @@ import org.threeten.bp.Duration; public class BigQueryDatabase extends SqlDatabase { private static final Logger LOGGER = LoggerFactory.getLogger(BigQueryDatabase.class); + private static final String AGENT_TEMPLATE = "%s (GPN: Airbyte; staging)"; private final BigQuery bigQuery; private final BigQuerySourceOperations sourceOperations; @@ -60,6 +66,7 @@ public class BigQueryDatabase extends SqlDatabase { bigQuery = bigQueryBuilder .setProjectId(projectId) .setCredentials(!isNull(credentials) ? credentials : ServiceAccountCredentials.getApplicationDefault()) + .setHeaderProvider(() -> ImmutableMap.of("user-agent", getUserAgentHeader(getConnectorVersion()))) .setRetrySettings(RetrySettings .newBuilder() .setMaxAttempts(10) @@ -73,6 +80,16 @@ public class BigQueryDatabase extends SqlDatabase { } } + private String getUserAgentHeader(String connectorVersion) { + return String.format(AGENT_TEMPLATE, connectorVersion); + } + + private String getConnectorVersion() { + return Optional.ofNullable(System.getenv(WorkerEnvConstants.WORKER_CONNECTOR_IMAGE)) + .orElse(EMPTY) + .replace("airbyte/", EMPTY).replace(":", "/"); + } + @Override public void execute(final String sql) throws SQLException { final ImmutablePair result = executeQuery(bigQuery, getQueryConfig(sql, Collections.emptyList())); diff --git a/airbyte-integrations/connectors/source-bigquery/Dockerfile b/airbyte-integrations/connectors/source-bigquery/Dockerfile index 43981ac09c1..b03d2204387 100644 --- a/airbyte-integrations/connectors/source-bigquery/Dockerfile +++ b/airbyte-integrations/connectors/source-bigquery/Dockerfile @@ -17,5 +17,5 @@ ENV APPLICATION source-bigquery COPY --from=build /airbyte /airbyte # Airbyte's build system uses these labels to know what to name and tag the docker images produced by this Dockerfile. -LABEL io.airbyte.version=0.2.1 +LABEL io.airbyte.version=0.2.2 LABEL io.airbyte.name=airbyte/source-bigquery diff --git a/docs/integrations/sources/bigquery.md b/docs/integrations/sources/bigquery.md index 3e0a74feace..75881e800c2 100644 --- a/docs/integrations/sources/bigquery.md +++ b/docs/integrations/sources/bigquery.md @@ -88,6 +88,7 @@ Once you've configured BigQuery as a source, delete the Service Account Key from | Version | Date | Pull Request | Subject | | :--- | :--- | :--- | :--- | +| 0.2.2 | 2022-09-22 | [16902](https://github.com/airbytehq/airbyte/pull/16902) | Source BigQuery: added user agent header | | 0.2.1 | 2022-09-14 | [15668](https://github.com/airbytehq/airbyte/pull/15668) | Wrap logs in AirbyteLogMessage | | 0.2.0 | 2022-07-26 | [14362](https://github.com/airbytehq/airbyte/pull/14362) | Integral columns are now discovered as int64 fields. | | 0.1.9 | 2022-07-14 | [14574](https://github.com/airbytehq/airbyte/pull/14574) | Removed additionalProperties:false from JDBC source connectors |