jprdonnelly/airbyte
1
0
Fork 0
mirror of synced 2025-12-25 02:09:19 -05:00
Code Issues Packages Projects Releases Wiki Activity

Log full vacuum info (#26633)

Browse Source 
* Log full vacuum info

* sanity

* commit change to force CI check

* improve queries

* Improve warning

* bump version and release note
This commit is contained in:
Rodi Reich Zilberman
2023-05-30 15:48:57 -07:00
committed by GitHub
parent c5936f09f6
commit c1ea190bfc
7 changed files with 43 additions and 4 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
airbyte-integrations/connectors/source-postgres-strict-encrypt/Dockerfile
View File

@@ -24,5 +24,5 @@ ENV APPLICATION source-postgres-strict-encrypt
COPY --from=build /airbyte /airbyte
LABEL io.airbyte.version=2.0.30
LABEL io.airbyte.version=2.0.31
LABEL io.airbyte.name=airbyte/source-postgres-strict-encrypt

2
airbyte-integrations/connectors/source-postgres-strict-encrypt/metadata.yaml
View File

@@ -12,7 +12,7 @@ data:
connectorType: source
definitionId: decd338e-5647-4c0b-adf4-da0e75f5a750
maxSecondsBetweenMessages: 7200
dockerImageTag: 2.0.30
dockerImageTag: 2.0.31
dockerRepository: airbyte/source-postgres-strict-encrypt
githubIssueLabel: source-postgres
icon: postgresql.svg

2
airbyte-integrations/connectors/source-postgres/Dockerfile
View File

@@ -24,5 +24,5 @@ ENV APPLICATION source-postgres
COPY --from=build /airbyte /airbyte
LABEL io.airbyte.version=2.0.30
LABEL io.airbyte.version=2.0.31
LABEL io.airbyte.name=airbyte/source-postgres

2
airbyte-integrations/connectors/source-postgres/metadata.yaml
View File

@@ -6,7 +6,7 @@ data:
connectorSubtype: database
connectorType: source
definitionId: decd338e-5647-4c0b-adf4-da0e75f5a750
dockerImageTag: 2.0.30
dockerImageTag: 2.0.31
maxSecondsBetweenMessages: 7200
dockerRepository: airbyte/source-postgres
githubIssueLabel: source-postgres

37
airbyte-integrations/connectors/source-postgres/src/main/java/io/airbyte/integrations/source/postgres/PostgresQueryUtils.java
View File

@@ -4,11 +4,14 @@
package io.airbyte.integrations.source.postgres;
import static io.airbyte.integrations.source.relationaldb.RelationalDbQueryUtils.getFullyQualifiedTableNameWithQuoting;
import com.fasterxml.jackson.databind.JsonNode;
import com.google.common.base.Preconditions;
import io.airbyte.db.jdbc.JdbcDatabase;
import io.airbyte.db.jdbc.JdbcUtils;
import io.airbyte.integrations.source.postgres.internal.models.XminStatus;
import io.airbyte.protocol.models.v0.ConfiguredAirbyteCatalog;
import java.sql.SQLException;
import java.util.List;
import org.slf4j.Logger;
@@ -52,6 +55,14 @@ public class PostgresQueryUtils {
txid_snapshot_xmin(txid_current_snapshot()) AS xmin_raw_value;
""";
public static final String CTID_FULL_VACUUM_IN_PROGRESS_QUERY =
"""
SELECT phase FROM pg_stat_progress_cluster WHERE command = 'VACUUM FULL' AND relid=to_regclass('%s')::oid
""";
public static final String CTID_FULL_VACUUM_REL_FILENODE_QUERY =
"""
SELECT pg_relation_filenode('%s')
""";
public static final String NUM_WRAPAROUND_COL = "num_wraparound";
public static final String XMIN_XID_VALUE_COL = "xmin_xid_value";
@@ -81,4 +92,30 @@ public class PostgresQueryUtils {
.withXminRawValue(result.get(XMIN_RAW_VALUE_COL).asLong());
}
public static void logFullVacuumStatus(final JdbcDatabase database, final ConfiguredAirbyteCatalog catalog, final String quoteString) {
catalog.getStreams().forEach(stream -> {
final String streamName = stream.getStream().getName();
final String schemaName = stream.getStream().getNamespace();
final String fullTableName =
getFullyQualifiedTableNameWithQuoting(schemaName, streamName, quoteString);
LOGGER.info("Full Vacuum information for {}", fullTableName);
try {
List<JsonNode> jsonNodes = database.bufferedResultSetQuery(conn -> conn.prepareStatement(CTID_FULL_VACUUM_REL_FILENODE_QUERY.formatted(fullTableName)).executeQuery(),
resultSet -> JdbcUtils.getDefaultSourceOperations().rowToJson(resultSet));
Preconditions.checkState(jsonNodes.size() == 1);
LOGGER.info("Relation filenode is {}", jsonNodes.get(0).get("pg_relation_filenode"));
jsonNodes = database.bufferedResultSetQuery(conn -> conn.prepareStatement(CTID_FULL_VACUUM_IN_PROGRESS_QUERY.formatted(fullTableName)).executeQuery(),
resultSet -> JdbcUtils.getDefaultSourceOperations().rowToJson(resultSet));
if (jsonNodes.size() == 0) {
LOGGER.info("No full vacuum currently in progress");
} else {
Preconditions.checkState(jsonNodes.size() == 1);
LOGGER.info("Full Vacuum currently in progress in {} phase", jsonNodes.get(0).get("phase"));
}
} catch (SQLException e) {
LOGGER.warn("Failed to log full vacuum in progress. This warning shouldn't affect the sync and can be ignored", e);
}
});
}
}

1
airbyte-integrations/connectors/source-postgres/src/main/java/io/airbyte/integrations/source/postgres/PostgresSource.java
View File

@@ -216,6 +216,7 @@ public class PostgresSource extends AbstractJdbcSource<PostgresType> implements
protected void logPreSyncDebugData(final JdbcDatabase database, final ConfiguredAirbyteCatalog catalog)
throws SQLException {
super.logPreSyncDebugData(database, catalog);
PostgresQueryUtils.logFullVacuumStatus(database, catalog, getQuoteString());
for (final ConfiguredAirbyteStream stream : catalog.getStreams()) {
final String streamName = stream.getStream().getName();
final String schemaName = stream.getStream().getNamespace();

1
docs/integrations/sources/postgres.md
View File

@@ -399,6 +399,7 @@ Some larger tables may encounter an error related to the temporary file size lim
| Version | Date | Pull Request | Subject |
|:--------|:-----------|:---------------------------------------------------------|:---------------------------------------------------------------------------------------------------------------------------------------------------------------------------|
| 2.0.31 | 2023-05-25 | [26633](https://github.com/airbytehq/airbyte/pull/26633) | Collect and log information related to full vacuum operation in db |
| 2.0.30 | 2023-05-25 | [26473](https://github.com/airbytehq/airbyte/pull/26473) | CDC : Limit queue size |
| 2.0.29 | 2023-05-18 | [25898](https://github.com/airbytehq/airbyte/pull/25898) | Translate Numeric values without decimal, e.g: NUMERIC(38,0), as BigInt instead of Double |
| 2.0.28 | 2023-04-27 | [25401](https://github.com/airbytehq/airbyte/pull/25401) | CDC : Upgrade Debezium to version 2.2.0 |