jprdonnelly/airbyte
1
0
Fork 0
mirror of synced 2025-12-25 02:09:19 -05:00
Code Issues Packages Projects Releases Wiki Activity

Expose cron scheduling in the Connections APIs (#15253)

Browse Source 
* Expose cron scheduling in the Connections APIs

* Update airbyte-api/src/main/openapi/config.yaml

Co-authored-by: terencecho <terence@airbyte.io>

* Update airbyte-server/src/test/java/io/airbyte/server/helpers/ConnectionHelpers.java

Co-authored-by: terencecho <terence@airbyte.io>

* update octavia-cli tests for new schedule schema, and fix update API impl

* check for null schedule data before updating

* handle new schedule related fields in generate / apply / import

* update octavia-cli changelog

* ensure that legacy manual schedule flag is consistent with schedule_type

* update octavia cli test coverage for new schedule schema

* fix failing octavia cli integration tests

* fix file diff check

* Update octavia-cli/unit_tests/test_apply/test_resources.py

Co-authored-by: Augustin <augustin.lafanechere@gmail.com>

Co-authored-by: terencecho <terence@airbyte.io>
Co-authored-by: alafanechere <augustin.lafanechere@gmail.com>
This commit is contained in:
Michael Siega
2022-08-11 19:27:08 +02:00
committed by GitHub
parent f664bc9ee1
commit 294ee8f226
26 changed files with 10542 additions and 8496 deletions
Download Patch File Download Diff File Expand all files Collapse all files

74
airbyte-api/src/main/openapi/config.yaml
View File

@@ -3218,6 +3218,10 @@ components:
$ref: "#/components/schemas/AirbyteCatalog"
schedule:
$ref: "#/components/schemas/ConnectionSchedule"
scheduleType:
$ref: "#/components/schemas/ConnectionScheduleType"
scheduleData:
$ref: "#/components/schemas/ConnectionScheduleData"
status:
$ref: "#/components/schemas/ConnectionStatus"
resourceRequirements:
@@ -3257,6 +3261,10 @@ components:
$ref: "#/components/schemas/AirbyteCatalog"
schedule:
$ref: "#/components/schemas/ConnectionSchedule"
scheduleType:
$ref: "#/components/schemas/ConnectionScheduleType"
scheduleData:
$ref: "#/components/schemas/ConnectionScheduleData"
status:
$ref: "#/components/schemas/ConnectionStatus"
resourceRequirements:
@@ -3298,6 +3306,10 @@ components:
$ref: "#/components/schemas/AirbyteCatalog"
schedule:
$ref: "#/components/schemas/ConnectionSchedule"
scheduleType:
$ref: "#/components/schemas/ConnectionScheduleType"
scheduleData:
$ref: "#/components/schemas/ConnectionScheduleData"
status:
$ref: "#/components/schemas/ConnectionStatus"
resourceRequirements:
@@ -3335,6 +3347,10 @@ components:
$ref: "#/components/schemas/AirbyteCatalog"
schedule:
$ref: "#/components/schemas/ConnectionSchedule"
scheduleType:
$ref: "#/components/schemas/ConnectionScheduleType"
scheduleData:
$ref: "#/components/schemas/ConnectionScheduleData"
status:
$ref: "#/components/schemas/ConnectionStatus"
resourceRequirements:
@@ -3386,6 +3402,10 @@ components:
$ref: "#/components/schemas/AirbyteCatalog"
schedule:
$ref: "#/components/schemas/ConnectionSchedule"
scheduleType:
$ref: "#/components/schemas/ConnectionScheduleType"
scheduleData:
$ref: "#/components/schemas/ConnectionScheduleData"
status:
$ref: "#/components/schemas/ConnectionStatus"
resourceRequirements:
@@ -3416,6 +3436,10 @@ components:
$ref: "#/components/schemas/DestinationId"
schedule:
$ref: "#/components/schemas/ConnectionSchedule"
scheduleType:
$ref: "#/components/schemas/ConnectionScheduleType"
scheduleData:
$ref: "#/components/schemas/ConnectionScheduleData"
status:
$ref: "#/components/schemas/ConnectionStatus"
source:
@@ -3445,6 +3469,10 @@ components:
$ref: "#/components/schemas/DestinationId"
schedule:
$ref: "#/components/schemas/ConnectionSchedule"
scheduleType:
$ref: "#/components/schemas/ConnectionScheduleType"
scheduleData:
$ref: "#/components/schemas/ConnectionScheduleData"
status:
$ref: "#/components/schemas/ConnectionStatus"
source:
@@ -3467,6 +3495,8 @@ components:
- active
- inactive
- deprecated
# TODO(https://github.com/airbytehq/airbyte/issues/11432): remove.
# Prefer the ConnectionScheduleType and ConnectionScheduleData properties.
ConnectionSchedule:
description: if null, then no schedule is set.
type: object
@@ -3485,6 +3515,46 @@ components:
- days
- weeks
- months
ConnectionScheduleType:
description: determine how the schedule data should be interpreted
type: string
enum:
- manual
- basic
- cron
ConnectionScheduleData:
description: schedule for when the the connection should run, per the schedule type
type: object
properties:
# This should be populated when schedule type is basic.
basicSchedule:
type: object
required:
- timeUnit
- units
properties:
timeUnit:
type: string
enum:
- minutes
- hours
- days
- weeks
- months
units:
type: integer
format: int64
# This should be populated when schedule type is cron.
cron:
type: object
required:
- cronExpression
- cronTimeZone
properties:
cronExpression:
type: string
cronTimeZone:
type: string
NamespaceDefinitionType:
type: string
description: Method used for computing final namespace in destination
@@ -4564,6 +4634,10 @@ components:
$ref: "#/components/schemas/AirbyteCatalog"
schedule:
$ref: "#/components/schemas/ConnectionSchedule"
scheduleType:
$ref: "#/components/schemas/ConnectionScheduleType"
scheduleData:
$ref: "#/components/schemas/ConnectionScheduleData"
status:
$ref: "#/components/schemas/ConnectionStatus"
operationIds:

1
airbyte-server/build.gradle
View File

@@ -32,6 +32,7 @@ dependencies {
implementation 'org.glassfish.jersey.inject:jersey-hk2'
implementation 'org.glassfish.jersey.media:jersey-media-json-jackson'
implementation 'org.glassfish.jersey.ext:jersey-bean-validation'
implementation 'org.quartz-scheduler:quartz:2.3.2'
testImplementation project(':airbyte-test-utils')

100
airbyte-server/src/main/java/io/airbyte/server/converters/ApiPojoConverters.java
View File

@@ -4,9 +4,13 @@
package io.airbyte.server.converters;
import io.airbyte.api.client.model.generated.ConnectionScheduleType;
import io.airbyte.api.model.generated.ActorDefinitionResourceRequirements;
import io.airbyte.api.model.generated.ConnectionRead;
import io.airbyte.api.model.generated.ConnectionSchedule;
import io.airbyte.api.model.generated.ConnectionScheduleData;
import io.airbyte.api.model.generated.ConnectionScheduleDataBasicSchedule;
import io.airbyte.api.model.generated.ConnectionScheduleDataCron;
import io.airbyte.api.model.generated.ConnectionStatus;
import io.airbyte.api.model.generated.ConnectionUpdate;
import io.airbyte.api.model.generated.JobType;
@@ -17,7 +21,10 @@ import io.airbyte.config.BasicSchedule;
import io.airbyte.config.JobSyncConfig.NamespaceDefinitionType;
import io.airbyte.config.Schedule;
import io.airbyte.config.StandardSync;
import io.airbyte.config.StandardSync.ScheduleType;
import io.airbyte.server.handlers.helpers.CatalogConverter;
import io.airbyte.server.handlers.helpers.ConnectionScheduleHelper;
import io.airbyte.validation.json.JsonValidationException;
import java.util.stream.Collectors;
public class ApiPojoConverters {
@@ -78,7 +85,7 @@ public class ApiPojoConverters {
.memoryLimit(resourceReqs.getMemoryLimit());
}
public static io.airbyte.config.StandardSync connectionUpdateToInternal(final ConnectionUpdate update) {
public static io.airbyte.config.StandardSync connectionUpdateToInternal(final ConnectionUpdate update) throws JsonValidationException {
final StandardSync newConnection = new StandardSync()
.withNamespaceDefinition(Enums.convertTo(update.getNamespaceDefinition(), NamespaceDefinitionType.class))
@@ -99,7 +106,9 @@ public class ApiPojoConverters {
}
// update sync schedule
if (update.getSchedule() != null) {
if (update.getScheduleType() != null) {
ConnectionScheduleHelper.populateSyncFromScheduleTypeAndData(newConnection, update.getScheduleType(), update.getScheduleData());
} else if (update.getSchedule() != null) {
final Schedule newSchedule = new Schedule()
.withTimeUnit(toPersistenceTimeUnit(update.getSchedule().getTimeUnit()))
.withUnits(update.getSchedule().getUnits());
@@ -112,21 +121,12 @@ public class ApiPojoConverters {
}
public static ConnectionRead internalToConnectionRead(final StandardSync standardSync) {
ConnectionSchedule apiSchedule = null;
if (!standardSync.getManual()) {
apiSchedule = new ConnectionSchedule()
.timeUnit(toApiTimeUnit(standardSync.getSchedule().getTimeUnit()))
.units(standardSync.getSchedule().getUnits());
}
final ConnectionRead connectionRead = new ConnectionRead()
.connectionId(standardSync.getConnectionId())
.sourceId(standardSync.getSourceId())
.destinationId(standardSync.getDestinationId())
.operationIds(standardSync.getOperationIds())
.status(toApiStatus(standardSync.getStatus()))
.schedule(apiSchedule)
.name(standardSync.getName())
.namespaceDefinition(Enums.convertTo(standardSync.getNamespaceDefinition(), io.airbyte.api.model.generated.NamespaceDefinitionType.class))
.namespaceFormat(standardSync.getNamespaceFormat())
@@ -138,6 +138,8 @@ public class ApiPojoConverters {
connectionRead.resourceRequirements(resourceRequirementsToApi(standardSync.getResourceRequirements()));
}
populateConnectionReadSchedule(standardSync, connectionRead);
return connectionRead;
}
@@ -149,10 +151,15 @@ public class ApiPojoConverters {
return Enums.convertTo(jobType, io.airbyte.config.JobTypeResourceLimit.JobType.class);
}
// TODO(https://github.com/airbytehq/airbyte/issues/11432): remove these helpers.
public static ConnectionSchedule.TimeUnitEnum toApiTimeUnit(final Schedule.TimeUnit apiTimeUnit) {
return Enums.convertTo(apiTimeUnit, ConnectionSchedule.TimeUnitEnum.class);
}
public static ConnectionSchedule.TimeUnitEnum toApiTimeUnit(final BasicSchedule.TimeUnit timeUnit) {
return Enums.convertTo(timeUnit, ConnectionSchedule.TimeUnitEnum.class);
}
public static ConnectionStatus toApiStatus(final StandardSync.Status status) {
return Enums.convertTo(status, ConnectionStatus.class);
}
@@ -169,4 +176,75 @@ public class ApiPojoConverters {
return Enums.convertTo(apiTimeUnit, BasicSchedule.TimeUnit.class);
}
public static BasicSchedule.TimeUnit toBasicScheduleTimeUnit(final ConnectionScheduleDataBasicSchedule.TimeUnitEnum apiTimeUnit) {
return Enums.convertTo(apiTimeUnit, BasicSchedule.TimeUnit.class);
}
public static ConnectionScheduleDataBasicSchedule.TimeUnitEnum toApiBasicScheduleTimeUnit(final BasicSchedule.TimeUnit timeUnit) {
return Enums.convertTo(timeUnit, ConnectionScheduleDataBasicSchedule.TimeUnitEnum.class);
}
public static ConnectionScheduleDataBasicSchedule.TimeUnitEnum toApiBasicScheduleTimeUnit(final Schedule.TimeUnit timeUnit) {
return Enums.convertTo(timeUnit, ConnectionScheduleDataBasicSchedule.TimeUnitEnum.class);
}
public static void populateConnectionReadSchedule(final StandardSync standardSync, final ConnectionRead connectionRead) {
// TODO(https://github.com/airbytehq/airbyte/issues/11432): only return new schema once frontend is
// ready.
if (standardSync.getScheduleType() != null) {
// Populate everything based on the new schema.
switch (standardSync.getScheduleType()) {
case MANUAL -> {
connectionRead.scheduleType(io.airbyte.api.model.generated.ConnectionScheduleType.MANUAL);
}
case BASIC_SCHEDULE -> {
connectionRead.scheduleType(io.airbyte.api.model.generated.ConnectionScheduleType.BASIC);
connectionRead.scheduleData(new ConnectionScheduleData()
.basicSchedule(new ConnectionScheduleDataBasicSchedule()
.timeUnit(toApiBasicScheduleTimeUnit(standardSync.getScheduleData().getBasicSchedule().getTimeUnit()))
.units(standardSync.getScheduleData().getBasicSchedule().getUnits())));
connectionRead.schedule(new ConnectionSchedule()
.timeUnit(toApiTimeUnit(standardSync.getScheduleData().getBasicSchedule().getTimeUnit()))
.units(standardSync.getScheduleData().getBasicSchedule().getUnits()));
}
case CRON -> {
// We don't populate any legacy data here.
connectionRead.scheduleType(io.airbyte.api.model.generated.ConnectionScheduleType.CRON);
connectionRead.scheduleData(new ConnectionScheduleData()
.cron(new ConnectionScheduleDataCron()
.cronExpression(standardSync.getScheduleData().getCron().getCronExpression())
.cronTimeZone(standardSync.getScheduleData().getCron().getCronTimeZone())));
}
}
} else if (standardSync.getManual()) {
// Legacy schema, manual sync.
connectionRead.scheduleType(io.airbyte.api.model.generated.ConnectionScheduleType.MANUAL);
} else {
// Legacy schema, basic schedule.
connectionRead.scheduleType(io.airbyte.api.model.generated.ConnectionScheduleType.BASIC);
connectionRead.schedule(new ConnectionSchedule()
.timeUnit(toApiTimeUnit(standardSync.getSchedule().getTimeUnit()))
.units(standardSync.getSchedule().getUnits()));
connectionRead.scheduleData(new ConnectionScheduleData()
.basicSchedule(new ConnectionScheduleDataBasicSchedule()
.timeUnit(toApiBasicScheduleTimeUnit(standardSync.getSchedule().getTimeUnit()))
.units(standardSync.getSchedule().getUnits())));
}
}
public static ConnectionScheduleType toApiScheduleType(final ScheduleType scheduleType) {
switch (scheduleType) {
case MANUAL -> {
return ConnectionScheduleType.MANUAL;
}
case BASIC_SCHEDULE -> {
return ConnectionScheduleType.BASIC;
}
case CRON -> {
return ConnectionScheduleType.CRON;
}
}
throw new RuntimeException("Unexpected schedule type");
}
}

44
airbyte-server/src/main/java/io/airbyte/server/handlers/ConnectionsHandler.java
View File

@@ -47,6 +47,7 @@ import io.airbyte.server.converters.ApiPojoConverters;
import io.airbyte.server.converters.CatalogDiffConverters;
import io.airbyte.server.handlers.helpers.CatalogConverter;
import io.airbyte.server.handlers.helpers.ConnectionMatcher;
import io.airbyte.server.handlers.helpers.ConnectionScheduleHelper;
import io.airbyte.server.handlers.helpers.DestinationMatcher;
import io.airbyte.server.handlers.helpers.SourceMatcher;
import io.airbyte.validation.json.JsonValidationException;
@@ -140,6 +141,34 @@ public class ConnectionsHandler {
standardSync.withCatalog(new ConfiguredAirbyteCatalog().withStreams(Collections.emptyList()));
}
if (connectionCreate.getSchedule() != null && connectionCreate.getScheduleType() != null) {
throw new JsonValidationException("supply old or new schedule schema but not both");
}
if (connectionCreate.getScheduleType() != null) {
ConnectionScheduleHelper.populateSyncFromScheduleTypeAndData(standardSync, connectionCreate.getScheduleType(),
connectionCreate.getScheduleData());
} else {
populateSyncFromLegacySchedule(standardSync, connectionCreate);
}
configRepository.writeStandardSync(standardSync);
trackNewConnection(standardSync);
try {
LOGGER.info("Starting a connection manager workflow");
eventRunner.createConnectionManagerWorkflow(connectionId);
} catch (final Exception e) {
LOGGER.error("Start of the connection manager workflow failed", e);
configRepository.deleteStandardSyncDefinition(standardSync.getConnectionId());
throw e;
}
return buildConnectionRead(connectionId);
}
private void populateSyncFromLegacySchedule(final StandardSync standardSync, final ConnectionCreate connectionCreate) {
if (connectionCreate.getSchedule() != null) {
final Schedule schedule = new Schedule()
.withTimeUnit(ApiPojoConverters.toPersistenceTimeUnit(connectionCreate.getSchedule().getTimeUnit()))
@@ -159,21 +188,6 @@ public class ConnectionsHandler {
standardSync.withManual(true);
standardSync.withScheduleType(ScheduleType.MANUAL);
}
configRepository.writeStandardSync(standardSync);
trackNewConnection(standardSync);
try {
LOGGER.info("Starting a connection manager workflow");
eventRunner.createConnectionManagerWorkflow(connectionId);
} catch (final Exception e) {
LOGGER.error("Start of the connection manager workflow failed", e);
configRepository.deleteStandardSyncDefinition(standardSync.getConnectionId());
throw e;
}
return buildConnectionRead(connectionId);
}
private void trackNewConnection(final StandardSync standardSync) {

8
airbyte-server/src/main/java/io/airbyte/server/handlers/WebBackendConnectionsHandler.java
View File

@@ -177,6 +177,8 @@ public class WebBackendConnectionsHandler {
.syncCatalog(connectionRead.getSyncCatalog())
.status(connectionRead.getStatus())
.schedule(connectionRead.getSchedule())
.scheduleType(connectionRead.getScheduleType())
.scheduleData(connectionRead.getScheduleData())
.source(source)
.destination(destination)
.operations(operations.getOperations())
@@ -495,6 +497,8 @@ public class WebBackendConnectionsHandler {
connectionCreate.operationIds(operationIds);
connectionCreate.syncCatalog(webBackendConnectionCreate.getSyncCatalog());
connectionCreate.schedule(webBackendConnectionCreate.getSchedule());
connectionCreate.scheduleType(webBackendConnectionCreate.getScheduleType());
connectionCreate.scheduleData(webBackendConnectionCreate.getScheduleData());
connectionCreate.status(webBackendConnectionCreate.getStatus());
connectionCreate.resourceRequirements(webBackendConnectionCreate.getResourceRequirements());
connectionCreate.sourceCatalogId(webBackendConnectionCreate.getSourceCatalogId());
@@ -514,6 +518,8 @@ public class WebBackendConnectionsHandler {
connectionUpdate.operationIds(operationIds);
connectionUpdate.syncCatalog(webBackendConnectionUpdate.getSyncCatalog());
connectionUpdate.schedule(webBackendConnectionUpdate.getSchedule());
connectionUpdate.scheduleType(webBackendConnectionUpdate.getScheduleType());
connectionUpdate.scheduleData(webBackendConnectionUpdate.getScheduleData());
connectionUpdate.status(webBackendConnectionUpdate.getStatus());
connectionUpdate.resourceRequirements(webBackendConnectionUpdate.getResourceRequirements());
connectionUpdate.sourceCatalogId(webBackendConnectionUpdate.getSourceCatalogId());
@@ -534,6 +540,8 @@ public class WebBackendConnectionsHandler {
.namespaceFormat(webBackendConnectionSearch.getNamespaceFormat())
.prefix(webBackendConnectionSearch.getPrefix())
.schedule(webBackendConnectionSearch.getSchedule())
.scheduleType(webBackendConnectionSearch.getScheduleType())
.scheduleData(webBackendConnectionSearch.getScheduleData())
.status(webBackendConnectionSearch.getStatus());
}

2
airbyte-server/src/main/java/io/airbyte/server/handlers/helpers/ConnectionMatcher.java
View File

@@ -33,6 +33,8 @@ public class ConnectionMatcher implements Matchable<ConnectionRead> {
search.getNamespaceDefinition() == null ? query.getNamespaceDefinition() : search.getNamespaceDefinition());
fromSearch.prefix(Strings.isBlank(search.getPrefix()) ? query.getPrefix() : search.getPrefix());
fromSearch.schedule(search.getSchedule() == null ? query.getSchedule() : search.getSchedule());
fromSearch.scheduleType(search.getScheduleType() == null ? query.getScheduleType() : search.getScheduleType());
fromSearch.scheduleData(search.getScheduleData() == null ? query.getScheduleData() : search.getScheduleData());
fromSearch.sourceId(search.getSourceId() == null ? query.getSourceId() : search.getSourceId());
fromSearch.status(search.getStatus() == null ? query.getStatus() : search.getStatus());

80
airbyte-server/src/main/java/io/airbyte/server/handlers/helpers/ConnectionScheduleHelper.java Normal file
View File

@@ -0,0 +1,80 @@
/*
* Copyright (c) 2022 Airbyte, Inc., all rights reserved.
*/
package io.airbyte.server.handlers.helpers;
import io.airbyte.api.model.generated.ConnectionScheduleData;
import io.airbyte.api.model.generated.ConnectionScheduleType;
import io.airbyte.config.BasicSchedule;
import io.airbyte.config.Cron;
import io.airbyte.config.ScheduleData;
import io.airbyte.config.StandardSync;
import io.airbyte.config.StandardSync.ScheduleType;
import io.airbyte.server.converters.ApiPojoConverters;
import io.airbyte.validation.json.JsonValidationException;
import java.text.ParseException;
import java.util.TimeZone;
import org.joda.time.DateTimeZone;
import org.quartz.CronExpression;
/**
* Helper class to handle connection schedules, including validation and translating between API and
* config.
*/
public class ConnectionScheduleHelper {
public static void populateSyncFromScheduleTypeAndData(final StandardSync standardSync,
final ConnectionScheduleType scheduleType,
final ConnectionScheduleData scheduleData)
throws JsonValidationException {
if (scheduleType != ConnectionScheduleType.MANUAL && scheduleData == null) {
throw new JsonValidationException("schedule data must be populated if schedule type is populated");
}
switch (scheduleType) {
// NOTE: the `manual` column is marked required, so we populate it until it's removed.
case MANUAL -> standardSync.withScheduleType(ScheduleType.MANUAL).withManual(true);
case BASIC -> {
if (scheduleData.getBasicSchedule() == null) {
throw new JsonValidationException("if schedule type is basic, then scheduleData.basic must be populated");
}
standardSync
.withScheduleType(ScheduleType.BASIC_SCHEDULE)
.withScheduleData(new ScheduleData().withBasicSchedule(
new BasicSchedule().withTimeUnit(ApiPojoConverters.toBasicScheduleTimeUnit(scheduleData.getBasicSchedule().getTimeUnit()))
.withUnits(scheduleData.getBasicSchedule().getUnits())))
.withManual(false);
}
case CRON -> {
if (scheduleData.getCron() == null) {
throw new JsonValidationException("if schedule type is cron, then scheduleData.cron must be populated");
}
// Validate that this is a valid cron expression and timezone.
final String cronExpression = scheduleData.getCron().getCronExpression();
final String cronTimeZone = scheduleData.getCron().getCronTimeZone();
if (cronExpression == null || cronTimeZone == null) {
throw new JsonValidationException("Cron expression and timezone are required");
}
if (cronTimeZone.toLowerCase().startsWith("etc")) {
throw new JsonValidationException("Etc/ timezones are unsupported");
}
try {
final TimeZone timeZone = DateTimeZone.forID(cronTimeZone).toTimeZone();
final CronExpression parsedCronExpression = new CronExpression(cronExpression);
parsedCronExpression.setTimeZone(timeZone);
} catch (ParseException e) {
throw (JsonValidationException) new JsonValidationException("invalid cron expression").initCause(e);
} catch (IllegalArgumentException e) {
throw (JsonValidationException) new JsonValidationException("invalid cron timezone").initCause(e);
}
standardSync
.withScheduleType(ScheduleType.CRON)
.withScheduleData(new ScheduleData().withCron(new Cron()
.withCronExpression(cronExpression)
.withCronTimeZone(cronTimeZone)))
.withManual(false);
}
}
}
}

90
airbyte-server/src/test/java/io/airbyte/server/handlers/ConnectionSchedulerHelperTest.java Normal file
View File

@@ -0,0 +1,90 @@
/*
* Copyright (c) 2022 Airbyte, Inc., all rights reserved.
*/
package io.airbyte.server.handlers;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertFalse;
import static org.junit.jupiter.api.Assertions.assertNull;
import static org.junit.jupiter.api.Assertions.assertThrows;
import static org.junit.jupiter.api.Assertions.assertTrue;
import io.airbyte.api.model.generated.ConnectionScheduleData;
import io.airbyte.api.model.generated.ConnectionScheduleDataBasicSchedule;
import io.airbyte.api.model.generated.ConnectionScheduleDataBasicSchedule.TimeUnitEnum;
import io.airbyte.api.model.generated.ConnectionScheduleDataCron;
import io.airbyte.api.model.generated.ConnectionScheduleType;
import io.airbyte.config.BasicSchedule.TimeUnit;
import io.airbyte.config.StandardSync;
import io.airbyte.config.StandardSync.ScheduleType;
import io.airbyte.server.handlers.helpers.ConnectionScheduleHelper;
import io.airbyte.validation.json.JsonValidationException;
import org.junit.jupiter.api.Test;
class ConnectionSchedulerHelperTest {
private final static String EXPECTED_CRON_TIMEZONE = "UTC";
private final static String EXPECTED_CRON_EXPRESSION = "* */2 * * * ?";
@Test
void testPopulateSyncScheduleFromManualType() throws JsonValidationException {
final StandardSync actual = new StandardSync();
ConnectionScheduleHelper.populateSyncFromScheduleTypeAndData(actual,
ConnectionScheduleType.MANUAL, null);
assertTrue(actual.getManual());
assertEquals(ScheduleType.MANUAL, actual.getScheduleType());
assertNull(actual.getSchedule());
assertNull(actual.getScheduleData());
}
@Test
void testPopulateSyncScheduleFromBasicType() throws JsonValidationException {
final StandardSync actual = new StandardSync();
ConnectionScheduleHelper.populateSyncFromScheduleTypeAndData(actual,
ConnectionScheduleType.BASIC, new ConnectionScheduleData()
.basicSchedule(new ConnectionScheduleDataBasicSchedule()
.timeUnit(TimeUnitEnum.HOURS)
.units(1L)));
assertFalse(actual.getManual());
assertEquals(ScheduleType.BASIC_SCHEDULE, actual.getScheduleType());
assertEquals(TimeUnit.HOURS, actual.getScheduleData().getBasicSchedule().getTimeUnit());
assertEquals(1L, actual.getScheduleData().getBasicSchedule().getUnits());
assertNull(actual.getSchedule());
}
@Test
void testPopulateSyncScheduleFromCron() throws JsonValidationException {
final StandardSync actual = new StandardSync();
ConnectionScheduleHelper.populateSyncFromScheduleTypeAndData(actual,
ConnectionScheduleType.CRON, new ConnectionScheduleData()
.cron(new ConnectionScheduleDataCron()
.cronTimeZone(EXPECTED_CRON_TIMEZONE)
.cronExpression(EXPECTED_CRON_EXPRESSION)));
assertEquals(ScheduleType.CRON, actual.getScheduleType());
assertEquals(EXPECTED_CRON_TIMEZONE, actual.getScheduleData().getCron().getCronTimeZone());
assertEquals(EXPECTED_CRON_EXPRESSION, actual.getScheduleData().getCron().getCronExpression());
assertNull(actual.getSchedule());
}
@Test
void testScheduleValidation() throws JsonValidationException {
final StandardSync actual = new StandardSync();
assertThrows(JsonValidationException.class, () -> ConnectionScheduleHelper.populateSyncFromScheduleTypeAndData(actual,
ConnectionScheduleType.CRON, null));
assertThrows(JsonValidationException.class,
() -> ConnectionScheduleHelper.populateSyncFromScheduleTypeAndData(actual, ConnectionScheduleType.BASIC, new ConnectionScheduleData()));
assertThrows(JsonValidationException.class,
() -> ConnectionScheduleHelper.populateSyncFromScheduleTypeAndData(actual, ConnectionScheduleType.CRON, new ConnectionScheduleData()));
assertThrows(JsonValidationException.class,
() -> ConnectionScheduleHelper.populateSyncFromScheduleTypeAndData(actual, ConnectionScheduleType.CRON, new ConnectionScheduleData()
.cron(new ConnectionScheduleDataCron())));
assertThrows(JsonValidationException.class,
() -> ConnectionScheduleHelper.populateSyncFromScheduleTypeAndData(actual, ConnectionScheduleType.CRON, new ConnectionScheduleData()
.cron(new ConnectionScheduleDataCron().cronExpression(EXPECTED_CRON_EXPRESSION).cronTimeZone("Etc/foo"))));
assertThrows(JsonValidationException.class,
() -> ConnectionScheduleHelper.populateSyncFromScheduleTypeAndData(actual, ConnectionScheduleType.CRON, new ConnectionScheduleData()
.cron(new ConnectionScheduleDataCron().cronExpression("bad cron").cronTimeZone(EXPECTED_CRON_TIMEZONE))));
}
}

10
airbyte-server/src/test/java/io/airbyte/server/handlers/ConnectionsHandlerTest.java
View File

@@ -22,6 +22,7 @@ import io.airbyte.api.model.generated.ConnectionCreate;
import io.airbyte.api.model.generated.ConnectionRead;
import io.airbyte.api.model.generated.ConnectionReadList;
import io.airbyte.api.model.generated.ConnectionSchedule;
import io.airbyte.api.model.generated.ConnectionScheduleType;
import io.airbyte.api.model.generated.ConnectionSearch;
import io.airbyte.api.model.generated.ConnectionStatus;
import io.airbyte.api.model.generated.ConnectionUpdate;
@@ -216,6 +217,13 @@ class ConnectionsHandlerTest {
assertEquals(expectedConnectionRead, actualConnectionRead);
verify(configRepository).writeStandardSync(standardSync);
// Use new schedule schema, verify that we get the same results.
connectionCreate
.schedule(null)
.scheduleType(ConnectionScheduleType.BASIC)
.scheduleData(ConnectionHelpers.generateBasicConnectionScheduleData());
assertEquals(expectedConnectionRead, connectionsHandler.createConnection(connectionCreate));
}
@Test
@@ -360,6 +368,8 @@ class ConnectionsHandlerTest {
standardSync.getOperationIds(),
newSourceCatalogId)
.schedule(null)
.scheduleType(ConnectionScheduleType.MANUAL)
.scheduleData(null)
.syncCatalog(catalog)
.status(ConnectionStatus.INACTIVE);

8
airbyte-server/src/test/java/io/airbyte/server/handlers/WebBackendConnectionsHandlerTest.java
View File

@@ -205,6 +205,8 @@ class WebBackendConnectionsHandlerTest {
.syncCatalog(connectionRead.getSyncCatalog())
.status(connectionRead.getStatus())
.schedule(connectionRead.getSchedule())
.scheduleType(connectionRead.getScheduleType())
.scheduleData(connectionRead.getScheduleData())
.source(sourceRead)
.destination(destinationRead)
.operations(operationReadList.getOperations())
@@ -239,6 +241,8 @@ class WebBackendConnectionsHandlerTest {
.syncCatalog(modifiedCatalog)
.status(expected.getStatus())
.schedule(expected.getSchedule())
.scheduleType(expected.getScheduleType())
.scheduleData(expected.getScheduleData())
.source(expected.getSource())
.destination(expected.getDestination())
.operations(expected.getOperations())
@@ -481,7 +485,7 @@ class WebBackendConnectionsHandlerTest {
void testForConnectionCreateCompleteness() {
final Set<String> handledMethods =
Set.of("name", "namespaceDefinition", "namespaceFormat", "prefix", "sourceId", "destinationId", "operationIds", "syncCatalog", "schedule",
"status", "resourceRequirements", "sourceCatalogId");
"scheduleType", "scheduleData", "status", "resourceRequirements", "sourceCatalogId");
final Set<String> methods = Arrays.stream(ConnectionCreate.class.getMethods())
.filter(method -> method.getReturnType() == ConnectionCreate.class)
@@ -502,7 +506,7 @@ class WebBackendConnectionsHandlerTest {
void testForConnectionUpdateCompleteness() {
final Set<String> handledMethods =
Set.of("schedule", "connectionId", "syncCatalog", "namespaceDefinition", "namespaceFormat", "prefix", "status", "operationIds",
"resourceRequirements", "name", "sourceCatalogId");
"resourceRequirements", "name", "sourceCatalogId", "scheduleType", "scheduleData");
final Set<String> methods = Arrays.stream(ConnectionUpdate.class.getMethods())
.filter(method -> method.getReturnType() == ConnectionUpdate.class)

18
airbyte-server/src/test/java/io/airbyte/server/helpers/ConnectionHelpers.java
View File

@@ -13,6 +13,9 @@ import io.airbyte.api.model.generated.AirbyteStreamConfiguration;
import io.airbyte.api.model.generated.ConnectionRead;
import io.airbyte.api.model.generated.ConnectionSchedule;
import io.airbyte.api.model.generated.ConnectionSchedule.TimeUnitEnum;
import io.airbyte.api.model.generated.ConnectionScheduleData;
import io.airbyte.api.model.generated.ConnectionScheduleDataBasicSchedule;
import io.airbyte.api.model.generated.ConnectionScheduleType;
import io.airbyte.api.model.generated.ConnectionStatus;
import io.airbyte.api.model.generated.ResourceRequirements;
import io.airbyte.api.model.generated.SyncMode;
@@ -30,6 +33,7 @@ import io.airbyte.protocol.models.DestinationSyncMode;
import io.airbyte.protocol.models.Field;
import io.airbyte.protocol.models.JsonSchemaType;
import io.airbyte.protocol.models.StreamDescriptor;
import io.airbyte.server.converters.ApiPojoConverters;
import io.airbyte.server.handlers.helpers.CatalogConverter;
import java.util.ArrayList;
import java.util.Collections;
@@ -105,6 +109,11 @@ public class ConnectionHelpers {
.withUnits(BASIC_SCHEDULE_UNITS);
}
public static ConnectionScheduleData generateBasicConnectionScheduleData() {
return new ConnectionScheduleData().basicSchedule(
new ConnectionScheduleDataBasicSchedule().timeUnit(ConnectionScheduleDataBasicSchedule.TimeUnitEnum.DAYS).units(BASIC_SCHEDULE_UNITS));
}
public static ScheduleData generateBasicScheduleData() {
return new ScheduleData().withBasicSchedule(new BasicSchedule()
.withTimeUnit(BasicSchedule.TimeUnit.fromValue((BASIC_SCHEDULE_DATA_TIME_UNITS)))
@@ -128,6 +137,8 @@ public class ConnectionHelpers {
.prefix("presto_to_hudi")
.status(ConnectionStatus.ACTIVE)
.schedule(generateBasicConnectionSchedule())
.scheduleType(ConnectionScheduleType.BASIC)
.scheduleData(generateBasicConnectionScheduleData())
.syncCatalog(ConnectionHelpers.generateBasicApiCatalog())
.resourceRequirements(new ResourceRequirements()
.cpuRequest(TESTING_RESOURCE_REQUIREMENTS.getCpuRequest())
@@ -175,11 +186,8 @@ public class ConnectionHelpers {
if (standardSync.getStatus() != null) {
connectionRead.status(io.airbyte.api.model.generated.ConnectionStatus.fromValue(standardSync.getStatus().value()));
}
if (standardSync.getSchedule() != null) {
connectionRead.schedule(new io.airbyte.api.model.generated.ConnectionSchedule()
.timeUnit(TimeUnitEnum.fromValue(standardSync.getSchedule().getTimeUnit().value()))
.units(standardSync.getSchedule().getUnits()));
}
ApiPojoConverters.populateConnectionReadSchedule(standardSync, connectionRead);
if (standardSync.getCatalog() != null) {
connectionRead.syncCatalog(CatalogConverter.toApi(standardSync.getCatalog()));
}

1
airbyte-workers/build.gradle
View File

@@ -23,6 +23,7 @@ dependencies {
implementation libs.micrometer.statsd
implementation project(':airbyte-analytics')
implementation project(':airbyte-api')
implementation project(':airbyte-commons-docker')
implementation project(':airbyte-config:config-models')
implementation project(':airbyte-config:config-persistence')

9
airbyte-workers/src/main/java/io/airbyte/workers/helper/ConnectionHelper.java
View File

@@ -86,8 +86,13 @@ public class ConnectionHelper {
newConnection.withResourceRequirements(original.getResourceRequirements());
}
// update sync schedule
if (update.getSchedule() != null) {
if (update.getScheduleType() != null) {
newConnection.withScheduleType(update.getScheduleType());
newConnection.withManual(update.getManual());
if (update.getScheduleData() != null) {
newConnection.withScheduleData(Jsons.clone(update.getScheduleData()));
}
} else if (update.getSchedule() != null) {
final Schedule newSchedule = new Schedule()
.withTimeUnit(update.getSchedule().getTimeUnit())
.withUnits(update.getSchedule().getUnits());

280
docs/reference/api/generated-api-html/index.html
View File

@@ -470,7 +470,17 @@ font-style: italic;
},
"connectionId" : "046b6c7f-0b8a-43b9-b35d-6489e6daee91",
"namespaceFormat" : "${SOURCE_NAMESPACE}",
"operationIds" : [ null, null ]
"operationIds" : [ null, null ],
"scheduleData" : {
"cron" : {
"cronExpression" : "cronExpression",
"cronTimeZone" : "cronTimeZone"
},
"basicSchedule" : {
"units" : 6,
"timeUnit" : "minutes"
}
}
}</code></pre>
<h3 class="field-label">Produces</h3>
@@ -623,7 +633,17 @@ font-style: italic;
},
"connectionId" : "046b6c7f-0b8a-43b9-b35d-6489e6daee91",
"namespaceFormat" : "${SOURCE_NAMESPACE}",
"operationIds" : [ null, null ]
"operationIds" : [ null, null ],
"scheduleData" : {
"cron" : {
"cronExpression" : "cronExpression",
"cronTimeZone" : "cronTimeZone"
},
"basicSchedule" : {
"units" : 6,
"timeUnit" : "minutes"
}
}
}</code></pre>
<h3 class="field-label">Produces</h3>
@@ -870,7 +890,17 @@ font-style: italic;
},
"connectionId" : "046b6c7f-0b8a-43b9-b35d-6489e6daee91",
"namespaceFormat" : "${SOURCE_NAMESPACE}",
"operationIds" : [ null, null ]
"operationIds" : [ null, null ],
"scheduleData" : {
"cron" : {
"cronExpression" : "cronExpression",
"cronTimeZone" : "cronTimeZone"
},
"basicSchedule" : {
"units" : 6,
"timeUnit" : "minutes"
}
}
}, {
"sourceId" : "046b6c7f-0b8a-43b9-b35d-6489e6daee91",
"sourceCatalogId" : "046b6c7f-0b8a-43b9-b35d-6489e6daee91",
@@ -922,7 +952,17 @@ font-style: italic;
},
"connectionId" : "046b6c7f-0b8a-43b9-b35d-6489e6daee91",
"namespaceFormat" : "${SOURCE_NAMESPACE}",
"operationIds" : [ null, null ]
"operationIds" : [ null, null ],
"scheduleData" : {
"cron" : {
"cronExpression" : "cronExpression",
"cronTimeZone" : "cronTimeZone"
},
"basicSchedule" : {
"units" : 6,
"timeUnit" : "minutes"
}
}
} ]
}</code></pre>
@@ -1032,7 +1072,17 @@ font-style: italic;
},
"connectionId" : "046b6c7f-0b8a-43b9-b35d-6489e6daee91",
"namespaceFormat" : "${SOURCE_NAMESPACE}",
"operationIds" : [ null, null ]
"operationIds" : [ null, null ],
"scheduleData" : {
"cron" : {
"cronExpression" : "cronExpression",
"cronTimeZone" : "cronTimeZone"
},
"basicSchedule" : {
"units" : 6,
"timeUnit" : "minutes"
}
}
}, {
"sourceId" : "046b6c7f-0b8a-43b9-b35d-6489e6daee91",
"sourceCatalogId" : "046b6c7f-0b8a-43b9-b35d-6489e6daee91",
@@ -1084,7 +1134,17 @@ font-style: italic;
},
"connectionId" : "046b6c7f-0b8a-43b9-b35d-6489e6daee91",
"namespaceFormat" : "${SOURCE_NAMESPACE}",
"operationIds" : [ null, null ]
"operationIds" : [ null, null ],
"scheduleData" : {
"cron" : {
"cronExpression" : "cronExpression",
"cronTimeZone" : "cronTimeZone"
},
"basicSchedule" : {
"units" : 6,
"timeUnit" : "minutes"
}
}
} ]
}</code></pre>
@@ -1369,7 +1429,17 @@ font-style: italic;
},
"connectionId" : "046b6c7f-0b8a-43b9-b35d-6489e6daee91",
"namespaceFormat" : "${SOURCE_NAMESPACE}",
"operationIds" : [ null, null ]
"operationIds" : [ null, null ],
"scheduleData" : {
"cron" : {
"cronExpression" : "cronExpression",
"cronTimeZone" : "cronTimeZone"
},
"basicSchedule" : {
"units" : 6,
"timeUnit" : "minutes"
}
}
}, {
"sourceId" : "046b6c7f-0b8a-43b9-b35d-6489e6daee91",
"sourceCatalogId" : "046b6c7f-0b8a-43b9-b35d-6489e6daee91",
@@ -1421,7 +1491,17 @@ font-style: italic;
},
"connectionId" : "046b6c7f-0b8a-43b9-b35d-6489e6daee91",
"namespaceFormat" : "${SOURCE_NAMESPACE}",
"operationIds" : [ null, null ]
"operationIds" : [ null, null ],
"scheduleData" : {
"cron" : {
"cronExpression" : "cronExpression",
"cronTimeZone" : "cronTimeZone"
},
"basicSchedule" : {
"units" : 6,
"timeUnit" : "minutes"
}
}
} ]
}</code></pre>
@@ -1702,7 +1782,17 @@ font-style: italic;
},
"connectionId" : "046b6c7f-0b8a-43b9-b35d-6489e6daee91",
"namespaceFormat" : "${SOURCE_NAMESPACE}",
"operationIds" : [ null, null ]
"operationIds" : [ null, null ],
"scheduleData" : {
"cron" : {
"cronExpression" : "cronExpression",
"cronTimeZone" : "cronTimeZone"
},
"basicSchedule" : {
"units" : 6,
"timeUnit" : "minutes"
}
}
}</code></pre>
<h3 class="field-label">Produces</h3>
@@ -8151,7 +8241,17 @@ font-style: italic;
},
"connectionId" : "046b6c7f-0b8a-43b9-b35d-6489e6daee91",
"namespaceFormat" : "${SOURCE_NAMESPACE}",
"operationIds" : [ null, null ]
"operationIds" : [ null, null ],
"scheduleData" : {
"cron" : {
"cronExpression" : "cronExpression",
"cronTimeZone" : "cronTimeZone"
},
"basicSchedule" : {
"units" : 6,
"timeUnit" : "minutes"
}
}
}</code></pre>
<h3 class="field-label">Produces</h3>
@@ -8350,7 +8450,17 @@ font-style: italic;
},
"connectionId" : "046b6c7f-0b8a-43b9-b35d-6489e6daee91",
"namespaceFormat" : "${SOURCE_NAMESPACE}",
"operationIds" : [ null, null ]
"operationIds" : [ null, null ],
"scheduleData" : {
"cron" : {
"cronExpression" : "cronExpression",
"cronTimeZone" : "cronTimeZone"
},
"basicSchedule" : {
"units" : 6,
"timeUnit" : "minutes"
}
}
}</code></pre>
<h3 class="field-label">Produces</h3>
@@ -8613,7 +8723,17 @@ font-style: italic;
},
"connectionId" : "046b6c7f-0b8a-43b9-b35d-6489e6daee91",
"namespaceFormat" : "${SOURCE_NAMESPACE}",
"operationIds" : [ null, null ]
"operationIds" : [ null, null ],
"scheduleData" : {
"cron" : {
"cronExpression" : "cronExpression",
"cronTimeZone" : "cronTimeZone"
},
"basicSchedule" : {
"units" : 6,
"timeUnit" : "minutes"
}
}
}, {
"sourceId" : "046b6c7f-0b8a-43b9-b35d-6489e6daee91",
"latestSyncJobCreatedAt" : 0,
@@ -8759,7 +8879,17 @@ font-style: italic;
},
"connectionId" : "046b6c7f-0b8a-43b9-b35d-6489e6daee91",
"namespaceFormat" : "${SOURCE_NAMESPACE}",
"operationIds" : [ null, null ]
"operationIds" : [ null, null ],
"scheduleData" : {
"cron" : {
"cronExpression" : "cronExpression",
"cronTimeZone" : "cronTimeZone"
},
"basicSchedule" : {
"units" : 6,
"timeUnit" : "minutes"
}
}
} ]
}</code></pre>
@@ -8963,7 +9093,17 @@ font-style: italic;
},
"connectionId" : "046b6c7f-0b8a-43b9-b35d-6489e6daee91",
"namespaceFormat" : "${SOURCE_NAMESPACE}",
"operationIds" : [ null, null ]
"operationIds" : [ null, null ],
"scheduleData" : {
"cron" : {
"cronExpression" : "cronExpression",
"cronTimeZone" : "cronTimeZone"
},
"basicSchedule" : {
"units" : 6,
"timeUnit" : "minutes"
}
}
}, {
"sourceId" : "046b6c7f-0b8a-43b9-b35d-6489e6daee91",
"latestSyncJobCreatedAt" : 0,
@@ -9109,7 +9249,17 @@ font-style: italic;
},
"connectionId" : "046b6c7f-0b8a-43b9-b35d-6489e6daee91",
"namespaceFormat" : "${SOURCE_NAMESPACE}",
"operationIds" : [ null, null ]
"operationIds" : [ null, null ],
"scheduleData" : {
"cron" : {
"cronExpression" : "cronExpression",
"cronTimeZone" : "cronTimeZone"
},
"basicSchedule" : {
"units" : 6,
"timeUnit" : "minutes"
}
}
} ]
}</code></pre>
@@ -9313,7 +9463,17 @@ font-style: italic;
},
"connectionId" : "046b6c7f-0b8a-43b9-b35d-6489e6daee91",
"namespaceFormat" : "${SOURCE_NAMESPACE}",
"operationIds" : [ null, null ]
"operationIds" : [ null, null ],
"scheduleData" : {
"cron" : {
"cronExpression" : "cronExpression",
"cronTimeZone" : "cronTimeZone"
},
"basicSchedule" : {
"units" : 6,
"timeUnit" : "minutes"
}
}
}, {
"sourceId" : "046b6c7f-0b8a-43b9-b35d-6489e6daee91",
"latestSyncJobCreatedAt" : 0,
@@ -9459,7 +9619,17 @@ font-style: italic;
},
"connectionId" : "046b6c7f-0b8a-43b9-b35d-6489e6daee91",
"namespaceFormat" : "${SOURCE_NAMESPACE}",
"operationIds" : [ null, null ]
"operationIds" : [ null, null ],
"scheduleData" : {
"cron" : {
"cronExpression" : "cronExpression",
"cronTimeZone" : "cronTimeZone"
},
"basicSchedule" : {
"units" : 6,
"timeUnit" : "minutes"
}
}
} ]
}</code></pre>
@@ -9659,7 +9829,17 @@ font-style: italic;
},
"connectionId" : "046b6c7f-0b8a-43b9-b35d-6489e6daee91",
"namespaceFormat" : "${SOURCE_NAMESPACE}",
"operationIds" : [ null, null ]
"operationIds" : [ null, null ],
"scheduleData" : {
"cron" : {
"cronExpression" : "cronExpression",
"cronTimeZone" : "cronTimeZone"
},
"basicSchedule" : {
"units" : 6,
"timeUnit" : "minutes"
}
}
}</code></pre>
<h3 class="field-label">Produces</h3>
@@ -9858,7 +10038,17 @@ font-style: italic;
},
"connectionId" : "046b6c7f-0b8a-43b9-b35d-6489e6daee91",
"namespaceFormat" : "${SOURCE_NAMESPACE}",
"operationIds" : [ null, null ]
"operationIds" : [ null, null ],
"scheduleData" : {
"cron" : {
"cronExpression" : "cronExpression",
"cronTimeZone" : "cronTimeZone"
},
"basicSchedule" : {
"units" : 6,
"timeUnit" : "minutes"
}
}
}</code></pre>
<h3 class="field-label">Produces</h3>
@@ -10516,6 +10706,10 @@ font-style: italic;
<li><a href="#ConnectionRead"><code>ConnectionRead</code> - </a></li>
<li><a href="#ConnectionReadList"><code>ConnectionReadList</code> - </a></li>
<li><a href="#ConnectionSchedule"><code>ConnectionSchedule</code> - </a></li>
<li><a href="#ConnectionScheduleData"><code>ConnectionScheduleData</code> - </a></li>
<li><a href="#ConnectionScheduleData_basicSchedule"><code>ConnectionScheduleData_basicSchedule</code> - </a></li>
<li><a href="#ConnectionScheduleData_cron"><code>ConnectionScheduleData_cron</code> - </a></li>
<li><a href="#ConnectionScheduleType"><code>ConnectionScheduleType</code> - </a></li>
<li><a href="#ConnectionSearch"><code>ConnectionSearch</code> - </a></li>
<li><a href="#ConnectionState"><code>ConnectionState</code> - </a></li>
<li><a href="#ConnectionStateType"><code>ConnectionStateType</code> - </a></li>
@@ -10861,6 +11055,8 @@ font-style: italic;
<div class="param">operationIds (optional)</div><div class="param-desc"><span class="param-type"><a href="#UUID">array[UUID]</a></span> format: uuid</div>
<div class="param">syncCatalog (optional)</div><div class="param-desc"><span class="param-type"><a href="#AirbyteCatalog">AirbyteCatalog</a></span> </div>
<div class="param">schedule (optional)</div><div class="param-desc"><span class="param-type"><a href="#ConnectionSchedule">ConnectionSchedule</a></span> </div>
<div class="param">scheduleType (optional)</div><div class="param-desc"><span class="param-type"><a href="#ConnectionScheduleType">ConnectionScheduleType</a></span> </div>
<div class="param">scheduleData (optional)</div><div class="param-desc"><span class="param-type"><a href="#ConnectionScheduleData">ConnectionScheduleData</a></span> </div>
<div class="param">status </div><div class="param-desc"><span class="param-type"><a href="#ConnectionStatus">ConnectionStatus</a></span> </div>
<div class="param">resourceRequirements (optional)</div><div class="param-desc"><span class="param-type"><a href="#ResourceRequirements">ResourceRequirements</a></span> </div>
<div class="param">sourceCatalogId (optional)</div><div class="param-desc"><span class="param-type"><a href="#UUID">UUID</a></span> format: uuid</div>
@@ -10887,6 +11083,8 @@ font-style: italic;
<div class="param">operationIds (optional)</div><div class="param-desc"><span class="param-type"><a href="#UUID">array[UUID]</a></span> format: uuid</div>
<div class="param">syncCatalog </div><div class="param-desc"><span class="param-type"><a href="#AirbyteCatalog">AirbyteCatalog</a></span> </div>
<div class="param">schedule (optional)</div><div class="param-desc"><span class="param-type"><a href="#ConnectionSchedule">ConnectionSchedule</a></span> </div>
<div class="param">scheduleType (optional)</div><div class="param-desc"><span class="param-type"><a href="#ConnectionScheduleType">ConnectionScheduleType</a></span> </div>
<div class="param">scheduleData (optional)</div><div class="param-desc"><span class="param-type"><a href="#ConnectionScheduleData">ConnectionScheduleData</a></span> </div>
<div class="param">status </div><div class="param-desc"><span class="param-type"><a href="#ConnectionStatus">ConnectionStatus</a></span> </div>
<div class="param">resourceRequirements (optional)</div><div class="param-desc"><span class="param-type"><a href="#ResourceRequirements">ResourceRequirements</a></span> </div>
<div class="param">sourceCatalogId (optional)</div><div class="param-desc"><span class="param-type"><a href="#UUID">UUID</a></span> format: uuid</div>
@@ -10909,6 +11107,38 @@ font-style: italic;
<div class="param-enum">minutes</div><div class="param-enum">hours</div><div class="param-enum">days</div><div class="param-enum">weeks</div><div class="param-enum">months</div>
</div> <!-- field-items -->
</div>
<div class="model">
<h3><a name="ConnectionScheduleData"><code>ConnectionScheduleData</code> - </a> <a class="up" href="#__Models">Up</a></h3>
<div class='model-description'>schedule for when the the connection should run, per the schedule type</div>
<div class="field-items">
<div class="param">basicSchedule (optional)</div><div class="param-desc"><span class="param-type"><a href="#ConnectionScheduleData_basicSchedule">ConnectionScheduleData_basicSchedule</a></span> </div>
<div class="param">cron (optional)</div><div class="param-desc"><span class="param-type"><a href="#ConnectionScheduleData_cron">ConnectionScheduleData_cron</a></span> </div>
</div> <!-- field-items -->
</div>
<div class="model">
<h3><a name="ConnectionScheduleData_basicSchedule"><code>ConnectionScheduleData_basicSchedule</code> - </a> <a class="up" href="#__Models">Up</a></h3>
<div class='model-description'></div>
<div class="field-items">
<div class="param">timeUnit </div><div class="param-desc"><span class="param-type"><a href="#string">String</a></span> </div>
<div class="param-enum-header">Enum:</div>
<div class="param-enum">minutes</div><div class="param-enum">hours</div><div class="param-enum">days</div><div class="param-enum">weeks</div><div class="param-enum">months</div>
<div class="param">units </div><div class="param-desc"><span class="param-type"><a href="#long">Long</a></span> format: int64</div>
</div> <!-- field-items -->
</div>
<div class="model">
<h3><a name="ConnectionScheduleData_cron"><code>ConnectionScheduleData_cron</code> - </a> <a class="up" href="#__Models">Up</a></h3>
<div class='model-description'></div>
<div class="field-items">
<div class="param">cronExpression </div><div class="param-desc"><span class="param-type"><a href="#string">String</a></span> </div>
<div class="param">cronTimeZone </div><div class="param-desc"><span class="param-type"><a href="#string">String</a></span> </div>
</div> <!-- field-items -->
</div>
<div class="model">
<h3><a name="ConnectionScheduleType"><code>ConnectionScheduleType</code> - </a> <a class="up" href="#__Models">Up</a></h3>
<div class='model-description'>determine how the schedule data should be interpreted</div>
<div class="field-items">
</div> <!-- field-items -->
</div>
<div class="model">
<h3><a name="ConnectionSearch"><code>ConnectionSearch</code> - </a> <a class="up" href="#__Models">Up</a></h3>
<div class='model-description'></div>
@@ -10921,6 +11151,8 @@ font-style: italic;
<div class="param">sourceId (optional)</div><div class="param-desc"><span class="param-type"><a href="#UUID">UUID</a></span> format: uuid</div>
<div class="param">destinationId (optional)</div><div class="param-desc"><span class="param-type"><a href="#UUID">UUID</a></span> format: uuid</div>
<div class="param">schedule (optional)</div><div class="param-desc"><span class="param-type"><a href="#ConnectionSchedule">ConnectionSchedule</a></span> </div>
<div class="param">scheduleType (optional)</div><div class="param-desc"><span class="param-type"><a href="#ConnectionScheduleType">ConnectionScheduleType</a></span> </div>
<div class="param">scheduleData (optional)</div><div class="param-desc"><span class="param-type"><a href="#ConnectionScheduleData">ConnectionScheduleData</a></span> </div>
<div class="param">status (optional)</div><div class="param-desc"><span class="param-type"><a href="#ConnectionStatus">ConnectionStatus</a></span> </div>
<div class="param">source (optional)</div><div class="param-desc"><span class="param-type"><a href="#SourceSearch">SourceSearch</a></span> </div>
<div class="param">destination (optional)</div><div class="param-desc"><span class="param-type"><a href="#DestinationSearch">DestinationSearch</a></span> </div>
@@ -10961,6 +11193,8 @@ font-style: italic;
<div class="param">operationIds (optional)</div><div class="param-desc"><span class="param-type"><a href="#UUID">array[UUID]</a></span> format: uuid</div>
<div class="param">syncCatalog </div><div class="param-desc"><span class="param-type"><a href="#AirbyteCatalog">AirbyteCatalog</a></span> </div>
<div class="param">schedule (optional)</div><div class="param-desc"><span class="param-type"><a href="#ConnectionSchedule">ConnectionSchedule</a></span> </div>
<div class="param">scheduleType (optional)</div><div class="param-desc"><span class="param-type"><a href="#ConnectionScheduleType">ConnectionScheduleType</a></span> </div>
<div class="param">scheduleData (optional)</div><div class="param-desc"><span class="param-type"><a href="#ConnectionScheduleData">ConnectionScheduleData</a></span> </div>
<div class="param">status </div><div class="param-desc"><span class="param-type"><a href="#ConnectionStatus">ConnectionStatus</a></span> </div>
<div class="param">resourceRequirements (optional)</div><div class="param-desc"><span class="param-type"><a href="#ResourceRequirements">ResourceRequirements</a></span> </div>
<div class="param">sourceCatalogId (optional)</div><div class="param-desc"><span class="param-type"><a href="#UUID">UUID</a></span> format: uuid</div>
@@ -11923,6 +12157,8 @@ if oauth parameters were contained inside the top level, rootObject=[] If they w
<div class="param">operationIds (optional)</div><div class="param-desc"><span class="param-type"><a href="#UUID">array[UUID]</a></span> format: uuid</div>
<div class="param">syncCatalog (optional)</div><div class="param-desc"><span class="param-type"><a href="#AirbyteCatalog">AirbyteCatalog</a></span> </div>
<div class="param">schedule (optional)</div><div class="param-desc"><span class="param-type"><a href="#ConnectionSchedule">ConnectionSchedule</a></span> </div>
<div class="param">scheduleType (optional)</div><div class="param-desc"><span class="param-type"><a href="#ConnectionScheduleType">ConnectionScheduleType</a></span> </div>
<div class="param">scheduleData (optional)</div><div class="param-desc"><span class="param-type"><a href="#ConnectionScheduleData">ConnectionScheduleData</a></span> </div>
<div class="param">status </div><div class="param-desc"><span class="param-type"><a href="#ConnectionStatus">ConnectionStatus</a></span> </div>
<div class="param">resourceRequirements (optional)</div><div class="param-desc"><span class="param-type"><a href="#ResourceRequirements">ResourceRequirements</a></span> </div>
<div class="param">operations (optional)</div><div class="param-desc"><span class="param-type"><a href="#OperationCreate">array[OperationCreate]</a></span> </div>
@@ -11942,6 +12178,8 @@ if oauth parameters were contained inside the top level, rootObject=[] If they w
<div class="param">destinationId </div><div class="param-desc"><span class="param-type"><a href="#UUID">UUID</a></span> format: uuid</div>
<div class="param">syncCatalog </div><div class="param-desc"><span class="param-type"><a href="#AirbyteCatalog">AirbyteCatalog</a></span> </div>
<div class="param">schedule (optional)</div><div class="param-desc"><span class="param-type"><a href="#ConnectionSchedule">ConnectionSchedule</a></span> </div>
<div class="param">scheduleType (optional)</div><div class="param-desc"><span class="param-type"><a href="#ConnectionScheduleType">ConnectionScheduleType</a></span> </div>
<div class="param">scheduleData (optional)</div><div class="param-desc"><span class="param-type"><a href="#ConnectionScheduleData">ConnectionScheduleData</a></span> </div>
<div class="param">status </div><div class="param-desc"><span class="param-type"><a href="#ConnectionStatus">ConnectionStatus</a></span> </div>
<div class="param">operationIds (optional)</div><div class="param-desc"><span class="param-type"><a href="#UUID">array[UUID]</a></span> format: uuid</div>
<div class="param">source </div><div class="param-desc"><span class="param-type"><a href="#SourceRead">SourceRead</a></span> </div>
@@ -11982,6 +12220,8 @@ if oauth parameters were contained inside the top level, rootObject=[] If they w
<div class="param">sourceId (optional)</div><div class="param-desc"><span class="param-type"><a href="#UUID">UUID</a></span> format: uuid</div>
<div class="param">destinationId (optional)</div><div class="param-desc"><span class="param-type"><a href="#UUID">UUID</a></span> format: uuid</div>
<div class="param">schedule (optional)</div><div class="param-desc"><span class="param-type"><a href="#ConnectionSchedule">ConnectionSchedule</a></span> </div>
<div class="param">scheduleType (optional)</div><div class="param-desc"><span class="param-type"><a href="#ConnectionScheduleType">ConnectionScheduleType</a></span> </div>
<div class="param">scheduleData (optional)</div><div class="param-desc"><span class="param-type"><a href="#ConnectionScheduleData">ConnectionScheduleData</a></span> </div>
<div class="param">status (optional)</div><div class="param-desc"><span class="param-type"><a href="#ConnectionStatus">ConnectionStatus</a></span> </div>
<div class="param">source (optional)</div><div class="param-desc"><span class="param-type"><a href="#SourceSearch">SourceSearch</a></span> </div>
<div class="param">destination (optional)</div><div class="param-desc"><span class="param-type"><a href="#DestinationSearch">DestinationSearch</a></span> </div>
@@ -11999,6 +12239,8 @@ if oauth parameters were contained inside the top level, rootObject=[] If they w
<div class="param">operationIds (optional)</div><div class="param-desc"><span class="param-type"><a href="#UUID">array[UUID]</a></span> format: uuid</div>
<div class="param">syncCatalog </div><div class="param-desc"><span class="param-type"><a href="#AirbyteCatalog">AirbyteCatalog</a></span> </div>
<div class="param">schedule (optional)</div><div class="param-desc"><span class="param-type"><a href="#ConnectionSchedule">ConnectionSchedule</a></span> </div>
<div class="param">scheduleType (optional)</div><div class="param-desc"><span class="param-type"><a href="#ConnectionScheduleType">ConnectionScheduleType</a></span> </div>
<div class="param">scheduleData (optional)</div><div class="param-desc"><span class="param-type"><a href="#ConnectionScheduleData">ConnectionScheduleData</a></span> </div>
<div class="param">status </div><div class="param-desc"><span class="param-type"><a href="#ConnectionStatus">ConnectionStatus</a></span> </div>
<div class="param">resourceRequirements (optional)</div><div class="param-desc"><span class="param-type"><a href="#ResourceRequirements">ResourceRequirements</a></span> </div>
<div class="param">withRefreshedCatalog (optional)</div><div class="param-desc"><span class="param-type"><a href="#boolean">Boolean</a></span> </div>

1
octavia-cli/README.md
View File

@@ -705,6 +705,7 @@ You can disable telemetry by setting the `OCTAVIA_ENABLE_TELEMETRY` environment
| Version | Date | Description | PR |
| ------- | ---------- | --------------------------------------------------------------------------------------| ----------------------------------------------------------- |
| 0.40.0 | 2022-08-10 | Enable cron and basic scheduling | [#15253](https://github.com/airbytehq/airbyte/pull/15253) |
| 0.39.33 | 2022-07-05 | Add `octavia import all` command | [#14374](https://github.com/airbytehq/airbyte/pull/14374) |
| 0.39.32 | 2022-06-30 | Create import command to import and manage existing Airbyte resource from octavia-cli | [#14137](https://github.com/airbytehq/airbyte/pull/14137) |
| 0.39.27 | 2022-06-24 | Create get command to retrieve resources JSON representation | [#13254](https://github.com/airbytehq/airbyte/pull/13254) |

18164
octavia-cli/integration_tests/cassettes/test_api_http_headers/test_api_http_headers.yaml
View File

File diff suppressed because it is too large Load Diff

8
octavia-cli/integration_tests/configurations/connections/poke_to_pg/configuration.yaml
View File

@@ -15,9 +15,11 @@ configuration:
cpu_request: "" # OPTIONAL
memory_limit: "" # OPTIONAL
memory_request: "" # OPTIONAL
schedule: # OPTIONAL | object
time_unit: hours # REQUIRED | string | Allowed values: minutes, hours, days, weeks, months
units: 1 # REQUIRED | integer
schedule_type: basic
schedule_data:
basic_schedule:
time_unit: hours # REQUIRED | string | Allowed values: minutes, hours, days, weeks, months
units: 1 # REQUIRED | integer
sync_catalog: # OPTIONAL | object | 🚨 ONLY edit streams.config, streams.stream should not be edited as schema cannot be changed.
streams:
- config:

8
octavia-cli/integration_tests/configurations/connections/poke_to_pg_normalization/configuration.yaml
View File

@@ -15,9 +15,11 @@ configuration:
cpu_request: "" # OPTIONAL
memory_limit: "" # OPTIONAL
memory_request: "" # OPTIONAL
schedule: # OPTIONAL | object
time_unit: hours # REQUIRED | string | Allowed values: minutes, hours, days, weeks, months
units: 1 # REQUIRED | integer
schedule_type: basic
schedule_data:
basic_schedule:
time_unit: hours # REQUIRED | string | Allowed values: minutes, hours, days, weeks, months
units: 1 # REQUIRED | integer
operations:
- name: "Normalization"
operator_configuration:

4
octavia-cli/integration_tests/test_apply/test_resources.py
View File

@@ -42,12 +42,10 @@ def test_connection_lifecycle(source, destination, connection, workspace_id):
connection.create()
connection.state = connection._get_state_from_file(connection.configuration_path, workspace_id)
assert connection.was_created
assert not connection.get_diff_with_remote_resource()
connection.raw_configuration["configuration"]["status"] = "inactive"
connection.configuration = connection._deserialize_raw_configuration()
assert 'changed from "active" to "inactive"' in connection.get_diff_with_remote_resource()
connection.update()
assert not connection.get_diff_with_remote_resource()
def test_connection_lifecycle_with_normalization(source, destination, connection_with_normalization, workspace_id):
@@ -61,9 +59,7 @@ def test_connection_lifecycle_with_normalization(source, destination, connection
assert connection_with_normalization.was_created
assert connection_with_normalization.remote_resource["operations"][0]["operation_id"] is not None
assert connection_with_normalization.remote_resource["operations"][0]["operator_configuration"]["normalization"]["option"] == "basic"
assert not connection_with_normalization.get_diff_with_remote_resource()
connection_with_normalization.raw_configuration["configuration"]["status"] = "inactive"
connection_with_normalization.configuration = connection_with_normalization._deserialize_raw_configuration()
assert 'changed from "active" to "inactive"' in connection_with_normalization.get_diff_with_remote_resource()
connection_with_normalization.update()
assert not connection_with_normalization.get_diff_with_remote_resource()

11
octavia-cli/integration_tests/test_generate/expected_rendered_yaml/connection/expected_with_normalization.yaml
View File

@@ -15,9 +15,14 @@ configuration:
cpu_request: "" # OPTIONAL
memory_limit: "" # OPTIONAL
memory_request: "" # OPTIONAL
schedule: # OPTIONAL | object
time_unit: hours # REQUIRED | string | Allowed values: minutes, hours, days, weeks, months
units: 1 # REQUIRED | integer
schedule_type: basic # OPTIONAL | string | Allowed values: basic, cron, manual
schedule_data: # OPTIONAL | object
basic_schedule:
time_unit: hours # REQUIRED | string | Allowed values: minutes, hours, days, weeks, months
units: 1 # REQUIRED | integer
# cron:
# cron_time_zone: "UTC" # REQUIRED | string
# cron_expression: "* */2 * * * ?" # REQUIRED | string
# operations:
## -------- Uncomment and edit the block below if you want to enable Airbyte normalization --------
# - name: "Normalization"

11
octavia-cli/integration_tests/test_generate/expected_rendered_yaml/connection/expected_without_normalization.yaml
View File

@@ -15,9 +15,14 @@ configuration:
cpu_request: "" # OPTIONAL
memory_limit: "" # OPTIONAL
memory_request: "" # OPTIONAL
schedule: # OPTIONAL | object
time_unit: hours # REQUIRED | string | Allowed values: minutes, hours, days, weeks, months
units: 1 # REQUIRED | integer
schedule_type: basic # OPTIONAL | string | Allowed values: basic, cron, manual
schedule_data: # OPTIONAL | object
basic_schedule:
time_unit: hours # REQUIRED | string | Allowed values: minutes, hours, days, weeks, months
units: 1 # REQUIRED | integer
# cron:
# cron_time_zone: "UTC" # REQUIRED | string
# cron_expression: "* */2 * * * ?" # REQUIRED | string
sync_catalog: # OPTIONAL | object | 🚨 ONLY edit streams.config, streams.stream should not be edited as schema cannot be changed.
streams:
- config:

8
octavia-cli/integration_tests/test_import/octavia_project_to_migrate/connections/poke_to_pg_to_import/configuration.yaml
View File

@@ -15,9 +15,11 @@ configuration:
cpu_request: "" # OPTIONAL
memory_limit: "" # OPTIONAL
memory_request: "" # OPTIONAL
schedule: # OPTIONAL | object
time_unit: hours # REQUIRED | string | Allowed values: minutes, hours, days, weeks, months
units: 1 # REQUIRED | integer
schedule_type: basic
schedule_data:
basic_schedule:
time_unit: hours # REQUIRED | string | Allowed values: minutes, hours, days, weeks, months
units: 1 # REQUIRED | integer
operations:
- name: "Normalization"
operator_configuration:

2
octavia-cli/octavia_cli/_import/commands.py
View File

@@ -85,6 +85,8 @@ def import_connection(
str: The generated import message.
"""
remote_configuration = json.loads(get_json_representation(api_client, workspace_id, UnmanagedConnection, resource_to_get))
# Since #15253 "schedule" is deprecated
remote_configuration.pop("schedule", None)
source_name, destination_name = remote_configuration["source"]["name"], remote_configuration["destination"]["name"]
source_configuration_path = renderers.ConnectorSpecificationRenderer.get_output_path(
project_path=".", definition_type="source", resource_name=source_name

32
octavia-cli/octavia_cli/apply/resources.py
View File

@@ -24,7 +24,10 @@ from airbyte_api_client.model.airbyte_stream import AirbyteStream
from airbyte_api_client.model.airbyte_stream_and_configuration import AirbyteStreamAndConfiguration
from airbyte_api_client.model.airbyte_stream_configuration import AirbyteStreamConfiguration
from airbyte_api_client.model.connection_read import ConnectionRead
from airbyte_api_client.model.connection_schedule import ConnectionSchedule
from airbyte_api_client.model.connection_schedule_data import ConnectionScheduleData
from airbyte_api_client.model.connection_schedule_data_basic_schedule import ConnectionScheduleDataBasicSchedule
from airbyte_api_client.model.connection_schedule_data_cron import ConnectionScheduleDataCron
from airbyte_api_client.model.connection_schedule_type import ConnectionScheduleType
from airbyte_api_client.model.connection_status import ConnectionStatus
from airbyte_api_client.model.destination_create import DestinationCreate
from airbyte_api_client.model.destination_definition_id_with_workspace_id import DestinationDefinitionIdWithWorkspaceId
@@ -577,6 +580,7 @@ class Connection(BaseResource):
"is_syncing",
"latest_sync_job_status",
"latest_sync_job_created_at",
"schedule",
] # We do not allow local editing of these keys
# We do not allow local editing of these keys
@@ -595,8 +599,20 @@ class Connection(BaseResource):
self._check_for_legacy_connection_configuration_keys(configuration)
configuration["sync_catalog"] = self._create_configured_catalog(configuration["sync_catalog"])
configuration["namespace_definition"] = NamespaceDefinitionType(configuration["namespace_definition"])
if "schedule" in configuration:
configuration["schedule"] = ConnectionSchedule(**configuration["schedule"])
if "schedule_type" in configuration:
# If schedule type is manual we do not expect a schedule_data field to be set
# TODO: sending a WebConnectionCreate payload without schedule_data (for manual) fails.
is_manual = configuration["schedule_type"] == "manual"
configuration["schedule_type"] = ConnectionScheduleType(configuration["schedule_type"])
if not is_manual:
if "basic_schedule" in configuration["schedule_data"]:
basic_schedule = ConnectionScheduleDataBasicSchedule(**configuration["schedule_data"]["basic_schedule"])
configuration["schedule_data"]["basic_schedule"] = basic_schedule
if "cron" in configuration["schedule_data"]:
cron = ConnectionScheduleDataCron(**configuration["schedule_data"]["cron"])
configuration["schedule_data"]["cron"] = cron
configuration["schedule_data"] = ConnectionScheduleData(**configuration["schedule_data"])
if "resource_requirements" in configuration:
configuration["resource_requirements"] = ResourceRequirements(**configuration["resource_requirements"])
configuration["status"] = ConnectionStatus(configuration["status"])
@@ -738,8 +754,16 @@ class Connection(BaseResource):
deserialized_operations.append(operation)
return deserialized_operations
# TODO this check can be removed when all our active user are on >= 0.37.0
def _check_for_legacy_connection_configuration_keys(self, configuration_to_check):
self._check_for_wrong_casing_in_connection_configurations_keys(configuration_to_check)
self._check_for_schedule_in_connection_configurations_keys(configuration_to_check)
# TODO this check can be removed when all our active user are on >= 0.37.0
def _check_for_schedule_in_connection_configurations_keys(self, configuration_to_check):
error_message = "The schedule key is deprecated since 0.40.0, please use a combination of schedule_type and schedule_data"
self._check_for_invalid_configuration_keys(configuration_to_check, {"schedule"}, error_message)
def _check_for_wrong_casing_in_connection_configurations_keys(self, configuration_to_check):
"""We changed connection configuration keys from camelCase to snake_case in 0.37.0.
This function check if the connection configuration has some camelCase keys and display a meaningful error message.
Args:

11
octavia-cli/octavia_cli/generate/templates/connection.yaml.j2
View File

@@ -15,9 +15,14 @@ configuration:
cpu_request: "" # OPTIONAL
memory_limit: "" # OPTIONAL
memory_request: "" # OPTIONAL
schedule: # OPTIONAL | object
time_unit: hours # REQUIRED | string | Allowed values: minutes, hours, days, weeks, months
units: 1 # REQUIRED | integer
schedule_type: basic # OPTIONAL | string | Allowed values: basic, cron, manual
schedule_data: # OPTIONAL | object
basic_schedule:
time_unit: hours # REQUIRED | string | Allowed values: minutes, hours, days, weeks, months
units: 1 # REQUIRED | integer
# cron:
# cron_time_zone: "UTC" # REQUIRED | string
# cron_expression: "* */2 * * * ?" # REQUIRED | string
{%- if supports_normalization or supports_dbt%}
# operations:
{%- endif %}

53
octavia-cli/unit_tests/test_apply/test_resources.py
View File

@@ -8,7 +8,8 @@ from unittest.mock import mock_open, patch
import pytest
from airbyte_api_client import ApiException
from airbyte_api_client.model.airbyte_catalog import AirbyteCatalog
from airbyte_api_client.model.connection_schedule import ConnectionSchedule
from airbyte_api_client.model.connection_schedule_data_basic_schedule import ConnectionScheduleDataBasicSchedule
from airbyte_api_client.model.connection_schedule_type import ConnectionScheduleType
from airbyte_api_client.model.connection_status import ConnectionStatus
from airbyte_api_client.model.destination_definition_id_with_workspace_id import DestinationDefinitionIdWithWorkspaceId
from airbyte_api_client.model.namespace_definition_type import NamespaceDefinitionType
@@ -468,12 +469,20 @@ class TestConnection:
}
]
},
"schedule": {"units": 1, "time_unit": "days"},
"schedule_type": "basic",
"schedule_data": {"units": 1, "time_unit": "days"},
"status": "active",
"resource_requirements": {"cpu_request": "foo", "cpu_limit": "foo", "memory_request": "foo", "memory_limit": "foo"},
},
}
@pytest.fixture
def connection_configuration_with_manual_schedule(self, connection_configuration):
connection_configuration_with_manual_schedule = deepcopy(connection_configuration)
connection_configuration_with_manual_schedule["configuration"]["schedule_type"] = "manual"
connection_configuration_with_manual_schedule["configuration"]["schedule_data"] = None
return connection_configuration_with_manual_schedule
@pytest.fixture
def connection_configuration_with_normalization(self, connection_configuration):
connection_configuration_with_normalization = deepcopy(connection_configuration)
@@ -559,6 +568,28 @@ class TestConnection:
"resource_requirements": {"cpu_request": "foo", "cpu_limit": "foo", "memory_request": "foo", "memory_limit": "foo"},
},
},
{
"definition_type": "connection",
"resource_name": "my_connection",
"source_id": "my_source",
"destination_id": "my_destination",
"configuration": {
"namespace_definition": "customformat",
"namespace_format": "foo",
"prefix": "foo",
"sync_catalog": {
"streams": [
{
"stream": {},
"config": {},
}
]
},
"schedule": {"units": 1, "time_unit": "days"},
"status": "active",
"resource_requirements": {"cpu_request": "foo", "cpu_limit": "foo", "memory_request": "foo", "memory_limit": "foo"},
},
},
]
@pytest.mark.parametrize(
@@ -769,14 +800,18 @@ class TestConnection:
assert update_result == resource._create_or_update.return_value
resource._create_or_update.assert_called_with(resource._update_fn, resource.update_payload)
def test__deserialize_raw_configuration(self, mock_api_client, connection_configuration):
def test__deserialize_raw_configuration(self, mock_api_client, connection_configuration, connection_configuration_with_manual_schedule):
resource = resources.Connection(mock_api_client, "workspace_id", connection_configuration, "bar.yaml")
configuration = resource._deserialize_raw_configuration()
assert isinstance(configuration["sync_catalog"], AirbyteCatalog)
assert configuration["namespace_definition"] == NamespaceDefinitionType(
connection_configuration["configuration"]["namespace_definition"]
)
assert configuration["schedule"] == ConnectionSchedule(**connection_configuration["configuration"]["schedule"])
assert configuration["schedule_type"] == ConnectionScheduleType(connection_configuration["configuration"]["schedule_type"])
assert (
configuration["schedule_data"].to_dict()
== ConnectionScheduleDataBasicSchedule(**connection_configuration["configuration"]["schedule_data"]).to_dict()
)
assert configuration["resource_requirements"] == ResourceRequirements(
**connection_configuration["configuration"]["resource_requirements"]
)
@@ -786,11 +821,19 @@ class TestConnection:
"namespace_format",
"prefix",
"sync_catalog",
"schedule",
"schedule_type",
"schedule_data",
"status",
"resource_requirements",
]
resource = resources.Connection(mock_api_client, "workspace_id", connection_configuration_with_manual_schedule, "bar.yaml")
configuration = resource._deserialize_raw_configuration()
assert configuration["schedule_type"] == ConnectionScheduleType(
connection_configuration_with_manual_schedule["configuration"]["schedule_type"]
)
assert configuration["schedule_data"] is None
def test__deserialize_operations(self, mock_api_client, connection_configuration):
resource = resources.Connection(mock_api_client, "workspace_id", connection_configuration, "bar.yaml")
operations = [