fc5ba66e1c
* use SHORT_ACTIVITY_OPTIONS on check connection activity so that it has retries * retry workflow after delay instead of quarantining * allow activity env vars to be configured in docker and kube * add env var for workflow restart delay and refactor slightly * update tests to handle new restart behavior * update test name * add empty env var values to .env files * fail attempt before job in cleanJobState to prevent state machine failure * change default value of max activity attempt retries from 10 to 5
293 lines
10 KiB
YAML
293 lines
10 KiB
YAML
apiVersion: apps/v1
|
|
kind: Deployment
|
|
metadata:
|
|
name: airbyte-worker
|
|
spec:
|
|
replicas: 1
|
|
selector:
|
|
matchLabels:
|
|
airbyte: worker
|
|
template:
|
|
metadata:
|
|
labels:
|
|
airbyte: worker
|
|
spec:
|
|
serviceAccountName: airbyte-admin
|
|
automountServiceAccountToken: true
|
|
containers:
|
|
- name: airbyte-worker-container
|
|
image: airbyte/worker
|
|
env:
|
|
- name: AIRBYTE_VERSION
|
|
valueFrom:
|
|
configMapKeyRef:
|
|
name: airbyte-env
|
|
key: AIRBYTE_VERSION
|
|
- name: CONFIG_ROOT
|
|
valueFrom:
|
|
configMapKeyRef:
|
|
name: airbyte-env
|
|
key: CONFIG_ROOT
|
|
- name: DATABASE_HOST
|
|
valueFrom:
|
|
configMapKeyRef:
|
|
name: airbyte-env
|
|
key: DATABASE_HOST
|
|
- name: DATABASE_PORT
|
|
valueFrom:
|
|
configMapKeyRef:
|
|
name: airbyte-env
|
|
key: DATABASE_PORT
|
|
- name: DATABASE_PASSWORD
|
|
valueFrom:
|
|
secretKeyRef:
|
|
name: airbyte-secrets
|
|
key: DATABASE_PASSWORD
|
|
- name: DATABASE_URL
|
|
valueFrom:
|
|
configMapKeyRef:
|
|
name: airbyte-env
|
|
key: DATABASE_URL
|
|
- name: DATABASE_USER
|
|
valueFrom:
|
|
secretKeyRef:
|
|
name: airbyte-secrets
|
|
key: DATABASE_USER
|
|
- name: TRACKING_STRATEGY
|
|
valueFrom:
|
|
configMapKeyRef:
|
|
name: airbyte-env
|
|
key: TRACKING_STRATEGY
|
|
- name: WORKSPACE_DOCKER_MOUNT
|
|
value: workspace
|
|
- name: WORKSPACE_ROOT
|
|
valueFrom:
|
|
configMapKeyRef:
|
|
name: airbyte-env
|
|
key: WORKSPACE_ROOT
|
|
- name: WORKER_ENVIRONMENT
|
|
valueFrom:
|
|
configMapKeyRef:
|
|
name: airbyte-env
|
|
key: WORKER_ENVIRONMENT
|
|
- name: LOCAL_ROOT
|
|
valueFrom:
|
|
configMapKeyRef:
|
|
name: airbyte-env
|
|
key: LOCAL_ROOT
|
|
- name: WEBAPP_URL
|
|
valueFrom:
|
|
configMapKeyRef:
|
|
name: airbyte-env
|
|
key: WEBAPP_URL
|
|
- name: TEMPORAL_HOST
|
|
valueFrom:
|
|
configMapKeyRef:
|
|
name: airbyte-env
|
|
key: TEMPORAL_HOST
|
|
- name: TEMPORAL_WORKER_PORTS
|
|
valueFrom:
|
|
configMapKeyRef:
|
|
name: airbyte-env
|
|
key: TEMPORAL_WORKER_PORTS
|
|
- name: LOG_LEVEL
|
|
valueFrom:
|
|
configMapKeyRef:
|
|
name: airbyte-env
|
|
key: LOG_LEVEL
|
|
- name: JOB_KUBE_NAMESPACE
|
|
valueFrom:
|
|
fieldRef:
|
|
fieldPath: metadata.namespace
|
|
- name: JOB_MAIN_CONTAINER_CPU_REQUEST
|
|
valueFrom:
|
|
configMapKeyRef:
|
|
name: airbyte-env
|
|
key: JOB_MAIN_CONTAINER_CPU_REQUEST
|
|
- name: JOB_MAIN_CONTAINER_CPU_LIMIT
|
|
valueFrom:
|
|
configMapKeyRef:
|
|
name: airbyte-env
|
|
key: JOB_MAIN_CONTAINER_CPU_LIMIT
|
|
- name: JOB_MAIN_CONTAINER_MEMORY_REQUEST
|
|
valueFrom:
|
|
configMapKeyRef:
|
|
name: airbyte-env
|
|
key: JOB_MAIN_CONTAINER_MEMORY_REQUEST
|
|
- name: JOB_MAIN_CONTAINER_MEMORY_LIMIT
|
|
valueFrom:
|
|
configMapKeyRef:
|
|
name: airbyte-env
|
|
key: JOB_MAIN_CONTAINER_MEMORY_LIMIT
|
|
- name: S3_LOG_BUCKET
|
|
valueFrom:
|
|
configMapKeyRef:
|
|
name: airbyte-env
|
|
key: S3_LOG_BUCKET
|
|
- name: S3_LOG_BUCKET_REGION
|
|
valueFrom:
|
|
configMapKeyRef:
|
|
name: airbyte-env
|
|
key: S3_LOG_BUCKET_REGION
|
|
- name: AWS_ACCESS_KEY_ID
|
|
valueFrom:
|
|
secretKeyRef:
|
|
name: airbyte-secrets
|
|
key: AWS_ACCESS_KEY_ID
|
|
- name: AWS_SECRET_ACCESS_KEY
|
|
valueFrom:
|
|
secretKeyRef:
|
|
name: airbyte-secrets
|
|
key: AWS_SECRET_ACCESS_KEY
|
|
- name: S3_MINIO_ENDPOINT
|
|
valueFrom:
|
|
configMapKeyRef:
|
|
name: airbyte-env
|
|
key: S3_MINIO_ENDPOINT
|
|
- name: S3_PATH_STYLE_ACCESS
|
|
valueFrom:
|
|
configMapKeyRef:
|
|
name: airbyte-env
|
|
key: S3_PATH_STYLE_ACCESS
|
|
- name: GOOGLE_APPLICATION_CREDENTIALS
|
|
valueFrom:
|
|
secretKeyRef:
|
|
name: airbyte-secrets
|
|
key: GOOGLE_APPLICATION_CREDENTIALS
|
|
- name: GCS_LOG_BUCKET
|
|
valueFrom:
|
|
configMapKeyRef:
|
|
name: airbyte-env
|
|
key: GCS_LOG_BUCKET
|
|
- name: INTERNAL_API_HOST
|
|
valueFrom:
|
|
configMapKeyRef:
|
|
name: airbyte-env
|
|
key: INTERNAL_API_HOST
|
|
- name: JOB_KUBE_TOLERATIONS
|
|
valueFrom:
|
|
configMapKeyRef:
|
|
name: airbyte-env
|
|
key: JOB_KUBE_TOLERATIONS
|
|
- name: JOB_KUBE_ANNOTATIONS
|
|
valueFrom:
|
|
configMapKeyRef:
|
|
name: airbyte-env
|
|
key: JOB_KUBE_ANNOTATIONS
|
|
- name: JOB_KUBE_NODE_SELECTORS
|
|
valueFrom:
|
|
configMapKeyRef:
|
|
name: airbyte-env
|
|
key: JOB_KUBE_NODE_SELECTORS
|
|
- name: JOB_KUBE_MAIN_CONTAINER_IMAGE_PULL_POLICY
|
|
valueFrom:
|
|
configMapKeyRef:
|
|
name: airbyte-env
|
|
key: JOB_KUBE_MAIN_CONTAINER_IMAGE_PULL_POLICY
|
|
# todo: add other state storage keys
|
|
- name: STATE_STORAGE_MINIO_BUCKET_NAME
|
|
valueFrom:
|
|
configMapKeyRef:
|
|
name: airbyte-env
|
|
key: STATE_STORAGE_MINIO_BUCKET_NAME
|
|
- name: STATE_STORAGE_MINIO_ENDPOINT
|
|
valueFrom:
|
|
configMapKeyRef:
|
|
name: airbyte-env
|
|
key: STATE_STORAGE_MINIO_ENDPOINT
|
|
- name: STATE_STORAGE_MINIO_ACCESS_KEY
|
|
valueFrom:
|
|
secretKeyRef:
|
|
name: airbyte-secrets
|
|
key: STATE_STORAGE_MINIO_ACCESS_KEY
|
|
- name: STATE_STORAGE_MINIO_SECRET_ACCESS_KEY
|
|
valueFrom:
|
|
secretKeyRef:
|
|
name: airbyte-secrets
|
|
key: STATE_STORAGE_MINIO_SECRET_ACCESS_KEY
|
|
- name: CONTAINER_ORCHESTRATOR_ENABLED
|
|
valueFrom:
|
|
configMapKeyRef:
|
|
name: airbyte-env
|
|
key: CONTAINER_ORCHESTRATOR_ENABLED
|
|
- name: CONFIGS_DATABASE_MINIMUM_FLYWAY_MIGRATION_VERSION
|
|
valueFrom:
|
|
configMapKeyRef:
|
|
name: airbyte-env
|
|
key: CONFIGS_DATABASE_MINIMUM_FLYWAY_MIGRATION_VERSION
|
|
- name: JOBS_DATABASE_MINIMUM_FLYWAY_MIGRATION_VERSION
|
|
valueFrom:
|
|
configMapKeyRef:
|
|
name: airbyte-env
|
|
key: JOBS_DATABASE_MINIMUM_FLYWAY_MIGRATION_VERSION
|
|
- name: METRIC_CLIENT
|
|
valueFrom:
|
|
configMapKeyRef:
|
|
name: airbyte-env
|
|
key: METRIC_CLIENT
|
|
- name: OTEL_COLLECTOR_ENDPOINT
|
|
valueFrom:
|
|
configMapKeyRef:
|
|
name: airbyte-env
|
|
key: OTEL_COLLECTOR_ENDPOINT
|
|
- name: ACTIVITY_MAX_ATTEMPT
|
|
valueFrom:
|
|
configMapKeyRef:
|
|
name: airbyte-env
|
|
key: ACTIVITY_MAX_ATTEMPT
|
|
- name: ACTIVITY_INITIAL_DELAY_BETWEEN_ATTEMPTS_SECONDS
|
|
valueFrom:
|
|
configMapKeyRef:
|
|
name: airbyte-env
|
|
key: ACTIVITY_INITIAL_DELAY_BETWEEN_ATTEMPTS_SECONDS
|
|
- name: ACTIVITY_MAX_DELAY_BETWEEN_ATTEMPTS_SECONDS
|
|
valueFrom:
|
|
configMapKeyRef:
|
|
name: airbyte-env
|
|
key: ACTIVITY_MAX_DELAY_BETWEEN_ATTEMPTS_SECONDS
|
|
- name: WORKFLOW_FAILURE_RESTART_DELAY_SECONDS
|
|
valueFrom:
|
|
configMapKeyRef:
|
|
name: airbyte-env
|
|
key: WORKFLOW_FAILURE_RESTART_DELAY_SECONDS
|
|
ports:
|
|
- containerPort: 9000 # for heartbeat server
|
|
- containerPort: 9001 # start temporal worker port pool
|
|
- containerPort: 9002
|
|
- containerPort: 9003
|
|
- containerPort: 9004
|
|
- containerPort: 9005
|
|
- containerPort: 9006
|
|
- containerPort: 9007
|
|
- containerPort: 9008
|
|
- containerPort: 9009
|
|
- containerPort: 9010
|
|
- containerPort: 9011
|
|
- containerPort: 9012
|
|
- containerPort: 9013
|
|
- containerPort: 9014
|
|
- containerPort: 9015
|
|
- containerPort: 9016
|
|
- containerPort: 9017
|
|
- containerPort: 9018
|
|
- containerPort: 9019
|
|
- containerPort: 9020
|
|
- containerPort: 9021
|
|
- containerPort: 9022
|
|
- containerPort: 9023
|
|
- containerPort: 9024
|
|
- containerPort: 9025
|
|
- containerPort: 9026
|
|
- containerPort: 9027
|
|
- containerPort: 9028
|
|
- containerPort: 9029
|
|
- containerPort: 9030 # end temporal worker port pool
|
|
volumeMounts:
|
|
- name: gcs-log-creds-volume
|
|
mountPath: /secrets/gcs-log-creds
|
|
readOnly: true
|
|
volumes:
|
|
- name: gcs-log-creds-volume
|
|
secret:
|
|
secretName: gcs-log-creds
|