✨ Source Datadog: add monitors and service_level_objectives (SLOs) streams (#30999)

Co-authored-by: Marcos Marx <marcosmarxm@users.noreply.github.com> Co-authored-by: marcosmarxm <marcosmarxm@gmail.com>
2025-12-19 18:14:56 -05:00 · 2023-12-04 14:08:46 -07:00
parent c11041a796
commit 7eb5f0e54f
10 changed files with 561 additions and 12 deletions
--- a/CONTRIBUTORS.md
+++ b/CONTRIBUTORS.md
@@ -108,6 +108,7 @@
 * [carlonuccio](https://github.com/carlonuccio)
 * [catpineapple](https://github.com/catpineapple)
 * [cgardens](https://github.com/cgardens)
+* [chadthman](https://github.com/chadthman)
 * [chandrasekharan98](https://github.com/chandrasekharan98)
 * [ChristoGrab](https://github.com/ChristoGrab)
 * [ChristopheDuong](https://github.com/ChristopheDuong)
--- a/airbyte-integrations/connectors/source-datadog/Dockerfile
+++ b/airbyte-integrations/connectors/source-datadog/Dockerfile
@@ -34,5 +34,5 @@ COPY source_datadog ./source_datadog
 ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py"
 ENTRYPOINT ["python", "/airbyte/integration_code/main.py"]

-LABEL io.airbyte.version=0.3.0
+LABEL io.airbyte.version=0.4.0
 LABEL io.airbyte.name=airbyte/source-datadog
--- a/airbyte-integrations/connectors/source-datadog/acceptance-test-config.yml
+++ b/airbyte-integrations/connectors/source-datadog/acceptance-test-config.yml
@@ -23,6 +23,8 @@ acceptance_tests:
            bypass_reason: Sandbox account can't seed this stream
          - name: incident_teams
            bypass_reason: Sandbox account can't seed this stream
+          - name: service_level_objectives
+            bypass_reason: Sandbox account can't seed this stream
  # TODO uncomment this block to specify that the tests should assert the connector outputs the records provided in the input file a file
  #        expect_records:
  #          path: "integration_tests/expected_records.jsonl"
@@ -41,3 +43,9 @@ acceptance_tests:
    tests:
      - config_path: "secrets/config.json"
        configured_catalog_path: "integration_tests/configured_catalog.json"
+        ignored_fields:
+          logs:
+            - name: "attributes.timestamp"
+              bypass_reason: "Change everytime"
+            - name: "id"
+              bypass_reason: "Change everytime"
--- a/airbyte-integrations/connectors/source-datadog/integration_tests/configured_catalog.json
+++ b/airbyte-integrations/connectors/source-datadog/integration_tests/configured_catalog.json
@@ -50,6 +50,16 @@
      "sync_mode": "full_refresh",
      "destination_sync_mode": "overwrite"
    },
+    {
+      "stream": {
+        "name": "monitors",
+        "json_schema": {},
+        "supported_sync_modes": ["full_refresh"],
+        "source_defined_primary_key": [["id"]]
+      },
+      "sync_mode": "full_refresh",
+      "destination_sync_mode": "overwrite"
+    },
    {
      "stream": {
        "name": "synthetic_tests",
--- a/airbyte-integrations/connectors/source-datadog/metadata.yaml
+++ b/airbyte-integrations/connectors/source-datadog/metadata.yaml
@@ -14,7 +14,7 @@ data:
  connectorSubtype: api
  connectorType: source
  definitionId: 1cfc30c7-82db-43f4-9fd7-ac1b42312cda
-  dockerImageTag: 0.3.0
+  dockerImageTag: 0.4.0
  dockerRepository: airbyte/source-datadog
  githubIssueLabel: source-datadog
  icon: datadog.svg
--- a/airbyte-integrations/connectors/source-datadog/setup.py
+++ b/airbyte-integrations/connectors/source-datadog/setup.py
@@ -6,14 +6,13 @@
 from setuptools import find_packages, setup

 MAIN_REQUIREMENTS = [
-    "airbyte-cdk~=0.1",
+    "airbyte-cdk",
 ]

 TEST_REQUIREMENTS = [
    "requests-mock~=1.9.3",
    "pytest~=6.2",
    "pytest-mock~=3.6.1",
-    "connector-acceptance-test",
 ]

 setup(
--- a/airbyte-integrations/connectors/source-datadog/source_datadog/manifest.yaml
+++ b/airbyte-integrations/connectors/source-datadog/source_datadog/manifest.yaml
@@ -83,6 +83,14 @@ definitions:
    end_datetime:
      datetime: "{{ config['end_date'] }}"
      datetime_format: "%Y-%m-%dT%H:%M:%SZ"
+    start_time_option:
+      type: RequestOption
+      field_name: filter[from]
+      inject_into: request_parameter
+    end_time_option:
+      type: RequestOption
+      field_name: filter[to]
+      inject_into: request_parameter
    step: "P1M"

  dashboards_stream:
@@ -121,6 +129,36 @@ definitions:
          type: DpathExtractor
          field_path: ["tests"]

+  monitors_stream:
+    $ref: "#/definitions/base_stream"
+    $parameters:
+      name: "monitors"
+      primary_key: "id"
+      path: "monitor"
+    retriever:
+      $ref: "#/definitions/retriever"
+      record_selector:
+        $ref: "#/definitions/selector"
+        type: RecordSelector
+        extractor:
+          type: DpathExtractor
+          field_path: []
+
+  service_level_objectives_stream:
+    $ref: "#/definitions/base_stream"
+    $parameters:
+      name: "service_level_objectives"
+      primary_key: "id"
+      path: "slo"
+    retriever:
+      $ref: "#/definitions/retriever"
+      record_selector:
+        $ref: "#/definitions/selector"
+        type: RecordSelector
+        extractor:
+          type: DpathExtractor
+          field_path: ["data"]
+
  audit_logs_stream:
    $ref: "#/definitions/base_stream"
    $parameters:
@@ -196,25 +234,29 @@ definitions:
        $ref: "#/definitions/base_paginator"

 streams:
+  - "#/definitions/audit_logs_stream"
  - "#/definitions/dashboards_stream"
  - "#/definitions/downtimes_stream"
-  - "#/definitions/synthetic_tests_stream"
-  - "#/definitions/audit_logs_stream"
+  - "#/definitions/incident_teams_stream"
+  - "#/definitions/incidents_stream"
  - "#/definitions/logs_stream"
  - "#/definitions/metrics_stream"
-  - "#/definitions/incidents_stream"
-  - "#/definitions/incident_teams_stream"
+  - "#/definitions/monitors_stream"
+  - "#/definitions/service_level_objectives_stream"
+  - "#/definitions/synthetic_tests_stream"
  - "#/definitions/users_stream"

 check:
  type: CheckStream
  stream_names:
+    - "audit_logs"
    - "dashboards"
    - "downtimes"
-    - "synthetic_tests"
-    - "audit_logs"
+    - "incident_teams"
+    - "incidents"
    - "logs"
    - "metrics"
-    - "incidents"
-    - "incident_teams"
+    - "monitors"
+    - "service_level_objectives"
+    - "synthetic_tests"
    - "users"
--- a/airbyte-integrations/connectors/source-datadog/source_datadog/schemas/monitors.json
+++ b/airbyte-integrations/connectors/source-datadog/source_datadog/schemas/monitors.json
@@ -0,0 +1,374 @@
+{
+  "$schema": "http://json-schema.org/draft-07/schema#",
+  "additionalProperties": true,
+  "properties": {
+    "created": {
+      "type": ["string", "null"]
+    },
+    "created_at": {
+      "type": ["integer", "null"]
+    },
+    "org_id": {
+      "type": ["integer", "null"]
+    },
+    "overall_state_modified": {
+      "type": ["string", "null"]
+    },
+    "creator": {
+      "additionalProperties": true,
+      "properties": {
+        "email": {
+          "type": ["string", "null"]
+        },
+        "handle": {
+          "type": ["string", "null"]
+        },
+        "name": {
+          "type": ["string", "null"]
+        }
+      },
+      "type": ["object", "null"]
+    },
+    "deleted": {
+      "type": ["string", "null"]
+    },
+    "id": {
+      "type": ["integer", "null"]
+    },
+    "matching_downtimes": {
+      "items": {
+        "additionalProperties": true,
+        "properties": {
+          "end": {
+            "type": ["integer", "null"]
+          },
+          "id": {
+            "type": ["integer", "null"]
+          },
+          "scope": {
+            "items": {
+              "type": ["string", "null"]
+            },
+            "type": ["array", "null"]
+          },
+          "start": {
+            "type": ["integer", "null"]
+          }
+        },
+        "type": ["object", "null"]
+      },
+      "type": ["array", "null"]
+    },
+    "message": {
+      "type": ["string", "null"]
+    },
+    "modified": {
+      "type": ["string", "null"]
+    },
+    "multi": {
+      "type": ["boolean", "null"]
+    },
+    "name": {
+      "type": ["string", "null"]
+    },
+    "options": {
+      "additionalProperties": true,
+      "properties": {
+        "aggregation": {
+          "additionalProperties": true,
+          "properties": {
+            "groupBy": {
+              "items": {
+                "type": ["string", "null"]
+              },
+              "type": ["array", "null"]
+            },
+            "metric": {
+              "type": ["string", "null"]
+            },
+            "type": {
+              "type": ["string", "null"]
+            }
+          },
+          "type": ["object", "null"]
+        },
+        "device_ids": {
+          "items": {
+            "type": ["string", "null"]
+          },
+          "type": ["array", "null"]
+        },
+        "enable_logs_sample": {
+          "type": ["boolean", "null"]
+        },
+        "enable_samples": {
+          "type": ["boolean", "null"]
+        },
+        "escalation_message": {
+          "type": ["string", "null"]
+        },
+        "evaluation_delay": {
+          "type": ["integer", "null"]
+        },
+        "group_retention_duration": {
+          "type": ["string", "null"]
+        },
+        "groupby_simple_monitor": {
+          "type": ["boolean", "null"]
+        },
+        "include_tags": {
+          "type": ["boolean", "null"]
+        },
+        "locked": {
+          "type": ["boolean", "null"]
+        },
+        "new_group_delay": {
+          "type": ["integer", "null"]
+        },
+        "new_host_delay": {
+          "type": ["integer", "null"]
+        },
+        "no_data_timeframe": {
+          "type": ["integer", "null"]
+        },
+        "notification_preset_name": {
+          "type": ["string", "null"]
+        },
+        "notify_audit": {
+          "type": ["boolean", "null"]
+        },
+        "notify_by": {
+          "items": {
+            "type": ["string", "null"]
+          },
+          "type": ["array", "null"]
+        },
+        "notify_no_data": {
+          "type": ["boolean", "null"]
+        },
+        "on_missing_data": {
+          "type": ["string", "null"]
+        },
+        "renotify_interval": {
+          "type": ["integer", "null"]
+        },
+        "renotify_occurrences": {
+          "type": ["integer", "null"]
+        },
+        "renotify_statuses": {
+          "items": {
+            "type": ["string", "null"]
+          },
+          "type": ["array", "null"]
+        },
+        "require_full_window": {
+          "type": ["boolean", "null"]
+        },
+        "scheduling_options": {
+          "additionalProperties": true,
+          "properties": {
+            "evaluation_window": {
+              "additionalProperties": true,
+              "properties": {
+                "day_starts": {
+                  "type": ["string", "null"]
+                },
+                "hour_starts": {
+                  "type": ["string", "null"]
+                },
+                "month_starts": {
+                  "type": ["integer", "null"]
+                }
+              },
+              "type": ["object", "null"]
+            }
+          },
+          "type": ["object", "null"]
+        },
+        "silenced": {
+          "patternProperties": {
+            ".+": {
+              "type": ["integer", "null"]
+            }
+          }
+        },
+        "synthetics_check_id": {
+          "type": ["string", "null"]
+        },
+        "threshold_windows": {
+          "additionalProperties": true,
+          "properties": {
+            "recovery_window": {
+              "type": ["string", "null"]
+            },
+            "trigger_window": {
+              "type": ["string", "null"]
+            }
+          },
+          "type": ["object", "null"]
+        },
+        "thresholds": {
+          "additionalProperties": true,
+          "properties": {
+            "critical": {
+              "type": ["number", "null"]
+            },
+            "critical_recovery": {
+              "type": ["number", "null"]
+            },
+            "ok": {
+              "type": ["number", "null"]
+            },
+            "unknown": {
+              "type": ["number", "null"]
+            },
+            "warning": {
+              "type": ["number", "null"]
+            },
+            "warning_recovery": {
+              "type": ["number", "null"]
+            }
+          },
+          "type": ["object", "null"]
+        },
+        "timeout_h": {
+          "type": ["integer", "null"]
+        },
+        "variables": {
+          "items": {
+            "additionalProperties": true,
+            "properties": {
+              "compute": {
+                "additionalProperties": true,
+                "properties": {
+                  "aggregation": {
+                    "type": ["string", "null"]
+                  },
+                  "interval": {
+                    "type": ["integer", "null"]
+                  },
+                  "metric": {
+                    "type": ["string", "null"]
+                  }
+                },
+                "type": ["object", "null"]
+              },
+              "data_source": {
+                "type": ["string", "null"]
+              },
+              "group_by": {
+                "items": {
+                  "additionalProperties": true,
+                  "properties": {
+                    "facet": {
+                      "type": ["string", "null"]
+                    },
+                    "limit": {
+                      "type": ["integer", "null"]
+                    },
+                    "sort": {
+                      "additionalProperties": true,
+                      "properties": {
+                        "aggregation": {
+                          "type": ["string", "null"]
+                        },
+                        "metric": {
+                          "type": ["string", "null"]
+                        },
+                        "order": {
+                          "type": ["string", "null"]
+                        }
+                      },
+                      "type": ["object", "null"]
+                    }
+                  },
+                  "type": ["object", "null"]
+                },
+                "type": ["array", "null"]
+              },
+              "indexes": {
+                "items": {
+                  "type": ["string", "null"]
+                },
+                "type": ["array", "null"]
+              },
+              "name": {
+                "type": ["string", "null"]
+              },
+              "search": {
+                "additionalProperties": true,
+                "properties": {
+                  "query": {
+                    "type": ["string", "null"]
+                  }
+                },
+                "type": ["object", "null"]
+              }
+            },
+            "type": ["object", "null"]
+          },
+          "type": ["array", "null"]
+        }
+      },
+      "type": ["object", "null"]
+    },
+    "overall_state": {
+      "type": ["string", "null"]
+    },
+    "priority": {
+      "type": ["integer", "null"]
+    },
+    "query": {
+      "type": ["string", "null"]
+    },
+    "restricted_roles": {
+      "items": {
+        "type": ["string", "null"]
+      },
+      "type": ["array", "null"]
+    },
+    "state": {
+      "properties": {
+        "groups": {
+          "patternProperties": {
+            ".+": {
+              "properties": {
+                "last_nodata_ts": {
+                  "type": ["integer", "null"]
+                },
+                "last_notified_ts": {
+                  "type": ["integer", "null"]
+                },
+                "last_resolved_ts": {
+                  "type": ["integer", "null"]
+                },
+                "last_triggered_ts": {
+                  "type": ["integer", "null"]
+                },
+                "name": {
+                  "type": ["string", "null"]
+                },
+                "status": {
+                  "type": ["string", "null"]
+                }
+              },
+              "type": ["object", "null"]
+            }
+          },
+          "type": ["object", "null"]
+        }
+      },
+      "type": ["object", "null"]
+    },
+    "tags": {
+      "items": {
+        "type": ["string", "null"]
+      },
+      "type": ["array", "null"]
+    },
+    "type": {
+      "type": ["string", "null"]
+    }
+  },
+  "type": ["object", "null"]
+}
--- a/airbyte-integrations/connectors/source-datadog/source_datadog/schemas/service_level_objectives.json
+++ b/airbyte-integrations/connectors/source-datadog/source_datadog/schemas/service_level_objectives.json
@@ -0,0 +1,112 @@
+{
+  "$schema": "http://json-schema.org/draft-07/schema#",
+  "additionalProperties": true,
+  "properties": {
+    "created_at": {
+      "type": ["integer", "null"]
+    },
+    "creator": {
+      "additionalProperties": true,
+      "properties": {
+        "email": {
+          "type": ["string", "null"]
+        },
+        "handle": {
+          "type": ["string", "null"]
+        },
+        "name": {
+          "type": ["string", "null"]
+        }
+      },
+      "type": ["object", "null"]
+    },
+    "description": {
+      "type": ["string", "null"]
+    },
+    "groups": {
+      "items":{
+        "type": ["string", "null"]
+      },
+      "type": ["array", "null"]
+    },
+    "id": {
+      "type": ["string", "null"]
+    },
+    "modified_at": {
+      "type": ["integer", "null"]
+    },
+    "monitor_ids": {
+      "items": {
+        "type": ["integer", "null"]
+      },
+      "type": ["array", "null"]
+    },
+    "monitor_tags": {
+      "items": {
+        "type": ["string", "null"]
+      },
+      "type": ["array", "null"]
+    },
+    "name": {
+      "type": ["string", "null"]
+    },
+    "query": {
+      "additionalProperties": true,
+      "properties": {
+        "denominator": {
+          "type": ["string", "null"]
+        },
+        "numerator": {
+          "type": ["string", "null"]
+        }
+      },
+      "type": ["object", "null"]
+    },
+    "tags": {
+      "items": {
+        "type": ["string", "null"]
+      },
+      "type": ["array", "null"]
+    },
+    "target_threshold": {
+      "type": ["number", "null"]
+    },
+    "thresholds": {
+      "items": {
+        "additionalProperties": true,
+        "properties": {
+          "target": {
+            "type": ["number", "null"]
+          },
+          "target_display": {
+            "type": ["string", "null"]
+          },
+          "timeframe": {
+            "type": ["string", "null"]
+          },
+          "warning": {
+            "type": ["number", "null"]
+          },
+          "warning_display": {
+            "type": ["string", "null"]
+          }
+        },
+        "type": ["object", "null"]
+      },
+      "type": ["array", "null"]
+    },
+    "timeframe": {
+      "type": ["string", "null"]
+    },
+    "type": {
+      "type": ["string", "null"]
+    },
+    "type_id": {
+      "type": ["integer", "null"]
+    },
+    "warning_threshold": {
+      "type": ["number", "null"]
+    }
+  },
+  "type": ["object", "null"]
+}
--- a/docs/integrations/sources/datadog.md
+++ b/docs/integrations/sources/datadog.md
@@ -63,6 +63,8 @@ The Datadog source connector supports the following [sync modes](https://docs.ai
 * [Incidents](https://docs.datadoghq.com/api/latest/incidents/#get-a-list-of-incidents)
 * [Logs](https://docs.datadoghq.com/api/latest/logs/#search-logs)
 * [Metrics](https://docs.datadoghq.com/api/latest/metrics/#get-a-list-of-metrics)
+* [Monitors](https://docs.datadoghq.com/api/latest/monitors/#get-all-monitor-details)
+* [ServiceLevelObjectives](https://docs.datadoghq.com/api/latest/service-level-objectives/#get-all-slos)
 * [SyntheticTests](https://docs.datadoghq.com/api/latest/synthetics/#get-the-list-of-all-tests)
 * [Users](https://docs.datadoghq.com/api/latest/users/#list-all-users)
 * [Series](https://docs.datadoghq.com/api/latest/metrics/?code-lang=curl#query-timeseries-data-across-multiple-products)
@@ -71,6 +73,7 @@ The Datadog source connector supports the following [sync modes](https://docs.ai

 | Version | Date       | Pull Request                                              | Subject                                                                      |
 |:--------|:-----------|:----------------------------------------------------------|:-----------------------------------------------------------------------------|
+| 0.4.0   | 2023-12-04 | [30999](https://github.com/airbytehq/airbyte/pull/30999)  | Add `monitors` and `service_level_objectives` Streams |
 | 0.3.0   | 2023-08-27 | [29885](https://github.com/airbytehq/airbyte/pull/29885)  | Migrate to low code                                                          |
 | 0.2.2   | 2023-07-10 | [28089](https://github.com/airbytehq/airbyte/pull/28089)  | Additional stream and query details in response                              |
 | 0.2.1   | 2023-06-28 | [26534](https://github.com/airbytehq/airbyte/pull/26534)  | Support multiple query streams and pulling data from different datadog sites |