jprdonnelly/airbyte
1
0
Fork 0
mirror of synced 2025-12-25 02:09:19 -05:00
Code Issues Packages Projects Releases Wiki Activity

Source Google Analytics v4: - add pk and lookback window (#26283)

Browse Source 
* primary_key added

Signed-off-by: Sergey Chvalyuk <grubberr@gmail.com>

* bump 0.2.3

Signed-off-by: Sergey Chvalyuk <grubberr@gmail.com>

* Automated Change

* test_parse_response fixed

Signed-off-by: Sergey Chvalyuk <grubberr@gmail.com>

* Automated Change

* primary_key added

Signed-off-by: Sergey Chvalyuk <grubberr@gmail.com>

* metadata = None added

Signed-off-by: Sergey Chvalyuk <grubberr@gmail.com>

* LOOKBACK_WINDOW added

Signed-off-by: Sergey Chvalyuk <grubberr@gmail.com>

* google-analytics-data-api.md updated

Signed-off-by: Sergey Chvalyuk <grubberr@gmail.com>

* threshold_days: 2 added

Signed-off-by: Sergey Chvalyuk <grubberr@gmail.com>

* remove uuid from acceptance-test-config.yml

Signed-off-by: Sergey Chvalyuk <grubberr@gmail.com>

* uuid removed

Signed-off-by: Sergey Chvalyuk <grubberr@gmail.com>

* Automated Change

* oss_registry.json - removed

Signed-off-by: Sergey Chvalyuk <grubberr@gmail.com>

* date_range added

Signed-off-by: Sergey Chvalyuk <grubberr@gmail.com>

* date_range added for pivot_report pk

Signed-off-by: Sergey Chvalyuk <grubberr@gmail.com>

* improve LOOKBACK_WINDOW

Signed-off-by: Sergey Chvalyuk <grubberr@gmail.com>

* date_range -> (startDate, endDate)

Signed-off-by: Sergey Chvalyuk <grubberr@gmail.com>

* bump 0.3.0

Signed-off-by: Sergey Chvalyuk <grubberr@gmail.com>

* get_source_defined_primary_key added

Signed-off-by: Sergey Chvalyuk <grubberr@gmail.com>

* added comment about cohort get_json_schema

* Major Version Number Bump

---------

Signed-off-by: Sergey Chvalyuk <grubberr@gmail.com>
Co-authored-by: grubberr <grubberr@users.noreply.github.com>
Co-authored-by: Daryna Ishchenko <darina.ishchenko17@gmail.com>
This commit is contained in:
Serhii Chvaliuk
2023-06-22 19:00:38 +03:00
committed by GitHub
parent a8e99a46e6
commit 019153f178
11 changed files with 220 additions and 298 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
airbyte-integrations/connectors/source-google-analytics-data-api/Dockerfile
View File

@@ -28,5 +28,5 @@ COPY source_google_analytics_data_api ./source_google_analytics_data_api
ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py"
ENTRYPOINT ["python", "/airbyte/integration_code/main.py"]
LABEL io.airbyte.version=0.2.7
LABEL io.airbyte.version=1.0.0
LABEL io.airbyte.name=airbyte/source-google-analytics-data-api

51
airbyte-integrations/connectors/source-google-analytics-data-api/acceptance-test-config.yml
View File

@@ -29,43 +29,21 @@ acceptance_tests:
exact_order: no
extra_records: yes
ignored_fields:
daily_active_users:
- name: uuid
bypass_reason: "dynamically created field"
weekly_active_users:
- name: uuid
bypass_reason: "dynamically created field"
four_weekly_active_users:
- name: uuid
bypass_reason: "dynamically created field"
devices:
- name: uuid
bypass_reason: "dynamically created field"
- name: averageSessionDuration
bypass_reason: "dynamic field"
locations:
- name: uuid
bypass_reason: "dynamically created field"
- name: averageSessionDuration
bypass_reason: "dynamic field"
pages:
- name: uuid
bypass_reason: "dynamically created field"
- name: screenPageViews
bypass_reason: "dynamically created field"
- name: bounceRate
bypass_reason: "dynamically created field"
website_overview:
- name: uuid
bypass_reason: "dynamically created field"
- name: averageSessionDuration
bypass_reason: "dynamically created field"
cohort_report:
- name: uuid
bypass_reason: "dynamically created field"
pivot_report:
- name: uuid
bypass_reason: "dynamically created field"
- name: sessions
bypass_reason: "volatile data"
full_refresh:
@@ -73,47 +51,22 @@ acceptance_tests:
- config_path: "secrets/config.json"
configured_catalog_path: "integration_tests/configured_catalog.json"
ignored_fields:
daily_active_users:
- name: uuid
bypass_reason: "dynamically created field"
weekly_active_users:
- name: uuid
bypass_reason: "dynamically created field"
four_weekly_active_users:
- name: uuid
bypass_reason: "dynamically created field"
devices:
- name: uuid
bypass_reason: "dynamically created field"
- name: averageSessionDuration
bypass_reason: "dynamic field"
locations:
- name: uuid
bypass_reason: "dynamically created field"
- name: averageSessionDuration
bypass_reason: "dynamic field"
pages:
- name: uuid
bypass_reason: "dynamically created field"
traffic_sources:
- name: uuid
bypass_reason: "dynamically created field"
- name: averageSessionDuration
bypass_reason: "dynamically created field"
website_overview:
- name: uuid
bypass_reason: "dynamically created field"
- name: averageSessionDuration
bypass_reason: "dynamically created field"
cohort_report:
- name: uuid
bypass_reason: "dynamically created field"
pivot_report:
- name: uuid
bypass_reason: "dynamically created field"
incremental:
tests:
- config_path: "secrets/config.json"
configured_catalog_path: "integration_tests/incremental_catalog.json"
configured_catalog_path: "integration_tests/configured_catalog.json"
future_state:
future_state_path: "integration_tests/abnormal_state.json"
threshold_days: 2

66
airbyte-integrations/connectors/source-google-analytics-data-api/integration_tests/configured_catalog.json
View File

@@ -7,10 +7,11 @@
"supported_sync_modes": ["full_refresh", "incremental"],
"source_defined_cursor": true,
"default_cursor_field": ["date"],
"source_defined_primary_key": [["uuid"]]
"source_defined_primary_key": [["property_id"], ["date"]]
},
"sync_mode": "full_refresh",
"destination_sync_mode": "overwrite"
"sync_mode": "incremental",
"destination_sync_mode": "overwrite",
"primary_key": [["property_id"], ["date"]]
},
{
"stream": {
@@ -19,10 +20,11 @@
"supported_sync_modes": ["full_refresh", "incremental"],
"source_defined_cursor": true,
"default_cursor_field": ["date"],
"source_defined_primary_key": [["uuid"]]
"source_defined_primary_key": [["property_id"], ["date"]]
},
"sync_mode": "full_refresh",
"destination_sync_mode": "overwrite"
"sync_mode": "incremental",
"destination_sync_mode": "overwrite",
"primary_key": [["property_id"], ["date"]]
},
{
"stream": {
@@ -31,10 +33,11 @@
"supported_sync_modes": ["full_refresh", "incremental"],
"source_defined_cursor": true,
"default_cursor_field": ["date"],
"source_defined_primary_key": [["uuid"]]
"source_defined_primary_key": [["property_id"], ["date"]]
},
"sync_mode": "full_refresh",
"destination_sync_mode": "overwrite"
"sync_mode": "incremental",
"destination_sync_mode": "overwrite",
"primary_key": [["property_id"], ["date"]]
},
{
"stream": {
@@ -43,10 +46,11 @@
"supported_sync_modes": ["full_refresh", "incremental"],
"source_defined_cursor": true,
"default_cursor_field": ["date"],
"source_defined_primary_key": [["uuid"]]
"source_defined_primary_key": [["property_id"], ["date"], ["deviceCategory"], ["operatingSystem"], ["browser"]]
},
"sync_mode": "full_refresh",
"destination_sync_mode": "overwrite"
"sync_mode": "incremental",
"destination_sync_mode": "overwrite",
"primary_key": [["property_id"], ["date"], ["deviceCategory"], ["operatingSystem"], ["browser"]]
},
{
"stream": {
@@ -55,10 +59,11 @@
"supported_sync_modes": ["full_refresh", "incremental"],
"source_defined_cursor": true,
"default_cursor_field": ["date"],
"source_defined_primary_key": [["uuid"]]
"source_defined_primary_key": [["property_id"], ["region"], ["country"], ["city"], ["date"]]
},
"sync_mode": "full_refresh",
"destination_sync_mode": "overwrite"
"sync_mode": "incremental",
"destination_sync_mode": "overwrite",
"primary_key": [["property_id"], ["region"], ["country"], ["city"], ["date"]]
},
{
"stream": {
@@ -67,10 +72,11 @@
"supported_sync_modes": ["full_refresh", "incremental"],
"source_defined_cursor": true,
"default_cursor_field": ["date"],
"source_defined_primary_key": [["uuid"]]
"source_defined_primary_key": [["property_id"], ["date"], ["hostName"], ["pagePathPlusQueryString"]]
},
"sync_mode": "full_refresh",
"destination_sync_mode": "overwrite"
"sync_mode": "incremental",
"destination_sync_mode": "overwrite",
"primary_key": [["property_id"], ["date"], ["hostName"], ["pagePathPlusQueryString"]]
},
{
"stream": {
@@ -79,10 +85,11 @@
"supported_sync_modes": ["full_refresh", "incremental"],
"source_defined_cursor": true,
"default_cursor_field": ["date"],
"source_defined_primary_key": [["uuid"]]
"source_defined_primary_key": [["property_id"], ["date"], ["sessionSource"], ["sessionMedium"]]
},
"sync_mode": "full_refresh",
"destination_sync_mode": "overwrite"
"sync_mode": "incremental",
"destination_sync_mode": "overwrite",
"primary_key": [["property_id"], ["date"], ["sessionSource"], ["sessionMedium"]]
},
{
"stream": {
@@ -91,30 +98,33 @@
"supported_sync_modes": ["full_refresh", "incremental"],
"source_defined_cursor": true,
"default_cursor_field": ["date"],
"source_defined_primary_key": [["uuid"]]
"source_defined_primary_key": [["property_id"], ["date"]]
},
"sync_mode": "full_refresh",
"destination_sync_mode": "overwrite"
"sync_mode": "incremental",
"destination_sync_mode": "overwrite",
"primary_key": [["property_id"], ["date"]]
},
{
"stream": {
"name": "cohort_report",
"json_schema": {},
"supported_sync_modes": ["full_refresh"],
"source_defined_primary_key": [["uuid"]]
"source_defined_primary_key": [["property_id"], ["cohort"], ["cohortNthDay"]]
},
"sync_mode": "full_refresh",
"destination_sync_mode": "overwrite"
"destination_sync_mode": "overwrite",
"primary_key": [["property_id"], ["cohort"], ["cohortNthDay"]]
},
{
"stream": {
"name": "pivot_report",
"json_schema": {},
"supported_sync_modes": ["full_refresh"],
"source_defined_primary_key": [["uuid"]]
"source_defined_primary_key": [["property_id"], ["browser"], ["country"], ["language"], ["startDate"], ["endDate"]]
},
"sync_mode": "full_refresh",
"destination_sync_mode": "overwrite"
"destination_sync_mode": "overwrite",
"primary_key": [["property_id"], ["browser"], ["country"], ["language"], ["startDate"], ["endDate"]]
}
]
}

180
airbyte-integrations/connectors/source-google-analytics-data-api/integration_tests/expected_records.jsonl
View File

@@ -1,90 +1,90 @@
{"stream": "daily_active_users", "data": {"uuid": "8958ad05-9217-4232-af3a-842b2ef3066a", "property_id": "314186564", "date": "20230406", "active1DayUsers": 2562}, "emitted_at": 1681405954033}
{"stream": "daily_active_users", "data": {"uuid": "2ededb64-296c-4cd3-8285-258f952e6d01", "property_id": "314186564", "date": "20230403", "active1DayUsers": 2521}, "emitted_at": 1681405954034}
{"stream": "daily_active_users", "data": {"uuid": "aee7c406-9668-4e7d-8f1e-e4b6e944781a", "property_id": "314186564", "date": "20230404", "active1DayUsers": 2386}, "emitted_at": 1681405954034}
{"stream": "daily_active_users", "data": {"uuid": "8fdcf306-c6ce-418d-a8b0-114c3e0853e3", "property_id": "314186564", "date": "20230405", "active1DayUsers": 2318}, "emitted_at": 1681405954035}
{"stream": "daily_active_users", "data": {"uuid": "ddae49fa-6a49-409c-a3d9-972043e1297e", "property_id": "314186564", "date": "20230411", "active1DayUsers": 2248}, "emitted_at": 1681405954035}
{"stream": "daily_active_users", "data": {"uuid": "d3c0fe8d-729b-49d5-ba09-836d11c72b36", "property_id": "314186564", "date": "20230412", "active1DayUsers": 2164}, "emitted_at": 1681405954036}
{"stream": "daily_active_users", "data": {"uuid": "3bed51b3-fe19-4cfe-aa7b-2f834444e4ac", "property_id": "314186564", "date": "20230410", "active1DayUsers": 2021}, "emitted_at": 1681405954036}
{"stream": "daily_active_users", "data": {"uuid": "930dc88c-2176-44d7-b89d-f7a0deed7cb6", "property_id": "314186564", "date": "20230407", "active1DayUsers": 1628}, "emitted_at": 1681405954037}
{"stream": "daily_active_users", "data": {"uuid": "862aee10-bc17-4fcb-9917-c03dcf930b7a", "property_id": "314186564", "date": "20230409", "active1DayUsers": 1009}, "emitted_at": 1681405954037}
{"stream": "daily_active_users", "data": {"uuid": "888e8a03-f908-49ea-bc24-47d3e0c71ce5", "property_id": "314186564", "date": "20230402", "active1DayUsers": 978}, "emitted_at": 1681405954038}
{"stream": "weekly_active_users", "data": {"uuid": "552f657c-894d-42a5-b136-621a27c83f20", "property_id": "314186564", "date": "20230403", "active7DayUsers": 11840}, "emitted_at": 1681405954684}
{"stream": "weekly_active_users", "data": {"uuid": "ab335ba9-16a4-47f7-9475-0cd0c71e13c5", "property_id": "314186564", "date": "20230406", "active7DayUsers": 11828}, "emitted_at": 1681405954685}
{"stream": "weekly_active_users", "data": {"uuid": "c2249443-a731-454f-a92d-dcabde1e053e", "property_id": "314186564", "date": "20230404", "active7DayUsers": 11812}, "emitted_at": 1681405954685}
{"stream": "weekly_active_users", "data": {"uuid": "9031b092-0365-45f6-a3fd-a42697fc60e5", "property_id": "314186564", "date": "20230405", "active7DayUsers": 11751}, "emitted_at": 1681405954685}
{"stream": "weekly_active_users", "data": {"uuid": "686ec0f2-245a-41ee-8b2f-28c6ff58cca3", "property_id": "314186564", "date": "20230408", "active7DayUsers": 11745}, "emitted_at": 1681405954685}
{"stream": "weekly_active_users", "data": {"uuid": "c230f0bb-e08d-410c-aaf7-92ea55c1eb3c", "property_id": "314186564", "date": "20230409", "active7DayUsers": 11739}, "emitted_at": 1681405954685}
{"stream": "weekly_active_users", "data": {"uuid": "360eeddb-8e7c-4cf8-ba68-fe0f74bdd4b2", "property_id": "314186564", "date": "20230407", "active7DayUsers": 11637}, "emitted_at": 1681405954685}
{"stream": "weekly_active_users", "data": {"uuid": "fa08f066-b0fe-4cce-9ff1-aa51533a36a2", "property_id": "314186564", "date": "20230401", "active7DayUsers": 11547}, "emitted_at": 1681405954685}
{"stream": "weekly_active_users", "data": {"uuid": "99849845-7511-46ee-bb0f-b99fcbe86094", "property_id": "314186564", "date": "20230402", "active7DayUsers": 11521}, "emitted_at": 1681405954685}
{"stream": "weekly_active_users", "data": {"uuid": "1537fbde-6a1e-4676-ad38-3952c6c49517", "property_id": "314186564", "date": "20230410", "active7DayUsers": 11369}, "emitted_at": 1681405954686}
{"stream": "four_weekly_active_users", "data": {"uuid": "3ea1880f-1e1b-42f8-858f-d66c49400592", "property_id": "314186564", "date": "20230401", "active28DayUsers": 48082}, "emitted_at": 1681405955854}
{"stream": "four_weekly_active_users", "data": {"uuid": "e3697089-6a56-466f-893c-57506cce44f4", "property_id": "314186564", "date": "20230402", "active28DayUsers": 47927}, "emitted_at": 1681405955854}
{"stream": "four_weekly_active_users", "data": {"uuid": "2a76a756-b625-4291-9012-09d6449b4460", "property_id": "314186564", "date": "20230403", "active28DayUsers": 44678}, "emitted_at": 1681405955854}
{"stream": "four_weekly_active_users", "data": {"uuid": "30f93662-7a47-4455-bb87-4e23753d7836", "property_id": "314186564", "date": "20230404", "active28DayUsers": 42997}, "emitted_at": 1681405955854}
{"stream": "four_weekly_active_users", "data": {"uuid": "bf5980dd-18bd-4dd3-b6f7-123a8d9bc7ba", "property_id": "314186564", "date": "20230405", "active28DayUsers": 42219}, "emitted_at": 1681405955855}
{"stream": "four_weekly_active_users", "data": {"uuid": "095af0ca-ca67-44a9-acaf-fbe1f04beb8f", "property_id": "314186564", "date": "20230406", "active28DayUsers": 42028}, "emitted_at": 1681405955855}
{"stream": "four_weekly_active_users", "data": {"uuid": "29958f7e-91a1-4377-a5e7-763ee1fc26f7", "property_id": "314186564", "date": "20230407", "active28DayUsers": 41851}, "emitted_at": 1681405955855}
{"stream": "four_weekly_active_users", "data": {"uuid": "b5cd639e-99a4-4fd9-871e-9883a509d35e", "property_id": "314186564", "date": "20230408", "active28DayUsers": 41775}, "emitted_at": 1681405955855}
{"stream": "four_weekly_active_users", "data": {"uuid": "b85e1864-aca2-45ae-8bc2-614da510ac29", "property_id": "314186564", "date": "20230409", "active28DayUsers": 41717}, "emitted_at": 1681405955855}
{"stream": "four_weekly_active_users", "data": {"uuid": "505deea8-b4d0-4d76-9c78-1faaa8e3c725", "property_id": "314186564", "date": "20230410", "active28DayUsers": 41212}, "emitted_at": 1681405955855}
{"stream": "devices", "data": {"uuid": "f43b1120-b565-4174-a974-251de4ddb3e7", "property_id": "314186564", "date": "20230411", "deviceCategory": "desktop", "operatingSystem": "Macintosh", "browser": "Chrome", "totalUsers": 973, "newUsers": 368, "sessions": 1667, "sessionsPerUser": 2.110126582278481, "averageSessionDuration": 308.8923676994601, "screenPageViews": 5367, "screenPageViewsPerSession": 3.2195560887822436, "bounceRate": 0.498500299940012}, "emitted_at": 1681405958296}
{"stream": "devices", "data": {"uuid": "11fa7956-54f4-42ae-bf50-ccd7f78acb43", "property_id": "314186564", "date": "20230412", "deviceCategory": "desktop", "operatingSystem": "Macintosh", "browser": "Chrome", "totalUsers": 969, "newUsers": 350, "sessions": 1588, "sessionsPerUser": 2.0025220680958387, "averageSessionDuration": 336.108126070529, "screenPageViews": 4726, "screenPageViewsPerSession": 2.9760705289672544, "bounceRate": 0.5012594458438288}, "emitted_at": 1681405958296}
{"stream": "devices", "data": {"uuid": "3a7a9be6-715f-407d-964d-36a335d1f89c", "property_id": "314186564", "date": "20230404", "deviceCategory": "desktop", "operatingSystem": "Macintosh", "browser": "Chrome", "totalUsers": 942, "newUsers": 352, "sessions": 1554, "sessionsPerUser": 2.007751937984496, "averageSessionDuration": 328.66656451029604, "screenPageViews": 5217, "screenPageViewsPerSession": 3.357142857142857, "bounceRate": 0.4954954954954955}, "emitted_at": 1681405958296}
{"stream": "devices", "data": {"uuid": "d174c54a-7091-4736-b921-c191bf5dd4b6", "property_id": "314186564", "date": "20230406", "deviceCategory": "desktop", "operatingSystem": "Macintosh", "browser": "Chrome", "totalUsers": 942, "newUsers": 389, "sessions": 1551, "sessionsPerUser": 1.9783163265306123, "averageSessionDuration": 357.5382107272727, "screenPageViews": 5102, "screenPageViewsPerSession": 3.289490651192779, "bounceRate": 0.49258542875564154}, "emitted_at": 1681405958297}
{"stream": "devices", "data": {"uuid": "889df6a1-f480-483e-8957-d4ca767028f8", "property_id": "314186564", "date": "20230403", "deviceCategory": "desktop", "operatingSystem": "Macintosh", "browser": "Chrome", "totalUsers": 929, "newUsers": 341, "sessions": 1546, "sessionsPerUser": 2.0558510638297873, "averageSessionDuration": 315.4776974385511, "screenPageViews": 5116, "screenPageViewsPerSession": 3.309184993531695, "bounceRate": 0.5071151358344114}, "emitted_at": 1681405958297}
{"stream": "devices", "data": {"uuid": "7a12fdb1-65bd-4f9d-9b28-f7d4bd9efc45", "property_id": "314186564", "date": "20230405", "deviceCategory": "desktop", "operatingSystem": "Macintosh", "browser": "Chrome", "totalUsers": 926, "newUsers": 363, "sessions": 1573, "sessionsPerUser": 2.0428571428571427, "averageSessionDuration": 346.09502719898285, "screenPageViews": 5032, "screenPageViewsPerSession": 3.1989828353464715, "bounceRate": 0.4869675778766688}, "emitted_at": 1681405958297}
{"stream": "devices", "data": {"uuid": "36283f3b-12f0-44bf-af7e-fa17790a536a", "property_id": "314186564", "date": "20230410", "deviceCategory": "desktop", "operatingSystem": "Macintosh", "browser": "Chrome", "totalUsers": 920, "newUsers": 374, "sessions": 1524, "sessionsPerUser": 2.0456375838926175, "averageSessionDuration": 255.77025801837266, "screenPageViews": 4025, "screenPageViewsPerSession": 2.641076115485564, "bounceRate": 0.5255905511811023}, "emitted_at": 1681405958297}
{"stream": "devices", "data": {"uuid": "9ed102f9-6670-4883-890a-86d647e3e08a", "property_id": "314186564", "date": "20230403", "deviceCategory": "desktop", "operatingSystem": "Windows", "browser": "Chrome", "totalUsers": 781, "newUsers": 366, "sessions": 1184, "sessionsPerUser": 1.8528951486697967, "averageSessionDuration": 278.84846059881755, "screenPageViews": 2993, "screenPageViewsPerSession": 2.5278716216216215, "bounceRate": 0.5616554054054054}, "emitted_at": 1681405958297}
{"stream": "devices", "data": {"uuid": "a4d1798d-40ec-4653-b3ec-2c4951fa7323", "property_id": "314186564", "date": "20230411", "deviceCategory": "desktop", "operatingSystem": "Windows", "browser": "Chrome", "totalUsers": 760, "newUsers": 365, "sessions": 1155, "sessionsPerUser": 1.896551724137931, "averageSessionDuration": 264.1307251896104, "screenPageViews": 2452, "screenPageViewsPerSession": 2.122943722943723, "bounceRate": 0.5316017316017316}, "emitted_at": 1681405958298}
{"stream": "devices", "data": {"uuid": "0f4cb5c7-703f-4e21-9efd-828da56eb03b", "property_id": "314186564", "date": "20230404", "deviceCategory": "desktop", "operatingSystem": "Windows", "browser": "Chrome", "totalUsers": 727, "newUsers": 345, "sessions": 1137, "sessionsPerUser": 1.8517915309446253, "averageSessionDuration": 252.06245670272648, "screenPageViews": 2601, "screenPageViewsPerSession": 2.287598944591029, "bounceRate": 0.5488126649076517}, "emitted_at": 1681405958298}
{"stream": "locations", "data": {"uuid": "5606f203-9304-4b42-82c8-af3d825cd34a", "property_id": "314186564", "region": "New York", "country": "United States", "city": "New York", "date": "20230406", "totalUsers": 108, "newUsers": 62, "sessions": 157, "sessionsPerUser": 1.6354166666666667, "averageSessionDuration": 435.44268001273895, "screenPageViews": 534, "screenPageViewsPerSession": 3.4012738853503186, "bounceRate": 0.5031847133757962}, "emitted_at": 1681405962136}
{"stream": "locations", "data": {"uuid": "759886f0-3723-44bf-ba3f-1b994169ed85", "property_id": "314186564", "region": "New York", "country": "United States", "city": "New York", "date": "20230405", "totalUsers": 95, "newUsers": 54, "sessions": 123, "sessionsPerUser": 1.5769230769230769, "averageSessionDuration": 499.2074986666667, "screenPageViews": 481, "screenPageViewsPerSession": 3.910569105691057, "bounceRate": 0.44715447154471544}, "emitted_at": 1681405962136}
{"stream": "locations", "data": {"uuid": "81c1cd25-8d43-48a9-836b-e10b1991466a", "property_id": "314186564", "region": "New York", "country": "United States", "city": "New York", "date": "20230403", "totalUsers": 94, "newUsers": 46, "sessions": 126, "sessionsPerUser": 1.68, "averageSessionDuration": 424.00281903174607, "screenPageViews": 499, "screenPageViewsPerSession": 3.9603174603174605, "bounceRate": 0.5238095238095238}, "emitted_at": 1681405962136}
{"stream": "locations", "data": {"uuid": "1a833419-629e-4678-9fb2-f97f10da5d40", "property_id": "314186564", "region": "New York", "country": "United States", "city": "New York", "date": "20230404", "totalUsers": 85, "newUsers": 47, "sessions": 121, "sessionsPerUser": 1.6575342465753424, "averageSessionDuration": 378.81275640495863, "screenPageViews": 434, "screenPageViewsPerSession": 3.5867768595041323, "bounceRate": 0.48760330578512395}, "emitted_at": 1681405962136}
{"stream": "locations", "data": {"uuid": "577fd852-67a2-43c3-8321-56fa00dac58c", "property_id": "314186564", "region": "New York", "country": "United States", "city": "New York", "date": "20230412", "totalUsers": 85, "newUsers": 49, "sessions": 131, "sessionsPerUser": 1.8194444444444444, "averageSessionDuration": 379.1322029236641, "screenPageViews": 391, "screenPageViewsPerSession": 2.984732824427481, "bounceRate": 0.5267175572519084}, "emitted_at": 1681405962137}
{"stream": "locations", "data": {"uuid": "bb0f3fa2-dc9c-475d-a47c-ace82c720da2", "property_id": "314186564", "region": "New York", "country": "United States", "city": "New York", "date": "20230410", "totalUsers": 81, "newUsers": 42, "sessions": 135, "sessionsPerUser": 1.9565217391304348, "averageSessionDuration": 303.13140742962963, "screenPageViews": 376, "screenPageViewsPerSession": 2.785185185185185, "bounceRate": 0.5407407407407407}, "emitted_at": 1681405962137}
{"stream": "locations", "data": {"uuid": "f7057f58-fbbd-4947-ae37-a7a29d6700ac", "property_id": "314186564", "region": "New York", "country": "United States", "city": "New York", "date": "20230411", "totalUsers": 81, "newUsers": 38, "sessions": 123, "sessionsPerUser": 1.9523809523809523, "averageSessionDuration": 362.51537134146344, "screenPageViews": 312, "screenPageViewsPerSession": 2.5365853658536586, "bounceRate": 0.5934959349593496}, "emitted_at": 1681405962137}
{"stream": "locations", "data": {"uuid": "afb62a0d-5e5a-4f88-8367-3fe45a35d06f", "property_id": "314186564", "region": "Karnataka", "country": "India", "city": "Bengaluru", "date": "20230411", "totalUsers": 76, "newUsers": 52, "sessions": 123, "sessionsPerUser": 1.8636363636363635, "averageSessionDuration": 203.00314456910567, "screenPageViews": 261, "screenPageViewsPerSession": 2.1219512195121952, "bounceRate": 0.4959349593495935}, "emitted_at": 1681405962137}
{"stream": "locations", "data": {"uuid": "ce319e7f-8f8f-4c73-9f45-9a80a08bcf33", "property_id": "314186564", "region": "Karnataka", "country": "India", "city": "Bengaluru", "date": "20230403", "totalUsers": 69, "newUsers": 34, "sessions": 102, "sessionsPerUser": 1.728813559322034, "averageSessionDuration": 256.4942830490196, "screenPageViews": 216, "screenPageViewsPerSession": 2.1176470588235294, "bounceRate": 0.5490196078431373}, "emitted_at": 1681405962137}
{"stream": "locations", "data": {"uuid": "28fa18ea-3a0c-4c09-9ffc-b24d2d5abeaf", "property_id": "314186564", "region": "New York", "country": "United States", "city": "New York", "date": "20230407", "totalUsers": 69, "newUsers": 30, "sessions": 98, "sessionsPerUser": 1.849056603773585, "averageSessionDuration": 489.54009168367344, "screenPageViews": 376, "screenPageViewsPerSession": 3.836734693877551, "bounceRate": 0.4489795918367347}, "emitted_at": 1681405962137}
{"stream": "pages", "data": {"uuid": "1d042583-6008-4bb8-abad-a1d3a171c9c5", "property_id": "314186564", "date": "20230405", "hostName": "airbyte.com", "pagePathPlusQueryString": "/", "screenPageViews": 1190, "bounceRate": 0.5516393442622951}, "emitted_at": 1681405967183}
{"stream": "pages", "data": {"uuid": "e24096ca-ad32-49eb-824a-2a65e52b9a08", "property_id": "314186564", "date": "20230411", "hostName": "airbyte.com", "pagePathPlusQueryString": "/", "screenPageViews": 1151, "bounceRate": 0.5400641025641025}, "emitted_at": 1681405967184}
{"stream": "pages", "data": {"uuid": "8911f922-c707-4af1-95b5-18a81e31b5af", "property_id": "314186564", "date": "20230404", "hostName": "airbyte.com", "pagePathPlusQueryString": "/", "screenPageViews": 1137, "bounceRate": 0.5617232808616405}, "emitted_at": 1681405967184}
{"stream": "pages", "data": {"uuid": "315dad3c-81e5-432f-a47c-134e89a7cf42", "property_id": "314186564", "date": "20230410", "hostName": "airbyte.com", "pagePathPlusQueryString": "/", "screenPageViews": 1099, "bounceRate": 0.5416666666666666}, "emitted_at": 1681405967184}
{"stream": "pages", "data": {"uuid": "8ba00398-80cf-4a17-afba-2cf223244cfe", "property_id": "314186564", "date": "20230403", "hostName": "airbyte.com", "pagePathPlusQueryString": "/", "screenPageViews": 1092, "bounceRate": 0.5569070373588184}, "emitted_at": 1681405967184}
{"stream": "pages", "data": {"uuid": "0d62e64a-4da9-46b8-8e90-e1180ea9b1ab", "property_id": "314186564", "date": "20230412", "hostName": "airbyte.com", "pagePathPlusQueryString": "/", "screenPageViews": 1089, "bounceRate": 0.5690515806988353}, "emitted_at": 1681405967184}
{"stream": "pages", "data": {"uuid": "d5a7b5bf-b8a9-4f29-8a06-0e4bcfee17a7", "property_id": "314186564", "date": "20230406", "hostName": "airbyte.com", "pagePathPlusQueryString": "/", "screenPageViews": 1005, "bounceRate": 0.5516279069767441}, "emitted_at": 1681405967185}
{"stream": "pages", "data": {"uuid": "e77044bf-1cd6-4ef0-a475-ff6255ac1cf8", "property_id": "314186564", "date": "20230407", "hostName": "airbyte.com", "pagePathPlusQueryString": "/", "screenPageViews": 734, "bounceRate": 0.571619812583668}, "emitted_at": 1681405967185}
{"stream": "pages", "data": {"uuid": "2c0db575-cf8a-4cca-955d-baa2032bec54", "property_id": "314186564", "date": "20230403", "hostName": "airbyte.com", "pagePathPlusQueryString": "/blog/data-modeling-unsung-hero-data-engineering-introduction", "screenPageViews": 541, "bounceRate": 0.7192691029900332}, "emitted_at": 1681405967185}
{"stream": "pages", "data": {"uuid": "4d4ea3e6-91ac-4da7-baf6-4e8cd6665b8c", "property_id": "314186564", "date": "20230402", "hostName": "airbyte.com", "pagePathPlusQueryString": "/", "screenPageViews": 529, "bounceRate": 0.5614678899082569}, "emitted_at": 1681405967185}
{"stream": "website_overview", "data": {"uuid": "3175b7c6-8257-4111-b15a-5c97e16437fe", "property_id": "314186564", "date": "20230406", "totalUsers": 3014, "newUsers": 1539, "sessions": 4257, "sessionsPerUser": 1.661592505854801, "averageSessionDuration": 270.9253856281419, "screenPageViews": 10839, "screenPageViewsPerSession": 2.5461592670894997, "bounceRate": 0.5391120507399577}, "emitted_at": 1681405971634}
{"stream": "website_overview", "data": {"uuid": "c376b15d-d38e-4caf-816d-9dc2b8f04d8f", "property_id": "314186564", "date": "20230403", "totalUsers": 2988, "newUsers": 1461, "sessions": 4350, "sessionsPerUser": 1.725505751685839, "averageSessionDuration": 246.36103450390806, "screenPageViews": 10749, "screenPageViewsPerSession": 2.4710344827586206, "bounceRate": 0.5618390804597702}, "emitted_at": 1681405971634}
{"stream": "website_overview", "data": {"uuid": "28294b7a-85ad-4d73-bd11-20e5041c7de9", "property_id": "314186564", "date": "20230404", "totalUsers": 2817, "newUsers": 1367, "sessions": 4153, "sessionsPerUser": 1.7405699916177704, "averageSessionDuration": 259.69049313965803, "screenPageViews": 10653, "screenPageViewsPerSession": 2.5651336383337346, "bounceRate": 0.5379243920057789}, "emitted_at": 1681405971635}
{"stream": "website_overview", "data": {"uuid": "7cba1969-8d97-4f56-b1c2-07795408ec05", "property_id": "314186564", "date": "20230405", "totalUsers": 2754, "newUsers": 1333, "sessions": 4004, "sessionsPerUser": 1.727351164797239, "averageSessionDuration": 290.08648263536463, "screenPageViews": 10737, "screenPageViewsPerSession": 2.6815684315684316, "bounceRate": 0.5072427572427572}, "emitted_at": 1681405971635}
{"stream": "website_overview", "data": {"uuid": "19b5c257-3ff5-4f76-af0a-e8d54915fcf5", "property_id": "314186564", "date": "20230411", "totalUsers": 2730, "newUsers": 1273, "sessions": 4006, "sessionsPerUser": 1.7820284697508897, "averageSessionDuration": 256.8832527284074, "screenPageViews": 10073, "screenPageViewsPerSession": 2.514478282576136, "bounceRate": 0.5162256615077384}, "emitted_at": 1681405971635}
{"stream": "website_overview", "data": {"uuid": "14c67449-e3d5-4e59-af8e-8fc8dd3880db", "property_id": "314186564", "date": "20230412", "totalUsers": 2642, "newUsers": 1215, "sessions": 3940, "sessionsPerUser": 1.820702402957486, "averageSessionDuration": 281.3629124893401, "screenPageViews": 10621, "screenPageViewsPerSession": 2.6956852791878174, "bounceRate": 0.5309644670050762}, "emitted_at": 1681405971635}
{"stream": "website_overview", "data": {"uuid": "ffa23a73-b5b1-41f6-ae2e-b27a4b6e020e", "property_id": "314186564", "date": "20230410", "totalUsers": 2409, "newUsers": 1173, "sessions": 3602, "sessionsPerUser": 1.7822859970311726, "averageSessionDuration": 252.51497996779568, "screenPageViews": 8973, "screenPageViewsPerSession": 2.491116046640755, "bounceRate": 0.524153248195447}, "emitted_at": 1681405971635}
{"stream": "website_overview", "data": {"uuid": "e42b5011-0d04-4e34-8b40-03b463f8537c", "property_id": "314186564", "date": "20230407", "totalUsers": 1950, "newUsers": 974, "sessions": 2710, "sessionsPerUser": 1.6646191646191646, "averageSessionDuration": 261.6388968815498, "screenPageViews": 6972, "screenPageViewsPerSession": 2.572693726937269, "bounceRate": 0.5431734317343173}, "emitted_at": 1681405971635}
{"stream": "website_overview", "data": {"uuid": "93775261-4ea3-4fb4-babb-4110b6843fbb", "property_id": "314186564", "date": "20230409", "totalUsers": 1277, "newUsers": 664, "sessions": 1661, "sessionsPerUser": 1.6461843409316155, "averageSessionDuration": 199.5610062384106, "screenPageViews": 3300, "screenPageViewsPerSession": 1.9867549668874172, "bounceRate": 0.5605057194461168}, "emitted_at": 1681405971635}
{"stream": "website_overview", "data": {"uuid": "4df90187-80fa-433e-8c99-bfa28764c5a3", "property_id": "314186564", "date": "20230402", "totalUsers": 1185, "newUsers": 605, "sessions": 1505, "sessionsPerUser": 1.5388548057259714, "averageSessionDuration": 221.2044838358804, "screenPageViews": 3260, "screenPageViewsPerSession": 2.166112956810631, "bounceRate": 0.5348837209302325}, "emitted_at": 1681405971636}
{"stream": "cohort_report", "data": {"uuid": "f75938fd-3274-4807-8c37-ce7dd0c3a790", "property_id": "314186564", "cohort": "cohort_0", "cohortNthDay": "0000", "cohortActiveUsers": 731}, "emitted_at": 1681405973101}
{"stream": "cohort_report", "data": {"uuid": "8f228a9c-57b8-4329-9de7-f5fb56ee18b6", "property_id": "314186564", "cohort": "cohort_0", "cohortNthDay": "0001", "cohortActiveUsers": 25}, "emitted_at": 1681405973101}
{"stream": "cohort_report", "data": {"uuid": "f79f8d4d-4166-4fcd-9ca0-0704c3b9af13", "property_id": "314186564", "cohort": "cohort_0", "cohortNthDay": "0002", "cohortActiveUsers": 9}, "emitted_at": 1681405973101}
{"stream": "cohort_report", "data": {"uuid": "d47f77ef-5e72-4c66-9360-cee92352deaf", "property_id": "314186564", "cohort": "cohort_0", "cohortNthDay": "0003", "cohortActiveUsers": 6}, "emitted_at": 1681405973101}
{"stream": "cohort_report", "data": {"uuid": "d90efbfd-b320-45ea-9b51-672fa86d01bb", "property_id": "314186564", "cohort": "cohort_0", "cohortNthDay": "0004", "cohortActiveUsers": 4}, "emitted_at": 1681405973101}
{"stream": "cohort_report", "data": {"uuid": "8b2ded37-ede8-4f27-932f-b6b357c363a8", "property_id": "314186564", "cohort": "cohort_0", "cohortNthDay": "0009", "cohortActiveUsers": 4}, "emitted_at": 1681405973102}
{"stream": "cohort_report", "data": {"uuid": "6e8e19a2-a5cb-4a82-b800-23f38a0f5268", "property_id": "314186564", "cohort": "cohort_0", "cohortNthDay": "0010", "cohortActiveUsers": 4}, "emitted_at": 1681405973102}
{"stream": "cohort_report", "data": {"uuid": "38fc20ce-d8f1-459c-b880-44851128a923", "property_id": "314186564", "cohort": "cohort_0", "cohortNthDay": "0011", "cohortActiveUsers": 4}, "emitted_at": 1681405973102}
{"stream": "cohort_report", "data": {"uuid": "8a77d306-0c5d-431c-b2a7-6b2c87383719", "property_id": "314186564", "cohort": "cohort_0", "cohortNthDay": "0013", "cohortActiveUsers": 4}, "emitted_at": 1681405973102}
{"stream": "cohort_report", "data": {"uuid": "105f7916-0d45-4e17-98db-c6ea3e7d5223", "property_id": "314186564", "cohort": "cohort_0", "cohortNthDay": "0025", "cohortActiveUsers": 4}, "emitted_at": 1681405973102}
{"stream": "pivot_report", "data": {"uuid": "a220f05f-5185-41b7-9730-379875cc8866", "property_id": "314186564", "browser": "Chrome", "country": "United States", "language": "English", "sessions": 10441}, "emitted_at": 1681405974686}
{"stream": "pivot_report", "data": {"uuid": "d01ac2b1-8947-4231-89f6-822fc7b08a01", "property_id": "314186564", "browser": "Chrome", "country": "India", "language": "English", "sessions": 4170}, "emitted_at": 1681405974686}
{"stream": "pivot_report", "data": {"uuid": "cd9027aa-8226-418c-9b60-9d6856327a93", "property_id": "314186564", "browser": "Safari", "country": "United States", "language": "English", "sessions": 1798}, "emitted_at": 1681405974686}
{"stream": "pivot_report", "data": {"uuid": "79a69e5c-0ceb-45e7-b789-5a4904ce273d", "property_id": "314186564", "browser": "Chrome", "country": "Canada", "language": "English", "sessions": 961}, "emitted_at": 1681405974686}
{"stream": "pivot_report", "data": {"uuid": "c9d18f96-3762-47e0-b322-72ad7953c645", "property_id": "314186564", "browser": "Chrome", "country": "United Kingdom", "language": "English", "sessions": 740}, "emitted_at": 1681405974686}
{"stream": "pivot_report", "data": {"uuid": "5688b831-a3be-4b5a-ae0b-f464b8098de4", "property_id": "314186564", "browser": "Chrome", "country": "Australia", "language": "English", "sessions": 621}, "emitted_at": 1681405974687}
{"stream": "pivot_report", "data": {"uuid": "fac29ad5-6fcb-42ef-ac2f-31b782c391cd", "property_id": "314186564", "browser": "Chrome", "country": "Brazil", "language": "Portuguese", "sessions": 566}, "emitted_at": 1681405974687}
{"stream": "pivot_report", "data": {"uuid": "a4e75b7f-034f-4511-8491-70c801950e93", "property_id": "314186564", "browser": "Chrome", "country": "Vietnam", "language": "English", "sessions": 565}, "emitted_at": 1681405974687}
{"stream": "pivot_report", "data": {"uuid": "ad237798-ae12-46bd-892d-0d8b28acca8e", "property_id": "314186564", "browser": "Edge", "country": "United States", "language": "English", "sessions": 555}, "emitted_at": 1681405974687}
{"stream": "pivot_report", "data": {"uuid": "8a06cd72-4789-4f36-bf5b-f2c860e1aece", "property_id": "314186564", "browser": "Chrome", "country": "Pakistan", "language": "English", "sessions": 522}, "emitted_at": 1681405974687}
{"stream":"daily_active_users","data":{"property_id":"314186564","date":"20230406","active1DayUsers":2562},"emitted_at":1681405954033}
{"stream":"daily_active_users","data":{"property_id":"314186564","date":"20230403","active1DayUsers":2521},"emitted_at":1681405954034}
{"stream":"daily_active_users","data":{"property_id":"314186564","date":"20230404","active1DayUsers":2386},"emitted_at":1681405954034}
{"stream":"daily_active_users","data":{"property_id":"314186564","date":"20230405","active1DayUsers":2318},"emitted_at":1681405954035}
{"stream":"daily_active_users","data":{"property_id":"314186564","date":"20230411","active1DayUsers":2248},"emitted_at":1681405954035}
{"stream":"daily_active_users","data":{"property_id":"314186564","date":"20230412","active1DayUsers":2164},"emitted_at":1681405954036}
{"stream":"daily_active_users","data":{"property_id":"314186564","date":"20230410","active1DayUsers":2021},"emitted_at":1681405954036}
{"stream":"daily_active_users","data":{"property_id":"314186564","date":"20230407","active1DayUsers":1628},"emitted_at":1681405954037}
{"stream":"daily_active_users","data":{"property_id":"314186564","date":"20230409","active1DayUsers":1009},"emitted_at":1681405954037}
{"stream":"daily_active_users","data":{"property_id":"314186564","date":"20230402","active1DayUsers":978},"emitted_at":1681405954038}
{"stream":"weekly_active_users","data":{"property_id":"314186564","date":"20230403","active7DayUsers":11840},"emitted_at":1681405954684}
{"stream":"weekly_active_users","data":{"property_id":"314186564","date":"20230406","active7DayUsers":11828},"emitted_at":1681405954685}
{"stream":"weekly_active_users","data":{"property_id":"314186564","date":"20230404","active7DayUsers":11812},"emitted_at":1681405954685}
{"stream":"weekly_active_users","data":{"property_id":"314186564","date":"20230405","active7DayUsers":11751},"emitted_at":1681405954685}
{"stream":"weekly_active_users","data":{"property_id":"314186564","date":"20230408","active7DayUsers":11745},"emitted_at":1681405954685}
{"stream":"weekly_active_users","data":{"property_id":"314186564","date":"20230409","active7DayUsers":11739},"emitted_at":1681405954685}
{"stream":"weekly_active_users","data":{"property_id":"314186564","date":"20230407","active7DayUsers":11637},"emitted_at":1681405954685}
{"stream":"weekly_active_users","data":{"property_id":"314186564","date":"20230401","active7DayUsers":11547},"emitted_at":1681405954685}
{"stream":"weekly_active_users","data":{"property_id":"314186564","date":"20230402","active7DayUsers":11521},"emitted_at":1681405954685}
{"stream":"weekly_active_users","data":{"property_id":"314186564","date":"20230410","active7DayUsers":11369},"emitted_at":1681405954686}
{"stream":"four_weekly_active_users","data":{"property_id":"314186564","date":"20230401","active28DayUsers":48082},"emitted_at":1681405955854}
{"stream":"four_weekly_active_users","data":{"property_id":"314186564","date":"20230402","active28DayUsers":47927},"emitted_at":1681405955854}
{"stream":"four_weekly_active_users","data":{"property_id":"314186564","date":"20230403","active28DayUsers":44678},"emitted_at":1681405955854}
{"stream":"four_weekly_active_users","data":{"property_id":"314186564","date":"20230404","active28DayUsers":42997},"emitted_at":1681405955854}
{"stream":"four_weekly_active_users","data":{"property_id":"314186564","date":"20230405","active28DayUsers":42219},"emitted_at":1681405955855}
{"stream":"four_weekly_active_users","data":{"property_id":"314186564","date":"20230406","active28DayUsers":42028},"emitted_at":1681405955855}
{"stream":"four_weekly_active_users","data":{"property_id":"314186564","date":"20230407","active28DayUsers":41851},"emitted_at":1681405955855}
{"stream":"four_weekly_active_users","data":{"property_id":"314186564","date":"20230408","active28DayUsers":41775},"emitted_at":1681405955855}
{"stream":"four_weekly_active_users","data":{"property_id":"314186564","date":"20230409","active28DayUsers":41717},"emitted_at":1681405955855}
{"stream":"four_weekly_active_users","data":{"property_id":"314186564","date":"20230410","active28DayUsers":41212},"emitted_at":1681405955855}
{"stream":"devices","data":{"property_id":"314186564","date":"20230411","deviceCategory":"desktop","operatingSystem":"Macintosh","browser":"Chrome","totalUsers":973,"newUsers":368,"sessions":1667,"sessionsPerUser":2.110126582278481,"averageSessionDuration":308.8923676994601,"screenPageViews":5367,"screenPageViewsPerSession":3.2195560887822436,"bounceRate":0.498500299940012},"emitted_at":1681405958296}
{"stream":"devices","data":{"property_id":"314186564","date":"20230412","deviceCategory":"desktop","operatingSystem":"Macintosh","browser":"Chrome","totalUsers":969,"newUsers":350,"sessions":1588,"sessionsPerUser":2.0025220680958387,"averageSessionDuration":336.108126070529,"screenPageViews":4726,"screenPageViewsPerSession":2.9760705289672544,"bounceRate":0.5012594458438288},"emitted_at":1681405958296}
{"stream":"devices","data":{"property_id":"314186564","date":"20230404","deviceCategory":"desktop","operatingSystem":"Macintosh","browser":"Chrome","totalUsers":942,"newUsers":352,"sessions":1554,"sessionsPerUser":2.007751937984496,"averageSessionDuration":328.66656451029604,"screenPageViews":5217,"screenPageViewsPerSession":3.357142857142857,"bounceRate":0.4954954954954955},"emitted_at":1681405958296}
{"stream":"devices","data":{"property_id":"314186564","date":"20230406","deviceCategory":"desktop","operatingSystem":"Macintosh","browser":"Chrome","totalUsers":942,"newUsers":389,"sessions":1551,"sessionsPerUser":1.9783163265306123,"averageSessionDuration":357.5382107272727,"screenPageViews":5102,"screenPageViewsPerSession":3.289490651192779,"bounceRate":0.49258542875564154},"emitted_at":1681405958297}
{"stream":"devices","data":{"property_id":"314186564","date":"20230403","deviceCategory":"desktop","operatingSystem":"Macintosh","browser":"Chrome","totalUsers":929,"newUsers":341,"sessions":1546,"sessionsPerUser":2.0558510638297873,"averageSessionDuration":315.4776974385511,"screenPageViews":5116,"screenPageViewsPerSession":3.309184993531695,"bounceRate":0.5071151358344114},"emitted_at":1681405958297}
{"stream":"devices","data":{"property_id":"314186564","date":"20230405","deviceCategory":"desktop","operatingSystem":"Macintosh","browser":"Chrome","totalUsers":926,"newUsers":363,"sessions":1573,"sessionsPerUser":2.0428571428571427,"averageSessionDuration":346.09502719898285,"screenPageViews":5032,"screenPageViewsPerSession":3.1989828353464715,"bounceRate":0.4869675778766688},"emitted_at":1681405958297}
{"stream":"devices","data":{"property_id":"314186564","date":"20230410","deviceCategory":"desktop","operatingSystem":"Macintosh","browser":"Chrome","totalUsers":920,"newUsers":374,"sessions":1524,"sessionsPerUser":2.0456375838926175,"averageSessionDuration":255.77025801837266,"screenPageViews":4025,"screenPageViewsPerSession":2.641076115485564,"bounceRate":0.5255905511811023},"emitted_at":1681405958297}
{"stream":"devices","data":{"property_id":"314186564","date":"20230403","deviceCategory":"desktop","operatingSystem":"Windows","browser":"Chrome","totalUsers":781,"newUsers":366,"sessions":1184,"sessionsPerUser":1.8528951486697967,"averageSessionDuration":278.84846059881755,"screenPageViews":2993,"screenPageViewsPerSession":2.5278716216216215,"bounceRate":0.5616554054054054},"emitted_at":1681405958297}
{"stream":"devices","data":{"property_id":"314186564","date":"20230411","deviceCategory":"desktop","operatingSystem":"Windows","browser":"Chrome","totalUsers":760,"newUsers":365,"sessions":1155,"sessionsPerUser":1.896551724137931,"averageSessionDuration":264.1307251896104,"screenPageViews":2452,"screenPageViewsPerSession":2.122943722943723,"bounceRate":0.5316017316017316},"emitted_at":1681405958298}
{"stream":"devices","data":{"property_id":"314186564","date":"20230404","deviceCategory":"desktop","operatingSystem":"Windows","browser":"Chrome","totalUsers":727,"newUsers":345,"sessions":1137,"sessionsPerUser":1.8517915309446253,"averageSessionDuration":252.06245670272648,"screenPageViews":2601,"screenPageViewsPerSession":2.287598944591029,"bounceRate":0.5488126649076517},"emitted_at":1681405958298}
{"stream":"locations","data":{"property_id":"314186564","region":"New York","country":"United States","city":"New York","date":"20230406","totalUsers":108,"newUsers":62,"sessions":157,"sessionsPerUser":1.6354166666666667,"averageSessionDuration":435.44268001273895,"screenPageViews":534,"screenPageViewsPerSession":3.4012738853503186,"bounceRate":0.5031847133757962},"emitted_at":1681405962136}
{"stream":"locations","data":{"property_id":"314186564","region":"New York","country":"United States","city":"New York","date":"20230405","totalUsers":95,"newUsers":54,"sessions":123,"sessionsPerUser":1.5769230769230769,"averageSessionDuration":499.2074986666667,"screenPageViews":481,"screenPageViewsPerSession":3.910569105691057,"bounceRate":0.44715447154471544},"emitted_at":1681405962136}
{"stream":"locations","data":{"property_id":"314186564","region":"New York","country":"United States","city":"New York","date":"20230403","totalUsers":94,"newUsers":46,"sessions":126,"sessionsPerUser":1.68,"averageSessionDuration":424.00281903174607,"screenPageViews":499,"screenPageViewsPerSession":3.9603174603174605,"bounceRate":0.5238095238095238},"emitted_at":1681405962136}
{"stream":"locations","data":{"property_id":"314186564","region":"New York","country":"United States","city":"New York","date":"20230404","totalUsers":85,"newUsers":47,"sessions":121,"sessionsPerUser":1.6575342465753424,"averageSessionDuration":378.81275640495863,"screenPageViews":434,"screenPageViewsPerSession":3.5867768595041323,"bounceRate":0.48760330578512395},"emitted_at":1681405962136}
{"stream":"locations","data":{"property_id":"314186564","region":"New York","country":"United States","city":"New York","date":"20230412","totalUsers":85,"newUsers":49,"sessions":131,"sessionsPerUser":1.8194444444444444,"averageSessionDuration":379.1322029236641,"screenPageViews":391,"screenPageViewsPerSession":2.984732824427481,"bounceRate":0.5267175572519084},"emitted_at":1681405962137}
{"stream":"locations","data":{"property_id":"314186564","region":"New York","country":"United States","city":"New York","date":"20230410","totalUsers":81,"newUsers":42,"sessions":135,"sessionsPerUser":1.9565217391304348,"averageSessionDuration":303.13140742962963,"screenPageViews":376,"screenPageViewsPerSession":2.785185185185185,"bounceRate":0.5407407407407407},"emitted_at":1681405962137}
{"stream":"locations","data":{"property_id":"314186564","region":"New York","country":"United States","city":"New York","date":"20230411","totalUsers":81,"newUsers":38,"sessions":123,"sessionsPerUser":1.9523809523809523,"averageSessionDuration":362.51537134146344,"screenPageViews":312,"screenPageViewsPerSession":2.5365853658536586,"bounceRate":0.5934959349593496},"emitted_at":1681405962137}
{"stream":"locations","data":{"property_id":"314186564","region":"Karnataka","country":"India","city":"Bengaluru","date":"20230411","totalUsers":76,"newUsers":52,"sessions":123,"sessionsPerUser":1.8636363636363635,"averageSessionDuration":203.00314456910567,"screenPageViews":261,"screenPageViewsPerSession":2.1219512195121952,"bounceRate":0.4959349593495935},"emitted_at":1681405962137}
{"stream":"locations","data":{"property_id":"314186564","region":"Karnataka","country":"India","city":"Bengaluru","date":"20230403","totalUsers":69,"newUsers":34,"sessions":102,"sessionsPerUser":1.728813559322034,"averageSessionDuration":256.4942830490196,"screenPageViews":216,"screenPageViewsPerSession":2.1176470588235294,"bounceRate":0.5490196078431373},"emitted_at":1681405962137}
{"stream":"locations","data":{"property_id":"314186564","region":"New York","country":"United States","city":"New York","date":"20230407","totalUsers":69,"newUsers":30,"sessions":98,"sessionsPerUser":1.849056603773585,"averageSessionDuration":489.54009168367344,"screenPageViews":376,"screenPageViewsPerSession":3.836734693877551,"bounceRate":0.4489795918367347},"emitted_at":1681405962137}
{"stream":"pages","data":{"property_id":"314186564","date":"20230405","hostName":"airbyte.com","pagePathPlusQueryString":"/","screenPageViews":1190,"bounceRate":0.5516393442622951},"emitted_at":1681405967183}
{"stream":"pages","data":{"property_id":"314186564","date":"20230411","hostName":"airbyte.com","pagePathPlusQueryString":"/","screenPageViews":1151,"bounceRate":0.5400641025641025},"emitted_at":1681405967184}
{"stream":"pages","data":{"property_id":"314186564","date":"20230404","hostName":"airbyte.com","pagePathPlusQueryString":"/","screenPageViews":1137,"bounceRate":0.5617232808616405},"emitted_at":1681405967184}
{"stream":"pages","data":{"property_id":"314186564","date":"20230410","hostName":"airbyte.com","pagePathPlusQueryString":"/","screenPageViews":1099,"bounceRate":0.5416666666666666},"emitted_at":1681405967184}
{"stream":"pages","data":{"property_id":"314186564","date":"20230403","hostName":"airbyte.com","pagePathPlusQueryString":"/","screenPageViews":1092,"bounceRate":0.5569070373588184},"emitted_at":1681405967184}
{"stream":"pages","data":{"property_id":"314186564","date":"20230412","hostName":"airbyte.com","pagePathPlusQueryString":"/","screenPageViews":1089,"bounceRate":0.5690515806988353},"emitted_at":1681405967184}
{"stream":"pages","data":{"property_id":"314186564","date":"20230406","hostName":"airbyte.com","pagePathPlusQueryString":"/","screenPageViews":1005,"bounceRate":0.5516279069767441},"emitted_at":1681405967185}
{"stream":"pages","data":{"property_id":"314186564","date":"20230407","hostName":"airbyte.com","pagePathPlusQueryString":"/","screenPageViews":734,"bounceRate":0.571619812583668},"emitted_at":1681405967185}
{"stream":"pages","data":{"property_id":"314186564","date":"20230403","hostName":"airbyte.com","pagePathPlusQueryString":"/blog/data-modeling-unsung-hero-data-engineering-introduction","screenPageViews":541,"bounceRate":0.7192691029900332},"emitted_at":1681405967185}
{"stream":"pages","data":{"property_id":"314186564","date":"20230402","hostName":"airbyte.com","pagePathPlusQueryString":"/","screenPageViews":529,"bounceRate":0.5614678899082569},"emitted_at":1681405967185}
{"stream":"website_overview","data":{"property_id":"314186564","date":"20230406","totalUsers":3014,"newUsers":1539,"sessions":4257,"sessionsPerUser":1.661592505854801,"averageSessionDuration":270.9253856281419,"screenPageViews":10839,"screenPageViewsPerSession":2.5461592670894997,"bounceRate":0.5391120507399577},"emitted_at":1681405971634}
{"stream":"website_overview","data":{"property_id":"314186564","date":"20230403","totalUsers":2988,"newUsers":1461,"sessions":4350,"sessionsPerUser":1.725505751685839,"averageSessionDuration":246.36103450390806,"screenPageViews":10749,"screenPageViewsPerSession":2.4710344827586206,"bounceRate":0.5618390804597702},"emitted_at":1681405971634}
{"stream":"website_overview","data":{"property_id":"314186564","date":"20230404","totalUsers":2817,"newUsers":1367,"sessions":4153,"sessionsPerUser":1.7405699916177704,"averageSessionDuration":259.69049313965803,"screenPageViews":10653,"screenPageViewsPerSession":2.5651336383337346,"bounceRate":0.5379243920057789},"emitted_at":1681405971635}
{"stream":"website_overview","data":{"property_id":"314186564","date":"20230405","totalUsers":2754,"newUsers":1333,"sessions":4004,"sessionsPerUser":1.727351164797239,"averageSessionDuration":290.08648263536463,"screenPageViews":10737,"screenPageViewsPerSession":2.6815684315684316,"bounceRate":0.5072427572427572},"emitted_at":1681405971635}
{"stream":"website_overview","data":{"property_id":"314186564","date":"20230411","totalUsers":2730,"newUsers":1273,"sessions":4006,"sessionsPerUser":1.7820284697508897,"averageSessionDuration":256.8832527284074,"screenPageViews":10073,"screenPageViewsPerSession":2.514478282576136,"bounceRate":0.5162256615077384},"emitted_at":1681405971635}
{"stream":"website_overview","data":{"property_id":"314186564","date":"20230412","totalUsers":2642,"newUsers":1215,"sessions":3940,"sessionsPerUser":1.820702402957486,"averageSessionDuration":281.3629124893401,"screenPageViews":10621,"screenPageViewsPerSession":2.6956852791878174,"bounceRate":0.5309644670050762},"emitted_at":1681405971635}
{"stream":"website_overview","data":{"property_id":"314186564","date":"20230410","totalUsers":2409,"newUsers":1173,"sessions":3602,"sessionsPerUser":1.7822859970311726,"averageSessionDuration":252.51497996779568,"screenPageViews":8973,"screenPageViewsPerSession":2.491116046640755,"bounceRate":0.524153248195447},"emitted_at":1681405971635}
{"stream":"website_overview","data":{"property_id":"314186564","date":"20230407","totalUsers":1950,"newUsers":974,"sessions":2710,"sessionsPerUser":1.6646191646191646,"averageSessionDuration":261.6388968815498,"screenPageViews":6972,"screenPageViewsPerSession":2.572693726937269,"bounceRate":0.5431734317343173},"emitted_at":1681405971635}
{"stream":"website_overview","data":{"property_id":"314186564","date":"20230409","totalUsers":1277,"newUsers":664,"sessions":1661,"sessionsPerUser":1.6461843409316155,"averageSessionDuration":199.5610062384106,"screenPageViews":3300,"screenPageViewsPerSession":1.9867549668874172,"bounceRate":0.5605057194461168},"emitted_at":1681405971635}
{"stream":"website_overview","data":{"property_id":"314186564","date":"20230402","totalUsers":1185,"newUsers":605,"sessions":1505,"sessionsPerUser":1.5388548057259714,"averageSessionDuration":221.2044838358804,"screenPageViews":3260,"screenPageViewsPerSession":2.166112956810631,"bounceRate":0.5348837209302325},"emitted_at":1681405971636}
{"stream":"cohort_report","data":{"property_id":"314186564","cohort":"cohort_0","cohortNthDay":"0000","cohortActiveUsers":731},"emitted_at":1681405973101}
{"stream":"cohort_report","data":{"property_id":"314186564","cohort":"cohort_0","cohortNthDay":"0001","cohortActiveUsers":25},"emitted_at":1681405973101}
{"stream":"cohort_report","data":{"property_id":"314186564","cohort":"cohort_0","cohortNthDay":"0002","cohortActiveUsers":9},"emitted_at":1681405973101}
{"stream":"cohort_report","data":{"property_id":"314186564","cohort":"cohort_0","cohortNthDay":"0003","cohortActiveUsers":6},"emitted_at":1681405973101}
{"stream":"cohort_report","data":{"property_id":"314186564","cohort":"cohort_0","cohortNthDay":"0004","cohortActiveUsers":4},"emitted_at":1681405973101}
{"stream":"cohort_report","data":{"property_id":"314186564","cohort":"cohort_0","cohortNthDay":"0009","cohortActiveUsers":4},"emitted_at":1681405973102}
{"stream":"cohort_report","data":{"property_id":"314186564","cohort":"cohort_0","cohortNthDay":"0010","cohortActiveUsers":4},"emitted_at":1681405973102}
{"stream":"cohort_report","data":{"property_id":"314186564","cohort":"cohort_0","cohortNthDay":"0011","cohortActiveUsers":4},"emitted_at":1681405973102}
{"stream":"cohort_report","data":{"property_id":"314186564","cohort":"cohort_0","cohortNthDay":"0013","cohortActiveUsers":4},"emitted_at":1681405973102}
{"stream":"cohort_report","data":{"property_id":"314186564","cohort":"cohort_0","cohortNthDay":"0025","cohortActiveUsers":4},"emitted_at":1681405973102}
{"stream":"pivot_report","data":{"property_id":"314186564","browser":"Chrome","country":"United States","language":"English","sessions":24293,"startDate":"2023-04-01","endDate":"2023-04-30"},"emitted_at":1685012261616}
{"stream":"pivot_report","data":{"property_id":"314186564","browser":"Chrome","country":"India","language":"English","sessions":9419,"startDate":"2023-04-01","endDate":"2023-04-30"},"emitted_at":1685012261618}
{"stream":"pivot_report","data":{"property_id":"314186564","browser":"Safari","country":"United States","language":"English","sessions":3863,"startDate":"2023-04-01","endDate":"2023-04-30"},"emitted_at":1685012261618}
{"stream":"pivot_report","data":{"property_id":"314186564","browser":"Chrome","country":"Canada","language":"English","sessions":2560,"startDate":"2023-04-01","endDate":"2023-04-30"},"emitted_at":1685012261618}
{"stream":"pivot_report","data":{"property_id":"314186564","browser":"Chrome","country":"United Kingdom","language":"English","sessions":1964,"startDate":"2023-04-01","endDate":"2023-04-30"},"emitted_at":1685012261618}
{"stream":"pivot_report","data":{"property_id":"314186564","browser":"Edge","country":"United States","language":"English","sessions":1351,"startDate":"2023-04-01","endDate":"2023-04-30"},"emitted_at":1685012261619}
{"stream":"pivot_report","data":{"property_id":"314186564","browser":"Chrome","country":"Australia","language":"English","sessions":1307,"startDate":"2023-04-01","endDate":"2023-04-30"},"emitted_at":1685012261619}
{"stream":"pivot_report","data":{"property_id":"314186564","browser":"Chrome","country":"Brazil","language":"Portuguese","sessions":1302,"startDate":"2023-04-01","endDate":"2023-04-30"},"emitted_at":1685012261619}
{"stream":"pivot_report","data":{"property_id":"314186564","browser":"Chrome","country":"Vietnam","language":"English","sessions":1196,"startDate":"2023-04-01","endDate":"2023-04-30"},"emitted_at":1685012261619}
{"stream":"pivot_report","data":{"property_id":"314186564","browser":"Chrome","country":"Germany","language":"English","sessions":963,"startDate":"2023-04-01","endDate":"2023-04-30"},"emitted_at":1685012261619}

100
airbyte-integrations/connectors/source-google-analytics-data-api/integration_tests/incremental_catalog.json
View File

@@ -1,100 +0,0 @@
{
"streams": [
{
"stream": {
"name": "daily_active_users",
"json_schema": {},
"supported_sync_modes": ["full_refresh", "incremental"],
"source_defined_cursor": true,
"default_cursor_field": ["date"],
"source_defined_primary_key": [["uuid"]]
},
"sync_mode": "incremental",
"destination_sync_mode": "overwrite"
},
{
"stream": {
"name": "weekly_active_users",
"json_schema": {},
"supported_sync_modes": ["full_refresh", "incremental"],
"source_defined_cursor": true,
"default_cursor_field": ["date"],
"source_defined_primary_key": [["uuid"]]
},
"sync_mode": "incremental",
"destination_sync_mode": "overwrite"
},
{
"stream": {
"name": "four_weekly_active_users",
"json_schema": {},
"supported_sync_modes": ["full_refresh", "incremental"],
"source_defined_cursor": true,
"default_cursor_field": ["date"],
"source_defined_primary_key": [["uuid"]]
},
"sync_mode": "incremental",
"destination_sync_mode": "overwrite"
},
{
"stream": {
"name": "devices",
"json_schema": {},
"supported_sync_modes": ["full_refresh", "incremental"],
"source_defined_cursor": true,
"default_cursor_field": ["date"],
"source_defined_primary_key": [["uuid"]]
},
"sync_mode": "incremental",
"destination_sync_mode": "overwrite"
},
{
"stream": {
"name": "locations",
"json_schema": {},
"supported_sync_modes": ["full_refresh", "incremental"],
"source_defined_cursor": true,
"default_cursor_field": ["date"],
"source_defined_primary_key": [["uuid"]]
},
"sync_mode": "incremental",
"destination_sync_mode": "overwrite"
},
{
"stream": {
"name": "pages",
"json_schema": {},
"supported_sync_modes": ["full_refresh", "incremental"],
"source_defined_cursor": true,
"default_cursor_field": ["date"],
"source_defined_primary_key": [["uuid"]]
},
"sync_mode": "incremental",
"destination_sync_mode": "overwrite"
},
{
"stream": {
"name": "traffic_sources",
"json_schema": {},
"supported_sync_modes": ["full_refresh", "incremental"],
"source_defined_cursor": true,
"default_cursor_field": ["date"],
"source_defined_primary_key": [["uuid"]]
},
"sync_mode": "incremental",
"destination_sync_mode": "overwrite"
},
{
"stream": {
"name": "website_overview",
"json_schema": {},
"supported_sync_modes": ["full_refresh", "incremental"],
"source_defined_cursor": true,
"default_cursor_field": ["date"],
"source_defined_primary_key": [["uuid"]]
},
"sync_mode": "incremental",
"destination_sync_mode": "overwrite"
}
]
}

2
airbyte-integrations/connectors/source-google-analytics-data-api/metadata.yaml
View File

@@ -7,7 +7,7 @@ data:
connectorSubtype: api
connectorType: source
definitionId: 3cc2eafd-84aa-4dca-93af-322d9dfeec1a
dockerImageTag: 0.2.7
dockerImageTag: 1.0.0
dockerRepository: airbyte/source-google-analytics-data-api
githubIssueLabel: source-google-analytics-data-api
icon: google-analytics.svg

58
airbyte-integrations/connectors/source-google-analytics-data-api/source_google_analytics_data_api/source.py
View File

@@ -30,6 +30,7 @@ from .utils import (
check_no_property_error,
get_dimensions_type,
get_metrics_type,
get_source_defined_primary_key,
metrics_type_to_python,
)
@@ -37,6 +38,7 @@ from .utils import (
# the initial values should be saved once and tracked for each stream, inclusivelly.
GoogleAnalyticsQuotaHandler: GoogleAnalyticsApiQuota = GoogleAnalyticsApiQuota()
LOOKBACK_WINDOW = datetime.timedelta(days=2)
# set page_size to 100000 due to determination of maximum limit value in official documentation
# https://developers.google.com/analytics/devguides/reporting/data/v1/basics#pagination
PAGE_SIZE = 100000
@@ -72,6 +74,7 @@ class GoogleAnalyticsDataApiAbstractStream(HttpStream, ABC):
def __init__(self, *, config: Mapping[str, Any], **kwargs):
super().__init__(**kwargs)
self._config = config
self._source_defined_primary_key = get_source_defined_primary_key(self.name)
@property
def config(self):
@@ -101,22 +104,21 @@ class GoogleAnalyticsDataApiBaseStream(GoogleAnalyticsDataApiAbstractStream):
"""
_record_date_format = "%Y%m%d"
primary_key = "uuid"
offset = 0
metadata = MetadataDescriptor()
@property
def cursor_field(self) -> Optional[str]:
return "date" if "date" in self.config.get("dimensions", {}) else []
return "date" if "date" in self.config.get("dimensions", []) else []
@staticmethod
def add_primary_key() -> dict:
return {"uuid": str(uuid.uuid4())}
@staticmethod
def add_property_id(property_id):
return {"property_id": property_id}
@property
def primary_key(self):
pk = ["property_id"] + self.config.get("dimensions", [])
if "cohort_spec" not in self.config and "date" not in pk:
pk.append("startDate")
pk.append("endDate")
return pk
@staticmethod
def add_dimensions(dimensions, row) -> dict:
@@ -141,7 +143,6 @@ class GoogleAnalyticsDataApiBaseStream(GoogleAnalyticsDataApiAbstractStream):
"additionalProperties": True,
"properties": {
"property_id": {"type": ["string"]},
"uuid": {"type": ["string"], "description": "Custom unique identifier for each record, to support primary key"},
},
}
@@ -151,6 +152,14 @@ class GoogleAnalyticsDataApiBaseStream(GoogleAnalyticsDataApiAbstractStream):
for d in self.config["dimensions"]
}
)
# skipping startDate and endDate fields for cohort stream, because it doesn't support startDate and endDate fields
if "cohort_spec" not in self.config and "date" not in self.config["dimensions"]:
schema["properties"].update(
{
"startDate": {"type": ["null", "string"], "format": "date"},
"endDate": {"type": ["null", "string"], "format": "date"},
}
)
schema["properties"].update(
{
@@ -201,9 +210,24 @@ class GoogleAnalyticsDataApiBaseStream(GoogleAnalyticsDataApiAbstractStream):
metrics_type_map = {h.get("name"): h.get("type") for h in r.get("metricHeaders", [{}])}
for row in r.get("rows", []):
yield self.add_primary_key() | self.add_property_id(self.config["property_id"]) | self.add_dimensions(
dimensions, row
) | self.add_metrics(metrics, metrics_type_map, row)
record = {
"property_id": self.config["property_id"],
**self.add_dimensions(dimensions, row),
**self.add_metrics(metrics, metrics_type_map, row),
}
# https://github.com/airbytehq/airbyte/pull/26283
# We pass the uuid field for synchronizations which still have the old
# configured_catalog with the old primary key. We need it to avoid of removal of rows
# in the deduplication process. As soon as the customer press "refresh source schema"
# this part is no longer needed.
if self._source_defined_primary_key == [["uuid"]]:
record["uuid"] = str(uuid.uuid4())
if "cohort_spec" not in self.config and "date" not in record:
record["startDate"] = stream_slice["startDate"]
record["endDate"] = stream_slice["endDate"]
yield record
def get_updated_state(self, current_stream_state: MutableMapping[str, Any], latest_record: Mapping[str, Any]):
updated_state = utils.string_to_date(latest_record[self.cursor_field], self._record_date_format)
@@ -242,6 +266,7 @@ class GoogleAnalyticsDataApiBaseStream(GoogleAnalyticsDataApiAbstractStream):
start_date = stream_state and stream_state.get(self.cursor_field)
if start_date:
start_date = utils.string_to_date(start_date, self._record_date_format, old_format=DATE_FORMAT)
start_date -= LOOKBACK_WINDOW
start_date = max(start_date, self.config["date_ranges_start_date"])
else:
start_date = self.config["date_ranges_start_date"]
@@ -387,6 +412,7 @@ class SourceGoogleAnalyticsDataApi(AbstractSource):
return False, str(e)
config["authenticator"] = self.get_authenticator(config)
metadata = None
try:
stream = GoogleAnalyticsDataApiMetadataStream(config=config, authenticator=config["authenticator"])
metadata = next(stream.read_records(sync_mode=SyncMode.full_refresh), None)
@@ -435,7 +461,11 @@ class SourceGoogleAnalyticsDataApi(AbstractSource):
def instantiate_report_class(report: dict, config: Mapping[str, Any]) -> GoogleAnalyticsDataApiBaseStream:
cohort_spec = report.get("cohortSpec")
pivots = report.get("pivots")
stream_config = {"metrics": report["metrics"], "dimensions": report["dimensions"], **config}
stream_config = {
"metrics": report["metrics"],
"dimensions": report["dimensions"],
**config,
}
report_class_tuple = (GoogleAnalyticsDataApiBaseStream,)
if pivots:
stream_config["pivots"] = pivots

21
airbyte-integrations/connectors/source-google-analytics-data-api/source_google_analytics_data_api/utils.py
View File

@@ -3,8 +3,11 @@
#
import argparse
import calendar
import datetime
import json
import sys
from typing import Dict
import jsonschema
@@ -114,3 +117,21 @@ def check_invalid_property_error(exc: jsonschema.ValidationError) -> str:
for property in mapper:
if property in exc.schema_path:
return mapper[property]
def get_source_defined_primary_key(stream):
"""
https://github.com/airbytehq/airbyte/pull/26283
It's not a very elegant way to get source_defined_primary_key inside the stream.
It's used only for a smooth transition to the new primary key.
As soon as the transition will complete we can remove this function.
"""
if len(sys.argv) > 1 and "read" == sys.argv[1]:
parser = argparse.ArgumentParser()
subparsers = parser.add_subparsers()
read_subparser = subparsers.add_parser("read")
read_subparser.add_argument("--catalog", type=str, required=True)
args, unknown = parser.parse_known_args()
catalog = json.loads(open(args.catalog).read())
res = {s["stream"]["name"]: s["stream"].get("source_defined_primary_key") for s in catalog["streams"]}
return res.get(stream)

14
airbyte-integrations/connectors/source-google-analytics-data-api/unit_tests/test_source.py
View File

@@ -66,7 +66,7 @@ def config_gen(config):
@pytest.mark.parametrize(
"config_values, status, message",
"config_values, is_successful, message",
[
({}, Status.SUCCEEDED, None),
({"custom_reports": ...}, Status.SUCCEEDED, None),
@@ -95,7 +95,7 @@ def config_gen(config):
Status.FAILED, '"The custom report pivot_report entered contains invalid pivots: {} is not of type \'null\', \'array\'. Ensure the pivot follow the syntax described in the docs (https://developers.google.com/analytics/devguides/reporting/data/v1/rest/v1beta/Pivot)."'),
],
)
def test_check(requests_mock, config_gen, config_values, status, message):
def test_check(requests_mock, config_gen, config_values, is_successful, message):
requests_mock.register_uri("POST", "https://oauth2.googleapis.com/token",
json={"access_token": "access_token", "expires_in": 3600, "token_type": "Bearer"})
@@ -114,11 +114,11 @@ def test_check(requests_mock, config_gen, config_values, status, message):
source = SourceGoogleAnalyticsDataApi()
logger = MagicMock()
assert source.check(logger, config_gen(**config_values)) == AirbyteConnectionStatus(status=status, message=message)
with pytest.raises(AirbyteTracedException) as e:
source.check(logger, config_gen(property_id="UA-11111111"))
assert e.value.failure_type == FailureType.config_error
assert source.check(logger, config_gen(**config_values)) == AirbyteConnectionStatus(status=is_successful, message=message)
if not is_successful:
with pytest.raises(AirbyteTracedException) as e:
source.check(logger, config_gen(property_id="UA-11111111"))
assert e.value.failure_type == FailureType.config_error
def test_streams(mocker, patch_base_class):

23
airbyte-integrations/connectors/source-google-analytics-data-api/unit_tests/test_streams.py
View File

@@ -227,8 +227,6 @@ def test_parse_response(patch_base_class):
response.json.return_value = response_data
inputs = {"response": response, "stream_state": {}}
actual_records: Mapping[str, Any] = list(stream.parse_response(**inputs))
for record in actual_records:
del record["uuid"]
assert actual_records == expected_data
@@ -336,6 +334,19 @@ def test_read_incremental(requests_mock):
"rows": [{"dimensionValues": [{"value": "20230101"}], "metricValues": [{"value": "130"}]}],
"rowCount": 1,
},
# 2-nd incremental read
{
"dimensionHeaders": [{"name": "date"}],
"metricHeaders": [{"name": "totalUsers", "type": "TYPE_INTEGER"}],
"rows": [{"dimensionValues": [{"value": "20221230"}], "metricValues": [{"value": "112"}]}],
"rowCount": 1
},
{
"dimensionHeaders": [{"name": "date"}],
"metricHeaders": [{"name": "totalUsers", "type": "TYPE_INTEGER"}],
"rows": [{"dimensionValues": [{"value": "20221231"}], "metricValues": [{"value": "125"}]}],
"rowCount": 1
},
{
"dimensionHeaders": [{"name": "date"}],
"metricHeaders": [{"name": "totalUsers", "type": "TYPE_INTEGER"}],
@@ -359,9 +370,6 @@ def test_read_incremental(requests_mock):
with freeze_time("2023-01-01 12:00:00"):
records = list(read_incremental(stream, stream_state))
for record in records:
del record["uuid"]
assert records == [
{"date": "20221229", "totalUsers": 100, "property_id": 123},
{"date": "20221230", "totalUsers": 110, "property_id": 123},
@@ -374,10 +382,9 @@ def test_read_incremental(requests_mock):
with freeze_time("2023-01-02 12:00:00"):
records = list(read_incremental(stream, stream_state))
for record in records:
del record["uuid"]
assert records == [
{"date": "20221230", "totalUsers": 112, "property_id": 123},
{"date": "20221231", "totalUsers": 125, "property_id": 123},
{"date": "20230101", "totalUsers": 140, "property_id": 123},
{"date": "20230102", "totalUsers": 150, "property_id": 123},
]