diff --git a/.github/workflows/sync-search-elasticsearch.yml b/.github/workflows/sync-search-elasticsearch.yml index 33ef94c60e..d8b03fe6ed 100644 --- a/.github/workflows/sync-search-elasticsearch.yml +++ b/.github/workflows/sync-search-elasticsearch.yml @@ -107,7 +107,7 @@ jobs: # the whole job fast. # As of June 2023, it takes about 10+ minutes to index one whole # language and we have 8 non-English languages. - max-parallel: 2 + max-parallel: 3 matrix: language: ${{ fromJSON(needs.figureOutMatrix.outputs.matrix) }} steps: @@ -197,7 +197,8 @@ jobs: run: | ./src/search/scripts/index-elasticsearch.js /tmp/records \ --language ${{ matrix.language }} \ - --stagger-seconds 5 + --stagger-seconds 5 \ + --retries 5 - name: Check created indexes and aliases run: | diff --git a/script/helpers/retry-on-error-test.js b/script/helpers/retry-on-error-test.js index 3384ee6598..c41b222b47 100644 --- a/script/helpers/retry-on-error-test.js +++ b/script/helpers/retry-on-error-test.js @@ -17,6 +17,9 @@ // } // const ok = await retry(errorTest, mainFunction, config) // +// Note that, by default, the sleep time is "exponential" by a factor of +// 1.5. So the first sleep will, in the above example, +// be 800ms. Then 1,200ms, Then 1,800ms. etc. // // [end-readme] @@ -25,19 +28,49 @@ const sleep = (ms) => new Promise((resolve) => setTimeout(resolve, ms)) export async function retryOnErrorTest( errorTest, callback, - { attempts = 10, sleepTime = 1000, onError = () => {} } = {}, + { + attempts = 4, + sleepTime = 1000, + exponential = 1.5, + jitterPercent = 25, + onError = () => {}, + } = {}, ) { - while (--attempts) { + while (true) { try { return await callback() } catch (error) { - if (errorTest(error)) { - // console.warn('Sleeping on', error.message, { attempts }) - if (onError) onError(error, attempts) - await sleep(sleepTime) + if (error instanceof Error && attempts > 0 && errorTest(error)) { + if (onError) onError(error, attempts, sleepTime) + attempts-- + // The reason for the jitter is to avoid a thundering herd problem. + // Suppose two independent processes/threads start at the same time. + // They both fail, perhaps due to rate limiting. Now, if they both + // sleep for 30 seconds in the first retry attempt, it'll just + // clash again 30 seconds later. But if you add a bit of jitter, at + // the next attempt these independent processes/threads will now + // start at slightly different times. + + // According to the Oxford English dictionary, they define "jitter" as: + // + // slight irregular movement, variation, or unsteadiness, + // especially in an electrical signal or electronic device. + // + await sleep(addJitter(sleepTime, jitterPercent)) + if (exponential) { + sleepTime *= 2 + } } else { throw error } } } } + +function addJitter(num, percent) { + // Return the number plus between 0 and $percent of that number. + // For example, for 1,000 with a 20% jitter you might get 1133.4 + // because you start with 1,000 and 13.4% is a random number between + // 0 and 20%. + return num + Math.random() * percent * 0.01 * num +} diff --git a/src/audit-logs/data/fpt/organization.json b/src/audit-logs/data/fpt/organization.json index 96446f4381..e47f838f04 100644 --- a/src/audit-logs/data/fpt/organization.json +++ b/src/audit-logs/data/fpt/organization.json @@ -1369,6 +1369,16 @@ "description": "N/A", "docs_reference_links": "N/A" }, + { + "action": "org.self_hosted_runner_offline", + "description": "The runner application was stopped. Only visible through the REST API. Not visible in the web UI or CSV/JSON exports.", + "docs_reference_links": "/actions/hosting-your-own-runners/managing-self-hosted-runners/monitoring-and-troubleshooting-self-hosted-runners#checking-the-status-of-a-self-hosted-runner" + }, + { + "action": "org.self_hosted_runner_online", + "description": "The runner application was started. Only visible through the REST API. Not visible in the web UI or CSV/JSON exports.", + "docs_reference_links": "/actions/hosting-your-own-runners/managing-self-hosted-runners/monitoring-and-troubleshooting-self-hosted-runners#checking-the-status-of-a-self-hosted-runner" + }, { "action": "org.self_hosted_runner_updated", "description": "The runner application was updated. This event is not included in the JSON/CSV export.", @@ -1409,6 +1419,11 @@ "description": "N/A", "docs_reference_links": "N/A" }, + { + "action": "org.sso_response", + "description": "A SAML single sign-on (SSO) response was generated when a member attempted to authenticate with your organization. Only visible through the REST API. Not visible in the web UI or CSV/JSON exports.", + "docs_reference_links": "N/A" + }, { "action": "org.transfer", "description": "An organization was transferred between enterprise accounts.", @@ -2329,6 +2344,16 @@ "description": "N/A", "docs_reference_links": "N/A" }, + { + "action": "repo.self_hosted_runner_offline", + "description": "The runner application was stopped. Only visible through the REST API. Not visible in the web UI or CSV/JSON exports.", + "docs_reference_links": "/actions/hosting-your-own-runners/managing-self-hosted-runners/monitoring-and-troubleshooting-self-hosted-runners#checking-the-status-of-a-self-hosted-runner" + }, + { + "action": "repo.self_hosted_runner_online", + "description": "The runner application was started. Only visible through the REST API. Not visible in the web UI or CSV/JSON exports.", + "docs_reference_links": "/actions/hosting-your-own-runners/managing-self-hosted-runners/monitoring-and-troubleshooting-self-hosted-runners#checking-the-status-of-a-self-hosted-runner" + }, { "action": "repo.self_hosted_runner_updated", "description": "The runner application was updated. This event is not included in the JSON/CSV export.", @@ -3079,6 +3104,16 @@ "description": "N/A", "docs_reference_links": "N/A" }, + { + "action": "workflows.completed_workflow_run", + "description": "A workflow status changed to completed. Only visible through the REST API. Not visible in the web UI or CSV/JSON exports.", + "docs_reference_links": "/actions/monitoring-and-troubleshooting-workflows/viewing-workflow-run-history" + }, + { + "action": "workflows.created_workflow_run", + "description": "A workflow run was create. Only visible through the REST API. Not visible in the web UI or CSV/JSON exports.", + "docs_reference_links": "/actions/learn-github-actions/understanding-github-actions#create-an-example-workflow" + }, { "action": "workflows.delete_workflow_run", "description": "A workflow run was deleted.", @@ -3094,6 +3129,11 @@ "description": "A workflow was enabled, after previously being disabled by disable_workflow.", "docs_reference_links": "N/A" }, + { + "action": "workflows.prepared_workflow_job", + "description": "A workflow job was started. Includes the list of secrets that were provided to the job. Only visible through the REST API. Not visible in the web UI or CSV/JSON exports.", + "docs_reference_links": "/actions/using-workflows/events-that-trigger-workflows" + }, { "action": "workflows.reject_workflow_job", "description": "A workflow job was rejected.", diff --git a/src/audit-logs/data/ghec/enterprise.json b/src/audit-logs/data/ghec/enterprise.json index ec98f58b7e..0c1c3b85be 100644 --- a/src/audit-logs/data/ghec/enterprise.json +++ b/src/audit-logs/data/ghec/enterprise.json @@ -34,6 +34,11 @@ "description": "Someone was removed from the credit section of a security advisory.", "docs_reference_links": "N/A" }, + { + "action": "api.request", + "description": "An API request was made to a security-significant endpoint for the enterprise. Only visible through the REST API. Not visible in the web UI or CSV/JSON exports.", + "docs_reference_links": "/admin/monitoring-activity-in-your-enterprise/reviewing-audit-logs-for-your-enterprise/streaming-the-audit-log-for-your-enterprise#enabling-audit-log-streaming-of-api-requests" + }, { "action": "artifact.destroy", "description": "A workflow run artifact was manually deleted.", @@ -399,6 +404,11 @@ "description": "N/A", "docs_reference_links": "N/A" }, + { + "action": "business.sso_response", + "description": "A SAML single sign-on (SSO) response was generated when a member attempted to authenticate with your enterprise. Only visible through the REST API. Not visible in the web UI or CSV/JSON exports.", + "docs_reference_links": "N/A" + }, { "action": "business.update_actions_settings", "description": "An enterprise owner or site administrator updated GitHub Actions policy settings for an enterprise.", @@ -919,6 +929,16 @@ "description": "The visibility of a GitHub Actions self-hosted runner group was updated via the REST API.", "docs_reference_links": "/rest/actions#update-a-self-hosted-runner-group-for-an-organization" }, + { + "action": "enterprise.self_hosted_runner_offline", + "description": "The GitHub Actions runner application was stopped. Only visible through the REST API. Not visible in the web UI or CSV/JSON exports.", + "docs_reference_links": "/actions/hosting-your-own-runners/managing-self-hosted-runners/monitoring-and-troubleshooting-self-hosted-runners#checking-the-status-of-a-self-hosted-runner" + }, + { + "action": "enterprise.self_hosted_runner_online", + "description": "The GitHub Actions runner application was started. Only visible through the REST API. Not visible in the web UI or CSV/JSON exports.", + "docs_reference_links": "/actions/hosting-your-own-runners/managing-self-hosted-runners/monitoring-and-troubleshooting-self-hosted-runners#checking-the-status-of-a-self-hosted-runner" + }, { "action": "enterprise.self_hosted_runner_updated", "description": "The GitHub Actions runner application was updated. This event is not included in the JSON/CSV export.", @@ -1989,6 +2009,16 @@ "description": "N/A", "docs_reference_links": "N/A" }, + { + "action": "org.self_hosted_runner_offline", + "description": "The runner application was stopped. Only visible through the REST API. Not visible in the web UI or CSV/JSON exports.", + "docs_reference_links": "/actions/hosting-your-own-runners/managing-self-hosted-runners/monitoring-and-troubleshooting-self-hosted-runners#checking-the-status-of-a-self-hosted-runner" + }, + { + "action": "org.self_hosted_runner_online", + "description": "The runner application was started. Only visible through the REST API. Not visible in the web UI or CSV/JSON exports.", + "docs_reference_links": "/actions/hosting-your-own-runners/managing-self-hosted-runners/monitoring-and-troubleshooting-self-hosted-runners#checking-the-status-of-a-self-hosted-runner" + }, { "action": "org.self_hosted_runner_updated", "description": "The runner application was updated. This event is not included in the JSON/CSV export.", @@ -2029,6 +2059,11 @@ "description": "N/A", "docs_reference_links": "N/A" }, + { + "action": "org.sso_response", + "description": "A SAML single sign-on (SSO) response was generated when a member attempted to authenticate with your organization. Only visible through the REST API. Not visible in the web UI or CSV/JSON exports.", + "docs_reference_links": "N/A" + }, { "action": "org.transfer", "description": "An organization was transferred between enterprise accounts.", @@ -2929,6 +2964,16 @@ "description": "N/A", "docs_reference_links": "N/A" }, + { + "action": "repo.self_hosted_runner_offline", + "description": "The runner application was stopped. Only visible through the REST API. Not visible in the web UI or CSV/JSON exports.", + "docs_reference_links": "/actions/hosting-your-own-runners/managing-self-hosted-runners/monitoring-and-troubleshooting-self-hosted-runners#checking-the-status-of-a-self-hosted-runner" + }, + { + "action": "repo.self_hosted_runner_online", + "description": "The runner application was started. Only visible through the REST API. Not visible in the web UI or CSV/JSON exports.", + "docs_reference_links": "/actions/hosting-your-own-runners/managing-self-hosted-runners/monitoring-and-troubleshooting-self-hosted-runners#checking-the-status-of-a-self-hosted-runner" + }, { "action": "repo.self_hosted_runner_updated", "description": "The runner application was updated. This event is not included in the JSON/CSV export.", @@ -3684,6 +3729,16 @@ "description": "N/A", "docs_reference_links": "N/A" }, + { + "action": "workflows.completed_workflow_run", + "description": "A workflow status changed to completed. Only visible through the REST API. Not visible in the web UI or CSV/JSON exports.", + "docs_reference_links": "/actions/monitoring-and-troubleshooting-workflows/viewing-workflow-run-history" + }, + { + "action": "workflows.created_workflow_run", + "description": "A workflow run was create. Only visible through the REST API. Not visible in the web UI or CSV/JSON exports.", + "docs_reference_links": "/actions/learn-github-actions/understanding-github-actions#create-an-example-workflow" + }, { "action": "workflows.delete_workflow_run", "description": "A workflow run was deleted.", @@ -3699,6 +3754,11 @@ "description": "A workflow was enabled, after previously being disabled by disable_workflow.", "docs_reference_links": "N/A" }, + { + "action": "workflows.prepared_workflow_job", + "description": "A workflow job was started. Includes the list of secrets that were provided to the job. Only visible through the REST API. Not visible in the web UI or CSV/JSON exports.", + "docs_reference_links": "/actions/using-workflows/events-that-trigger-workflows" + }, { "action": "workflows.reject_workflow_job", "description": "A workflow job was rejected.", diff --git a/src/audit-logs/data/ghec/organization.json b/src/audit-logs/data/ghec/organization.json index 8a8137cb22..cb9acc432a 100644 --- a/src/audit-logs/data/ghec/organization.json +++ b/src/audit-logs/data/ghec/organization.json @@ -1289,6 +1289,16 @@ "description": "N/A", "docs_reference_links": "N/A" }, + { + "action": "org.self_hosted_runner_offline", + "description": "The runner application was stopped. Only visible through the REST API. Not visible in the web UI or CSV/JSON exports.", + "docs_reference_links": "/actions/hosting-your-own-runners/managing-self-hosted-runners/monitoring-and-troubleshooting-self-hosted-runners#checking-the-status-of-a-self-hosted-runner" + }, + { + "action": "org.self_hosted_runner_online", + "description": "The runner application was started. Only visible through the REST API. Not visible in the web UI or CSV/JSON exports.", + "docs_reference_links": "/actions/hosting-your-own-runners/managing-self-hosted-runners/monitoring-and-troubleshooting-self-hosted-runners#checking-the-status-of-a-self-hosted-runner" + }, { "action": "org.self_hosted_runner_updated", "description": "The runner application was updated. This event is not included in the JSON/CSV export.", @@ -1329,6 +1339,11 @@ "description": "N/A", "docs_reference_links": "N/A" }, + { + "action": "org.sso_response", + "description": "A SAML single sign-on (SSO) response was generated when a member attempted to authenticate with your organization. Only visible through the REST API. Not visible in the web UI or CSV/JSON exports.", + "docs_reference_links": "N/A" + }, { "action": "org.transfer", "description": "An organization was transferred between enterprise accounts.", @@ -2229,6 +2244,16 @@ "description": "N/A", "docs_reference_links": "N/A" }, + { + "action": "repo.self_hosted_runner_offline", + "description": "The runner application was stopped. Only visible through the REST API. Not visible in the web UI or CSV/JSON exports.", + "docs_reference_links": "/actions/hosting-your-own-runners/managing-self-hosted-runners/monitoring-and-troubleshooting-self-hosted-runners#checking-the-status-of-a-self-hosted-runner" + }, + { + "action": "repo.self_hosted_runner_online", + "description": "The runner application was started. Only visible through the REST API. Not visible in the web UI or CSV/JSON exports.", + "docs_reference_links": "/actions/hosting-your-own-runners/managing-self-hosted-runners/monitoring-and-troubleshooting-self-hosted-runners#checking-the-status-of-a-self-hosted-runner" + }, { "action": "repo.self_hosted_runner_updated", "description": "The runner application was updated. This event is not included in the JSON/CSV export.", @@ -2954,6 +2979,16 @@ "description": "N/A", "docs_reference_links": "N/A" }, + { + "action": "workflows.completed_workflow_run", + "description": "A workflow status changed to completed. Only visible through the REST API. Not visible in the web UI or CSV/JSON exports.", + "docs_reference_links": "/actions/monitoring-and-troubleshooting-workflows/viewing-workflow-run-history" + }, + { + "action": "workflows.created_workflow_run", + "description": "A workflow run was create. Only visible through the REST API. Not visible in the web UI or CSV/JSON exports.", + "docs_reference_links": "/actions/learn-github-actions/understanding-github-actions#create-an-example-workflow" + }, { "action": "workflows.delete_workflow_run", "description": "A workflow run was deleted.", @@ -2969,6 +3004,11 @@ "description": "A workflow was enabled, after previously being disabled by disable_workflow.", "docs_reference_links": "N/A" }, + { + "action": "workflows.prepared_workflow_job", + "description": "A workflow job was started. Includes the list of secrets that were provided to the job. Only visible through the REST API. Not visible in the web UI or CSV/JSON exports.", + "docs_reference_links": "/actions/using-workflows/events-that-trigger-workflows" + }, { "action": "workflows.reject_workflow_job", "description": "A workflow job was rejected.", diff --git a/src/audit-logs/data/ghes/enterprise.json b/src/audit-logs/data/ghes/enterprise.json index f9741ff3fe..a5aceda55b 100644 --- a/src/audit-logs/data/ghes/enterprise.json +++ b/src/audit-logs/data/ghes/enterprise.json @@ -1,4 +1,9 @@ [ + { + "action": "api.request", + "description": "An API request was made to a security-significant endpoint for the enterprise. Only visible through the REST API. Not visible in the web UI or CSV/JSON exports.", + "docs_reference_links": "/admin/monitoring-activity-in-your-enterprise/reviewing-audit-logs-for-your-enterprise/streaming-the-audit-log-for-your-enterprise#enabling-audit-log-streaming-of-api-requests" + }, { "action": "business.add_admin", "description": "An enterprise owner was added to an enterprise.", @@ -109,6 +114,11 @@ "description": "The retention period for GitHub Actions artifacts and logs was changed for an enterprise.", "docs_reference_links": "/admin/policies/enforcing-policies-for-your-enterprise/enforcing-policies-for-github-actions-in-your-enterprise#enforcing-a-policy-for-artifact-and-log-retention-in-your-enterprise" }, + { + "action": "business.sso_response", + "description": "A SAML single sign-on (SSO) response was generated when a member attempted to authenticate with your enterprise. Only visible through the REST API. Not visible in the web UI or CSV/JSON exports.", + "docs_reference_links": "N/A" + }, { "action": "business.update_actions_settings", "description": "An enterprise owner or site administrator updated GitHub Actions policy settings for an enterprise.", @@ -369,6 +379,16 @@ "description": "The configuration of a GitHub Actions self-hosted runner group was changed.", "docs_reference_links": "/actions/hosting-your-own-runners/managing-self-hosted-runners/managing-access-to-self-hosted-runners-using-groups#changing-the-access-policy-of-a-self-hosted-runner-group" }, + { + "action": "enterprise.self_hosted_runner_offline", + "description": "The GitHub Actions runner application was stopped. Only visible through the REST API. Not visible in the web UI or CSV/JSON exports.", + "docs_reference_links": "/actions/hosting-your-own-runners/managing-self-hosted-runners/monitoring-and-troubleshooting-self-hosted-runners#checking-the-status-of-a-self-hosted-runner" + }, + { + "action": "enterprise.self_hosted_runner_online", + "description": "The GitHub Actions runner application was started. Only visible through the REST API. Not visible in the web UI or CSV/JSON exports.", + "docs_reference_links": "/actions/hosting-your-own-runners/managing-self-hosted-runners/monitoring-and-troubleshooting-self-hosted-runners#checking-the-status-of-a-self-hosted-runner" + }, { "action": "enterprise.self_hosted_runner_updated", "description": "The GitHub Actions runner application was updated. This event is not included in the JSON/CSV export.", @@ -954,6 +974,16 @@ "description": "N/A", "docs_reference_links": "N/A" }, + { + "action": "org.self_hosted_runner_offline", + "description": "The runner application was stopped. Only visible through the REST API. Not visible in the web UI or CSV/JSON exports.", + "docs_reference_links": "/actions/hosting-your-own-runners/managing-self-hosted-runners/monitoring-and-troubleshooting-self-hosted-runners#checking-the-status-of-a-self-hosted-runner" + }, + { + "action": "org.self_hosted_runner_online", + "description": "The runner application was started. Only visible through the REST API. Not visible in the web UI or CSV/JSON exports.", + "docs_reference_links": "/actions/hosting-your-own-runners/managing-self-hosted-runners/monitoring-and-troubleshooting-self-hosted-runners#checking-the-status-of-a-self-hosted-runner" + }, { "action": "org.self_hosted_runner_updated", "description": "The runner application was updated. This event is not included in the JSON/CSV export.", @@ -976,7 +1006,7 @@ }, { "action": "org.sso_response", - "description": "A SAML single sign-on (SSO) response was generated when a member attempted to authenticate with your organization.", + "description": "A SAML single sign-on (SSO) response was generated when a member attempted to authenticate with your organization. Only visible through the REST API. Not visible in the web UI or CSV/JSON exports.", "docs_reference_links": "N/A" }, { @@ -1549,6 +1579,16 @@ "description": "N/A", "docs_reference_links": "N/A" }, + { + "action": "repo.self_hosted_runner_offline", + "description": "The runner application was stopped. Only visible through the REST API. Not visible in the web UI or CSV/JSON exports.", + "docs_reference_links": "/actions/hosting-your-own-runners/managing-self-hosted-runners/monitoring-and-troubleshooting-self-hosted-runners#checking-the-status-of-a-self-hosted-runner" + }, + { + "action": "repo.self_hosted_runner_online", + "description": "The runner application was started. Only visible through the REST API. Not visible in the web UI or CSV/JSON exports.", + "docs_reference_links": "/actions/hosting-your-own-runners/managing-self-hosted-runners/monitoring-and-troubleshooting-self-hosted-runners#checking-the-status-of-a-self-hosted-runner" + }, { "action": "repo.self_hosted_runner_updated", "description": "The runner application was updated. This event is not included in the JSON/CSV export.", @@ -2199,6 +2239,16 @@ "description": "N/A", "docs_reference_links": "N/A" }, + { + "action": "workflows.completed_workflow_run", + "description": "A workflow status changed to completed. Only visible through the REST API. Not visible in the web UI or CSV/JSON exports.", + "docs_reference_links": "/actions/monitoring-and-troubleshooting-workflows/viewing-workflow-run-history" + }, + { + "action": "workflows.created_workflow_run", + "description": "A workflow run was create. Only visible through the REST API. Not visible in the web UI or CSV/JSON exports.", + "docs_reference_links": "/actions/learn-github-actions/understanding-github-actions#create-an-example-workflow" + }, { "action": "workflows.delete_workflow_run", "description": "A workflow run was deleted.", @@ -2214,6 +2264,11 @@ "description": "A workflow was enabled, after previously being disabled by disable_workflow.", "docs_reference_links": "N/A" }, + { + "action": "workflows.prepared_workflow_job", + "description": "A workflow job was started. Includes the list of secrets that were provided to the job. Only visible through the REST API. Not visible in the web UI or CSV/JSON exports.", + "docs_reference_links": "/actions/using-workflows/events-that-trigger-workflows" + }, { "action": "workflows.reject_workflow_job", "description": "A workflow job was rejected.", diff --git a/src/audit-logs/data/ghes/organization.json b/src/audit-logs/data/ghes/organization.json index cd57601bd4..0004a6533e 100644 --- a/src/audit-logs/data/ghes/organization.json +++ b/src/audit-logs/data/ghes/organization.json @@ -569,6 +569,16 @@ "description": "N/A", "docs_reference_links": "N/A" }, + { + "action": "org.self_hosted_runner_offline", + "description": "The runner application was stopped. Only visible through the REST API. Not visible in the web UI or CSV/JSON exports.", + "docs_reference_links": "/actions/hosting-your-own-runners/managing-self-hosted-runners/monitoring-and-troubleshooting-self-hosted-runners#checking-the-status-of-a-self-hosted-runner" + }, + { + "action": "org.self_hosted_runner_online", + "description": "The runner application was started. Only visible through the REST API. Not visible in the web UI or CSV/JSON exports.", + "docs_reference_links": "/actions/hosting-your-own-runners/managing-self-hosted-runners/monitoring-and-troubleshooting-self-hosted-runners#checking-the-status-of-a-self-hosted-runner" + }, { "action": "org.self_hosted_runner_updated", "description": "The runner application was updated. This event is not included in the JSON/CSV export.", @@ -589,6 +599,11 @@ "description": "N/A", "docs_reference_links": "N/A" }, + { + "action": "org.sso_response", + "description": "A SAML single sign-on (SSO) response was generated when a member attempted to authenticate with your organization. Only visible through the REST API. Not visible in the web UI or CSV/JSON exports.", + "docs_reference_links": "N/A" + }, { "action": "org.unblock_user", "description": "A user was unblocked from an organization.", @@ -1119,6 +1134,16 @@ "description": "N/A", "docs_reference_links": "N/A" }, + { + "action": "repo.self_hosted_runner_offline", + "description": "The runner application was stopped. Only visible through the REST API. Not visible in the web UI or CSV/JSON exports.", + "docs_reference_links": "/actions/hosting-your-own-runners/managing-self-hosted-runners/monitoring-and-troubleshooting-self-hosted-runners#checking-the-status-of-a-self-hosted-runner" + }, + { + "action": "repo.self_hosted_runner_online", + "description": "The runner application was started. Only visible through the REST API. Not visible in the web UI or CSV/JSON exports.", + "docs_reference_links": "/actions/hosting-your-own-runners/managing-self-hosted-runners/monitoring-and-troubleshooting-self-hosted-runners#checking-the-status-of-a-self-hosted-runner" + }, { "action": "repo.self_hosted_runner_updated", "description": "The runner application was updated. This event is not included in the JSON/CSV export.", @@ -1474,6 +1499,16 @@ "description": "N/A", "docs_reference_links": "N/A" }, + { + "action": "workflows.completed_workflow_run", + "description": "A workflow status changed to completed. Only visible through the REST API. Not visible in the web UI or CSV/JSON exports.", + "docs_reference_links": "/actions/monitoring-and-troubleshooting-workflows/viewing-workflow-run-history" + }, + { + "action": "workflows.created_workflow_run", + "description": "A workflow run was create. Only visible through the REST API. Not visible in the web UI or CSV/JSON exports.", + "docs_reference_links": "/actions/learn-github-actions/understanding-github-actions#create-an-example-workflow" + }, { "action": "workflows.delete_workflow_run", "description": "A workflow run was deleted.", @@ -1489,6 +1524,11 @@ "description": "A workflow was enabled, after previously being disabled by disable_workflow.", "docs_reference_links": "N/A" }, + { + "action": "workflows.prepared_workflow_job", + "description": "A workflow job was started. Includes the list of secrets that were provided to the job. Only visible through the REST API. Not visible in the web UI or CSV/JSON exports.", + "docs_reference_links": "/actions/using-workflows/events-that-trigger-workflows" + }, { "action": "workflows.reject_workflow_job", "description": "A workflow job was rejected.", diff --git a/src/audit-logs/scripts/sync.js b/src/audit-logs/scripts/sync.js index d6496c333e..b302a50a89 100755 --- a/src/audit-logs/scripts/sync.js +++ b/src/audit-logs/scripts/sync.js @@ -126,6 +126,59 @@ async function main() { auditLogData.ghec.user.push(minimalEvent) } } + + // API only events have either `business_api_only` or `org_api_only` + // allowlist values. + // + // * business_api_only: goes to the enterprise events page and is versioned + // for GHEC. If the event has any `ghes` versions, also verisoned for GHES. + // * org_api_only: goes to the organzation events page and is versioned + // for fpt and GHEC. If the event has any `ghes` versions, also versioned + // for GHES. + // + // There's currently one case (`org.sso_response`) where an event is an + // api only event but also has a non-api-only allowlist value so that's + // why we have the checks to prevent adding the event twice. + if (event._allowlists.includes('business_api_only')) { + if (!auditLogData.ghec.enterprise.find((e) => e.action === event.action)) { + auditLogData.ghec.enterprise.push(minimalEvent) + } + + if ( + Object.keys(event.ghes).length > 0 && + !auditLogData.ghes.enterprise.find((e) => e.action === event.action) + ) { + auditLogData.ghes.enterprise.push(minimalEvent) + } + } + + if (event._allowlists.includes('org_api_only')) { + if (!auditLogData.ghec.organization.find((e) => e.action === event.action)) { + auditLogData.fpt.organization.push(minimalEvent) + auditLogData.ghec.organization.push(minimalEvent) + } + + if ( + Object.keys(event.ghes).length > 0 && + !auditLogData.ghes.organization.find((e) => e.action === event.action) + ) { + auditLogData.ghes.organization.push(minimalEvent) + } + } + + // For the API only events we append an extra note to the description + // that explains the event is api only. It's better to do it here instead + // of in the schema where it would have to be updated and maintained manually. + // + // Some events have both API only allowlist values so we make sure to append + // to the description only once. + if ( + event._allowlists.includes('org_api_only') || + event._allowlists.includes('business_api_only') + ) { + minimalEvent.description += + ' Only visible through the REST API. Not visible in the web UI or CSV/JSON exports.' + } }) // write out audit log event data to page event files per version e.g.: diff --git a/src/search/scripts/index-elasticsearch.js b/src/search/scripts/index-elasticsearch.js index 1133497901..98f2848b0e 100755 --- a/src/search/scripts/index-elasticsearch.js +++ b/src/search/scripts/index-elasticsearch.js @@ -18,7 +18,6 @@ import dotenv from 'dotenv' import { retryOnErrorTest } from '../../../script/helpers/retry-on-error-test.js' import { languageKeys } from '#src/languages/lib/languages.js' import { allVersions } from '#src/versions/lib/all-versions.js' -import statsd from '#src/observability/lib/statsd.js' // Now you can optionally have set the ELASTICSEARCH_URL in your .env file. dotenv.config() @@ -49,6 +48,8 @@ const shortNames = Object.fromEntries( const allVersionKeys = Object.keys(shortNames) +const DEFAULT_SLEEPTIME_SECONDS = 30 + program .description('Creates Elasticsearch index from records') .option('-v, --verbose', 'Verbose outputs') @@ -72,6 +73,28 @@ program return parsed }, ) + .option( + '-r, --retries ', + 'Number of retry attempts on recoverable network errors', + (value) => { + const parsed = parseInt(value, 10) + if (isNaN(parsed)) { + throw new InvalidArgumentError('Not a number.') + } + return parsed + }, + ) + .option( + '--sleep-time ', + `Number of seconds to sleep between each retry attempt (defaults to ${DEFAULT_SLEEPTIME_SECONDS})`, + (value) => { + const parsed = parseInt(value, 10) + if (isNaN(parsed)) { + throw new InvalidArgumentError('Not a number.') + } + return parsed + }, + ) .argument('', 'where the indexable files are') .parse(process.argv) @@ -195,21 +218,27 @@ async function indexAll(node, sourceDirectory, opts) { const prefix = indexPrefix ? `${indexPrefix}_` : '' for (const language of languages) { + let count = 0 for (const versionKey of versionKeys) { console.log(chalk.yellow(`Indexing ${chalk.bold(versionKey)} in ${chalk.bold(language)}`)) const indexName = `${prefix}github-docs-${versionKey}-${language}` - console.time(`Indexing ${indexName}`) - await indexVersion(client, indexName, versionKey, language, sourceDirectory, verbose) - console.timeEnd(`Indexing ${indexName}`) + const t0 = new Date() + await indexVersion(client, indexName, versionKey, language, sourceDirectory, opts) + const t1 = new Date() + console.log(chalk.green(`Finished indexing ${indexName}. Took ${formatTime(t1 - t0)}`)) if (verbose) { console.log(`To view index: ${safeUrlDisplay(node + `/${indexName}`)}`) console.log(`To search index: ${safeUrlDisplay(node + `/${indexName}/_search`)}`) } - if (staggerSeconds) { + count++ + // console.log({ count, versionKeysLength: versionKeys.length }) + if (staggerSeconds && count < versionKeys.length - 1) { console.log(`Sleeping for ${staggerSeconds} seconds...`) await sleep(1000 * staggerSeconds) } + // A bit of visual separation betweeen each version + console.log('') } } } @@ -247,21 +276,16 @@ function utcTimestamp() { } // Consider moving this to lib -async function indexVersion( - client, - indexName, - version, - language, - sourceDirectory, - verbose = false, -) { +async function indexVersion(client, indexName, version, language, sourceDirectory, opts) { + const { verbose } = opts + // Note, it's a bit "weird" that numbered releases versions are // called the number but that's the convention the previous // search backend used - const indexVersion = shortNames[version].hasNumberedReleases + const indexVersionName = shortNames[version].hasNumberedReleases ? shortNames[version].currentRelease : shortNames[version].miscBaseName - const recordsName = `github-docs-${indexVersion}-${language}` + const recordsName = `github-docs-${indexVersionName}-${language}` const records = await loadRecords(recordsName, sourceDirectory) @@ -354,15 +378,6 @@ async function indexVersion( return [{ index: { _index: thisAlias } }, record] }) - // It's important to use `client.bulk.bind(client)` here because - // `client.bulk` is a meta-function that is attached to the Client - // class. Internally, it depends on `this.` even though it's a - // free-standing function. So if called indirectly by the `statsd.asyncTimer` - // the `this` becomes undefined. - const timed = statsd.asyncTimer(client.bulk.bind(client), 'search.bulk_index', [ - `version:${version}`, - `language:${language}`, - ]) const bulkOptions = { // Default is 'false'. // It means that the index is NOT refreshed as documents are inserted. @@ -373,7 +388,36 @@ async function indexVersion( // by a bot on a schedeule (GitHub Actions). timeout: '5m', } - const bulkResponse = await timed({ operations, ...bulkOptions }) + + const attempts = opts.retries || 0 + const sleepTime = (opts.sleepTime || DEFAULT_SLEEPTIME_SECONDS) * 1000 + + console.log(`About to bulk index ${allRecords.length.toLocaleString()} records with retry %O`, { + attempts, + sleepTime, + }) + const t0 = new Date() + const bulkResponse = await retryOnErrorTest( + (error) => { + // Rate limiting can happen when you're indexing too much at + // same time. + return error instanceof errors.ResponseError && error.meta.statusCode === 429 + }, + () => client.bulk({ operations, ...bulkOptions }), + { + attempts, + sleepTime, + onError: (_, attempts, sleepTime) => { + console.warn( + chalk.yellow( + `Failed to bulk index ${indexName}. Will attempt ${attempts} more times (after ${ + sleepTime / 1000 + }s sleep).`, + ), + ) + }, + }, + ) if (bulkResponse.errors) { // Some day, when we're more confident how and why this might happen @@ -384,9 +428,28 @@ async function indexVersion( console.error(`Bulk response errors: ${bulkResponse.errors}`) throw new Error('Bulk errors happened.') } + const t1 = new Date() + console.log(`Bulk indexed ${thisAlias}. Took ${formatTime(t1 - t0)}`) - const { count } = await client.count({ index: thisAlias }) - console.log(`Documents now in ${chalk.bold(thisAlias)}: ${chalk.bold(count.toLocaleString())}`) + // The counting of documents in the index is async and can take a while + // to reflect. So send count requests until we get the right number. + let documentsInIndex = 0 + let countAttempts = 3 + while (documentsInIndex < allRecords.length) { + const { count } = await client.count({ index: thisAlias }) + documentsInIndex = count + if (documentsInIndex >= allRecords.length) break + countAttempts-- + if (!countAttempts) { + console.log(`After ${countAttempts} attempts still haven't matched the expected number.`) + break + } + await sleep(1000) + } + + console.log( + `Documents now in ${chalk.bold(thisAlias)}: ${chalk.bold(documentsInIndex.toLocaleString())}`, + ) // To perform an atomic operation that creates the new alias and removes // the old indexes, we can use the updateAliases API with a body that @@ -404,21 +467,23 @@ async function indexVersion( ] console.log(`Alias ${indexName} -> ${thisAlias}`) - // const indices = await client.cat.indices({ format: 'json' }) + console.log('About to get indices with retry %O', { attempts, sleepTime }) const indices = await retryOnErrorTest( (error) => { - return error instanceof errors.ResponseError && error.statusCode === 404 + // 404 can happen when you're trying to get an index that + // doesn't exist. ...yet! + return error instanceof errors.ResponseError && error.meta.statusCode === 404 }, () => client.cat.indices({ format: 'json' }), { - // Combined, this is a total of 30 seconds which is not long - // for an Action that runs based on automation. - attempts: 10, - sleepTime: 3000, - onError: (error, attempts) => { + attempts, + sleepTime, + onError: (error, attempts, sleepTime) => { console.warn( chalk.yellow( - `Failed to get a list of indexes for '${indexName}' (${error.message}). Will attempt ${attempts} more times.`, + `Failed to get index ${indexName} (${ + error.message || error.toString() + }). Will attempt ${attempts} more times (after ${formatTime(sleepTime)}s sleep).`, ), ) }, @@ -458,3 +523,14 @@ function getSnowballLanguage(language) { pt: 'Portuguese', }[language] } + +function formatTime(ms) { + if (ms < 1000) { + return `${ms.toFixed(1)}ms` + } + const seconds = ms / 1000 + if (seconds > 60) { + return `${Math.round(seconds / 60)}m${Math.round(seconds % 60)}s` + } + return `${seconds.toFixed(1)}s` +}