From d48ad66dcb44dd688bd58d3bde954d3cff9f309b Mon Sep 17 00:00:00 2001 From: Peter Bengtsson Date: Tue, 12 Dec 2023 09:07:59 -0500 Subject: [PATCH] Use retrying Octokit to for GraphQL sync (#47656) --- package-lock.json | 25 +++++++++++++++++-------- package.json | 1 + src/graphql/scripts/sync.js | 8 +++----- src/workflows/git-utils.js | 12 ++++++------ src/workflows/github.js | 9 +++++++++ 5 files changed, 36 insertions(+), 19 deletions(-) diff --git a/package-lock.json b/package-lock.json index 9464034466..a158cc4b0a 100644 --- a/package-lock.json +++ b/package-lock.json @@ -10,6 +10,7 @@ "dependencies": { "@elastic/elasticsearch": "8.10.0", "@github/failbot": "0.8.3", + "@octokit/plugin-retry": "6.0.1", "@octokit/request-error": "5.0.1", "@primer/behaviors": "^1.3.5", "@primer/css": "^21.0.7", @@ -2428,7 +2429,6 @@ "version": "4.0.0", "resolved": "https://registry.npmjs.org/@octokit/auth-token/-/auth-token-4.0.0.tgz", "integrity": "sha512-tY/msAuJo6ARbK6SPIxZrPBms3xPbfwBrulZe0Wtr/DIY9lje2HeV1uoebShn6mx7SjCHif6EjMvoREj+gZ+SA==", - "dev": true, "engines": { "node": ">= 18" } @@ -2437,7 +2437,6 @@ "version": "5.0.1", "resolved": "https://registry.npmjs.org/@octokit/core/-/core-5.0.1.tgz", "integrity": "sha512-lyeeeZyESFo+ffI801SaBKmCfsvarO+dgV8/0gD8u1d87clbEdWsP5yC+dSj3zLhb2eIf5SJrn6vDz9AheETHw==", - "dev": true, "dependencies": { "@octokit/auth-token": "^4.0.0", "@octokit/graphql": "^7.0.0", @@ -2455,7 +2454,6 @@ "version": "9.0.1", "resolved": "https://registry.npmjs.org/@octokit/endpoint/-/endpoint-9.0.1.tgz", "integrity": "sha512-hRlOKAovtINHQPYHZlfyFwaM8OyetxeoC81lAkBy34uLb8exrZB50SQdeW3EROqiY9G9yxQTpp5OHTV54QD+vA==", - "dev": true, "dependencies": { "@octokit/types": "^12.0.0", "is-plain-object": "^5.0.0", @@ -2469,7 +2467,6 @@ "version": "7.0.2", "resolved": "https://registry.npmjs.org/@octokit/graphql/-/graphql-7.0.2.tgz", "integrity": "sha512-OJ2iGMtj5Tg3s6RaXH22cJcxXRi7Y3EBqbHTBRq+PQAqfaS8f/236fUrWhfSn8P4jovyzqucxme7/vWSSZBX2Q==", - "dev": true, "dependencies": { "@octokit/request": "^8.0.1", "@octokit/types": "^12.0.0", @@ -2514,11 +2511,26 @@ "@octokit/core": ">=5" } }, + "node_modules/@octokit/plugin-retry": { + "version": "6.0.1", + "resolved": "https://registry.npmjs.org/@octokit/plugin-retry/-/plugin-retry-6.0.1.tgz", + "integrity": "sha512-SKs+Tz9oj0g4p28qkZwl/topGcb0k0qPNX/i7vBKmDsjoeqnVfFUquqrE/O9oJY7+oLzdCtkiWSXLpLjvl6uog==", + "dependencies": { + "@octokit/request-error": "^5.0.0", + "@octokit/types": "^12.0.0", + "bottleneck": "^2.15.3" + }, + "engines": { + "node": ">= 18" + }, + "peerDependencies": { + "@octokit/core": ">=5" + } + }, "node_modules/@octokit/request": { "version": "8.1.4", "resolved": "https://registry.npmjs.org/@octokit/request/-/request-8.1.4.tgz", "integrity": "sha512-M0aaFfpGPEKrg7XoA/gwgRvc9MSXHRO2Ioki1qrPDbl1e9YhjIwVoHE7HIKmv/m3idzldj//xBujcFNqGX6ENA==", - "dev": true, "dependencies": { "@octokit/endpoint": "^9.0.0", "@octokit/request-error": "^5.0.0", @@ -3918,7 +3930,6 @@ }, "node_modules/before-after-hook": { "version": "2.2.2", - "dev": true, "license": "Apache-2.0" }, "node_modules/big-integer": { @@ -8464,7 +8475,6 @@ }, "node_modules/is-plain-object": { "version": "5.0.0", - "dev": true, "license": "MIT", "engines": { "node": ">=0.10.0" @@ -15748,7 +15758,6 @@ }, "node_modules/universal-user-agent": { "version": "6.0.0", - "dev": true, "license": "ISC" }, "node_modules/universalify": { diff --git a/package.json b/package.json index df86c7f317..f52059695c 100644 --- a/package.json +++ b/package.json @@ -228,6 +228,7 @@ "dependencies": { "@elastic/elasticsearch": "8.10.0", "@github/failbot": "0.8.3", + "@octokit/plugin-retry": "6.0.1", "@octokit/request-error": "5.0.1", "@primer/behaviors": "^1.3.5", "@primer/css": "^21.0.7", diff --git a/src/graphql/scripts/sync.js b/src/graphql/scripts/sync.js index 4f56fd497e..2a36687439 100755 --- a/src/graphql/scripts/sync.js +++ b/src/graphql/scripts/sync.js @@ -11,11 +11,9 @@ import processUpcomingChanges from './utils/process-upcoming-changes.js' import processSchemas from './utils/process-schemas.js' import { prependDatedEntry, createChangelogEntry } from './build-changelog.js' -const graphqlDataDir = path.join(process.cwd(), 'data/graphql') -const graphqlStaticDir = path.join(process.cwd(), 'src/graphql/data') -const dataFilenames = JSON.parse( - await fs.readFile(path.join(process.cwd(), './src/graphql/scripts/utils/data-filenames.json')), -) +const graphqlDataDir = 'data/graphql' +const graphqlStaticDir = 'src/graphql/data' +const dataFilenames = JSON.parse(await fs.readFile('src/graphql/scripts/utils/data-filenames.json')) // check for required PAT if (!process.env.GITHUB_TOKEN) { diff --git a/src/workflows/git-utils.js b/src/workflows/git-utils.js index 1ff934fb6c..28f18e6261 100644 --- a/src/workflows/git-utils.js +++ b/src/workflows/git-utils.js @@ -4,8 +4,8 @@ import fs from 'fs/promises' import { RequestError } from '@octokit/request-error' -import Github from './github.js' -const github = Github() +import { retryingGithub } from './github.js' +const github = retryingGithub() // https://docs.github.com/rest/reference/git#get-a-reference export async function getCommitSha(owner, repo, ref) { @@ -17,7 +17,7 @@ export async function getCommitSha(owner, repo, ref) { }) return data.object.sha } catch (err) { - console.log('error getting tree') + console.log('error getting commit sha', owner, repo, ref) throw err } } @@ -35,7 +35,7 @@ export async function hasMatchingRef(owner, repo, ref) { if (err instanceof RequestError && err.status === 404) { return false } - console.log('error getting tree') + console.log('error getting matching ref', owner, repo, ref) throw err } } @@ -50,7 +50,7 @@ export async function getTreeSha(owner, repo, commitSha) { }) return data.tree.sha } catch (err) { - console.log('error getting tree') + console.log('error getting tree sha', owner, repo, commitSha) throw err } } @@ -70,7 +70,7 @@ export async function getTree(owner, repo, ref) { // skip actions/changes files return data.tree } catch (err) { - console.log('error getting tree') + console.log('error getting tree', owner, repo, ref) throw err } } diff --git a/src/workflows/github.js b/src/workflows/github.js index d9d847010a..f324047e7b 100644 --- a/src/workflows/github.js +++ b/src/workflows/github.js @@ -1,11 +1,14 @@ #!/usr/bin/env node import dotenv from 'dotenv' import { Octokit } from '@octokit/rest' +import { retry } from '@octokit/plugin-retry' if (!process.env.GITHUB_TOKEN) { dotenv.config() } +const RetryingOctokit = Octokit.plugin(retry) + // this module needs to work in development, production, and GitHub Actions // // GITHUB_TOKEN comes from one of the following sources: @@ -20,3 +23,9 @@ export default function github() { auth: `token ${apiToken}`, }) } + +export function retryingGithub() { + return new RetryingOctokit({ + auth: `token ${apiToken}`, + }) +}