Move files out of script/ (#45454)

Co-authored-by: Peter Bengtsson <peterbe@github.com>

src/workflows/git-utils.js

@@ -0,0 +1,243 @@
+#!/usr/bin/env node
+import crypto from 'crypto'
+import fs from 'fs/promises'
+
+import Github from './github.js'
+const github = Github()
+
+// https://docs.github.com/rest/reference/git#get-a-reference
+export async function getCommitSha(owner, repo, ref) {
+  try {
+    const { data } = await github.git.getRef({
+      owner,
+      repo,
+      ref,
+    })
+    return data.object.sha
+  } catch (err) {
+    console.log('error getting tree')
+    throw err
+  }
+}
+
+// https://docs.github.com/rest/reference/git#list-matching-references
+export async function listMatchingRefs(owner, repo, ref) {
+  try {
+    // if the ref is found, this returns an array of objects;
+    // if the ref is not found, this returns an empty array
+    const { data } = await github.git.listMatchingRefs({
+      owner,
+      repo,
+      ref,
+    })
+    return data
+  } catch (err) {
+    console.log('error getting tree')
+    throw err
+  }
+}
+
+// https://docs.github.com/rest/reference/git#get-a-commit
+export async function getTreeSha(owner, repo, commitSha) {
+  try {
+    const { data } = await github.git.getCommit({
+      owner,
+      repo,
+      commit_sha: commitSha,
+    })
+    return data.tree.sha
+  } catch (err) {
+    console.log('error getting tree')
+    throw err
+  }
+}
+
+// https://docs.github.com/rest/reference/git#get-a-tree
+export async function getTree(owner, repo, ref) {
+  const commitSha = await getCommitSha(owner, repo, ref)
+  const treeSha = await getTreeSha(owner, repo, commitSha)
+  try {
+    const { data } = await github.git.getTree({
+      owner,
+      repo,
+      tree_sha: treeSha,
+      recursive: 1,
+    })
+    // only return files that match the patterns in allowedPaths
+    // skip actions/changes files
+    return data.tree
+  } catch (err) {
+    console.log('error getting tree')
+    throw err
+  }
+}
+
+// https://docs.github.com/rest/reference/git#get-a-blob
+export async function getContentsForBlob(owner, repo, sha) {
+  const { data } = await github.git.getBlob({
+    owner,
+    repo,
+    file_sha: sha,
+  })
+  // decode blob contents
+  return Buffer.from(data.content, 'base64')
+}
+
+// https://docs.github.com/rest/reference/repos#get-repository-content
+export async function getContents(owner, repo, ref, path) {
+  try {
+    const { data } = await github.repos.getContent({
+      owner,
+      repo,
+      ref,
+      path,
+    })
+
+    if (!data.content) {
+      const blob = await getContentsForBlob(owner, repo, data.sha)
+      // decode Base64 encoded contents
+      return Buffer.from(blob, 'base64').toString()
+    }
+    // decode Base64 encoded contents
+    return Buffer.from(data.content, 'base64').toString()
+  } catch (err) {
+    console.log(`error getting ${path} from ${owner}/${repo} at ref ${ref}`)
+    throw err
+  }
+}
+
+// https://docs.github.com/en/rest/reference/pulls#list-pull-requests
+export async function listPulls(owner, repo) {
+  try {
+    const { data } = await github.pulls.list({
+      owner,
+      repo,
+      per_page: 100,
+    })
+    return data
+  } catch (err) {
+    console.log(`error listing pulls in ${owner}/${repo}`)
+    throw err
+  }
+}
+
+export async function createIssueComment(owner, repo, pullNumber, body) {
+  try {
+    const { data } = await github.issues.createComment({
+      owner,
+      repo,
+      issue_number: pullNumber,
+      body,
+    })
+    return data
+  } catch (err) {
+    console.log(`error creating a review comment on PR ${pullNumber} in ${owner}/${repo}`)
+    throw err
+  }
+}
+
+// Search for a string in a file in code and return the array of paths to files that contain string
+export async function getPathsWithMatchingStrings(
+  strArr,
+  org,
+  repo,
+  { cache = true, forceDownload = false } = {},
+) {
+  const perPage = 100
+  const paths = new Set()
+
+  for (const str of strArr) {
+    try {
+      const q = `q=${str}+in:file+repo:${org}/${repo}`
+      let currentPage = 1
+      let totalCount = 0
+      let currentCount = 0
+
+      do {
+        const data = await searchCode(q, perPage, currentPage, cache, forceDownload)
+        data.items.map((el) => paths.add(el.path))
+        totalCount = data.total_count
+        currentCount += data.items.length
+        currentPage++
+      } while (currentCount < totalCount)
+    } catch (err) {
+      console.log(`error searching for ${str} in ${org}/${repo}`)
+      throw err
+    }
+  }
+
+  return paths
+}
+
+async function searchCode(q, perPage, currentPage, cache = true, forceDownload = false) {
+  const cacheKey = `searchCode-${q}-${perPage}-${currentPage}`
+  const tempFilename = `/tmp/searchCode-${crypto
+    .createHash('md5')
+    .update(cacheKey)
+    .digest('hex')}.json`
+
+  if (!forceDownload && cache) {
+    try {
+      return JSON.parse(await fs.readFile(tempFilename, 'utf8'))
+    } catch (error) {
+      if (error.code !== 'ENOENT') {
+        throw error
+      }
+      console.log(`Cache miss on ${tempFilename} (${cacheKey})`)
+    }
+  }
+
+  try {
+    const { data } = await secondaryRateLimitRetry(github.rest.search.code, {
+      q,
+      per_page: perPage,
+      page: currentPage,
+    })
+    if (cache) {
+      await fs.writeFile(tempFilename, JSON.stringify(data))
+      console.log(`Wrote search results to ${tempFilename}`)
+    }
+
+    return data
+  } catch (err) {
+    console.log(`error searching for ${q} in code`)
+    throw err
+  }
+}
+
+async function secondaryRateLimitRetry(callable, args, maxAttempts = 10, sleepTime = 1000) {
+  try {
+    const response = await callable(args)
+    return response
+  } catch (err) {
+    // If you get a secondary rate limit error (403) you'll get a data
+    // response that includes:
+    //
+    //  {
+    //    documentation_url: 'https://docs.github.com/en/free-pro-team@latest/rest/overview/resources-in-the-rest-api#secondary-rate-limits',
+    //    message: 'You have exceeded a secondary rate limit. Please wait a few minutes before you try again.'
+    //  }
+    //
+    // Let's look for that an manually self-recurse, under certain conditions
+    const lookFor = 'You have exceeded a secondary rate limit.'
+    if (
+      err.status &&
+      err.status === 403 &&
+      err.response?.data?.message.includes(lookFor) &&
+      maxAttempts > 0
+    ) {
+      console.warn(
+        `Got secondary rate limit blocked. Sleeping for ${
+          sleepTime / 1000
+        } seconds. (attempts left: ${maxAttempts})`,
+      )
+      return new Promise((resolve) => {
+        setTimeout(() => {
+          resolve(secondaryRateLimitRetry(callable, args, maxAttempts - 1, sleepTime * 2))
+        }, sleepTime)
+      })
+    }
+
+    throw err
+  }
+}