docs/script/helpers/git-utils.js

#!/usr/bin/env node
import crypto from 'crypto'
import fs from 'fs/promises'

import Github from './github.js'
const github = Github()

// https://docs.github.com/rest/reference/git#get-a-reference
export async function getCommitSha(owner, repo, ref) {
  try {
    const { data } = await github.git.getRef({
      owner,
      repo,
      ref,
    })
    return data.object.sha
  } catch (err) {
    console.log('error getting tree')
    throw err
  }
}

// https://docs.github.com/rest/reference/git#list-matching-references
export async function listMatchingRefs(owner, repo, ref) {
  try {
    // if the ref is found, this returns an array of objects;
    // if the ref is not found, this returns an empty array
    const { data } = await github.git.listMatchingRefs({
      owner,
      repo,
      ref,
    })
    return data
  } catch (err) {
    console.log('error getting tree')
    throw err
  }
}

// https://docs.github.com/rest/reference/git#get-a-commit
export async function getTreeSha(owner, repo, commitSha) {
  try {
    const { data } = await github.git.getCommit({
      owner,
      repo,
      commit_sha: commitSha,
    })
    return data.tree.sha
  } catch (err) {
    console.log('error getting tree')
    throw err
  }
}

// https://docs.github.com/rest/reference/git#get-a-tree
export async function getTree(owner, repo, ref) {
  const commitSha = await getCommitSha(owner, repo, ref)
  const treeSha = await getTreeSha(owner, repo, commitSha)
  try {
    const { data } = await github.git.getTree({
      owner,
      repo,
      tree_sha: treeSha,
      recursive: 1,
    })
    // only return files that match the patterns in allowedPaths
    // skip actions/changes files
    return data.tree
  } catch (err) {
    console.log('error getting tree')
    throw err
  }
}

// https://docs.github.com/rest/reference/git#get-a-blob
export async function getContentsForBlob(owner, repo, sha) {
  const { data } = await github.git.getBlob({
    owner,
    repo,
    file_sha: sha,
  })
  // decode blob contents
  return Buffer.from(data.content, 'base64')
}

// https://docs.github.com/rest/reference/repos#get-repository-content
export async function getContents(owner, repo, ref, path) {
  try {
    const { data } = await github.repos.getContent({
      owner,
      repo,
      ref,
      path,
    })

    if (!data.content) {
      const blob = await getContentsForBlob(owner, repo, data.sha)
      // decode Base64 encoded contents
      return Buffer.from(blob, 'base64').toString()
    }
    // decode Base64 encoded contents
    return Buffer.from(data.content, 'base64').toString()
  } catch (err) {
    console.log(`error getting ${path} from ${owner}/${repo} at ref ${ref}`)
    throw err
  }
}

// https://docs.github.com/en/rest/reference/pulls#list-pull-requests
export async function listPulls(owner, repo) {
  try {
    const { data } = await github.pulls.list({
      owner,
      repo,
      per_page: 100,
    })
    return data
  } catch (err) {
    console.log(`error listing pulls in ${owner}/${repo}`)
    throw err
  }
}

export async function createIssueComment(owner, repo, pullNumber, body) {
  try {
    const { data } = await github.issues.createComment({
      owner,
      repo,
      issue_number: pullNumber,
      body,
    })
    return data
  } catch (err) {
    console.log(`error creating a review comment on PR ${pullNumber} in ${owner}/${repo}`)
    throw err
  }
}

// Search for a string in a file in code and return the array of paths to files that contain string
export async function getPathsWithMatchingStrings(
  strArr,
  org,
  repo,
  { cache = true, forceDownload = false } = {}
) {
  const perPage = 100
  const paths = new Set()

  for (const str of strArr) {
    try {
      const q = `q=${str}+in:file+repo:${org}/${repo}`
      let currentPage = 1
      let totalCount = 0
      let currentCount = 0

      do {
        const data = await searchCode(q, perPage, currentPage, cache, forceDownload)
        data.items.map((el) => paths.add(el.path))
        totalCount = data.total_count
        currentCount += data.items.length
        currentPage++
      } while (currentCount < totalCount)
    } catch (err) {
      console.log(`error searching for ${str} in ${org}/${repo}`)
      throw err
    }
  }

  return paths
}

async function searchCode(q, perPage, currentPage, cache = true, forceDownload = false) {
  const cacheKey = `searchCode-${q}-${perPage}-${currentPage}`
  const tempFilename = `/tmp/searchCode-${crypto
    .createHash('md5')
    .update(cacheKey)
    .digest('hex')}.json`

  if (!forceDownload && cache) {
    try {
      return JSON.parse(await fs.readFile(tempFilename, 'utf8'))
    } catch (error) {
      if (error.code !== 'ENOENT') {
        throw error
      }
      console.log(`Cache miss on ${tempFilename} (${cacheKey})`)
    }
  }

  try {
    const { data } = await secondaryRateLimitRetry(github.rest.search.code, {
      q,
      per_page: perPage,
      page: currentPage,
    })
    if (cache) {
      await fs.writeFile(tempFilename, JSON.stringify(data))
      console.log(`Wrote search results to ${tempFilename}`)
    }

    return data
  } catch (err) {
    console.log(`error searching for ${q} in code`)
    throw err
  }
}

async function secondaryRateLimitRetry(callable, args, maxAttempts = 10, sleepTime = 1000) {
  try {
    const response = await callable(args)
    return response
  } catch (err) {
    // If you get a secondary rate limit error (403) you'll get a data
    // response that includes:
    //
    //  {
    //    documentation_url: 'https://docs.github.com/en/free-pro-team@latest/rest/overview/resources-in-the-rest-api#secondary-rate-limits',
    //    message: 'You have exceeded a secondary rate limit. Please wait a few minutes before you try again.'
    //  }
    //
    // Let's look for that an manually self-recurse, under certain conditions
    const lookFor = 'You have exceeded a secondary rate limit.'
    if (
      err.status &&
      err.status === 403 &&
      err.response?.data?.message.includes(lookFor) &&
      maxAttempts > 0
    ) {
      console.warn(
        `Got secondary rate limit blocked. Sleeping for ${
          sleepTime / 1000
        } seconds. (attempts left: ${maxAttempts})`
      )
      return new Promise((resolve) => {
        setTimeout(() => {
          resolve(secondaryRateLimitRetry(callable, args, maxAttempts - 1, sleepTime * 2))
        }, sleepTime)
      })
    }

    throw err
  }
}