Do not use build.status as a looping condition for Heroku deployment (#21909)

* Do not use `build.status` of 'pending' as a looping condition for Heroku deployment
* Don't wait for `appSetup.status` either
* Fix incorrect Octokit method usage in local deploy script
* Bump the number of allowable errors from 5 to 10
* More logging!
* Add an environment variable for easily increasing the number of allowed Heroku failures per phase of polling

.github/workflows/prod-build-deploy.yml

@@ -143,6 +143,7 @@ jobs:
           HYDRO_SECRET: ${{ secrets.HYDRO_SECRET }}
           SOURCE_BLOB_URL: ${{ steps.build-source.outputs.download_url }}
           DELAY_FOR_PREBOOT: 'true'
+          ALLOWED_POLLING_FAILURES_PER_PHASE: '15'
         with:
           script: |
             const {

.github/workflows/staging-deploy-pr.yml

@@ -472,6 +472,7 @@ jobs:
           CONTEXT_NAME: ${{ env.CONTEXT_NAME }}
           ACTIONS_RUN_LOG: ${{ env.ACTIONS_RUN_LOG }}
           HEAD_SHA: ${{ needs.pr-metadata.outputs.head_sha }}
+          ALLOWED_POLLING_FAILURES_PER_PHASE: '15'
         with:
           script: |
             const { GITHUB_TOKEN, HEROKU_API_TOKEN } = process.env

script/deploy.js

@@ -234,7 +234,7 @@ async function deployStaging({ owner, repo, pullNumber, forceRebuild = false, de
         pullRequest,
       })
     } else {
-      await octokit.repos.createStatus({
+      await octokit.repos.createCommitStatus({
         owner,
         repo,
         sha: pullRequest.head.sha,
@@ -249,7 +249,7 @@ async function deployStaging({ owner, repo, pullNumber, forceRebuild = false, de
         forceRebuild,
       })
 
-      await octokit.repos.createStatus({
+      await octokit.repos.createCommitStatus({
         owner,
         repo,
         sha: pullRequest.head.sha,
@@ -264,7 +264,7 @@ async function deployStaging({ owner, repo, pullNumber, forceRebuild = false, de
     console.error(error)
 
     if (!destroy) {
-      await octokit.repos.createStatus({
+      await octokit.repos.createCommitStatus({
         owner,
         repo,
         sha: pullRequest.head.sha,

script/deployment/deploy-to-production.js

@@ -10,7 +10,8 @@ const DELAY_FOR_PREBOOT_SWAP = 135000 // 2:15
 
 // Allow for a few 404 (Not Found), 429 (Too Many Requests), etc. responses from
 // the semi-unreliable Heroku API when we're polling for status updates
-const ALLOWED_MISSING_RESPONSE_COUNT = 5
+const ALLOWED_MISSING_RESPONSE_COUNT =
+  parseInt(process.env.ALLOWED_POLLING_FAILURES_PER_PHASE, 10) || 10
 const ALLOWABLE_ERROR_CODES = [404, 429, 500, 503]
 
 export default async function deployToProduction({
@@ -175,7 +176,7 @@ export default async function deployToProduction({
 
     // Poll until the Build's status changes from "pending" to "succeeded" or "failed".
     let buildAcceptableErrorCount = 0
-    while (!build || build.status === 'pending' || !build.release || !build.release.id) {
+    while (!build || !build.release || !build.release.id) {
       await sleep(SLEEP_INTERVAL)
       try {
         build = await heroku.get(`/apps/${appName}/builds/${buildId}`)
@@ -184,6 +185,9 @@ export default async function deployToProduction({
         if (isAllowableHerokuError(error)) {
           buildAcceptableErrorCount += 1
           if (buildAcceptableErrorCount <= ALLOWED_MISSING_RESPONSE_COUNT) {
+            console.warn(
+              `Ignoring allowable Heroku error #${buildAcceptableErrorCount}: ${error.statusCode}`
+            )
             continue
           }
         }
@@ -210,6 +214,7 @@ export default async function deployToProduction({
       `Finished Heroku build after ${Math.round((Date.now() - buildStartTime) / 1000)} seconds.`,
       build
     )
+    console.log('Heroku release detected', build.release)
 
     const releaseStartTime = Date.now() // Close enough...
     const releaseId = build.release.id
@@ -237,6 +242,9 @@ export default async function deployToProduction({
         if (isAllowableHerokuError(error)) {
           releaseAcceptableErrorCount += 1
           if (releaseAcceptableErrorCount <= ALLOWED_MISSING_RESPONSE_COUNT) {
+            console.warn(
+              `Ignoring allowable Heroku error #${releaseAcceptableErrorCount}: ${error.statusCode}`
+            )
             continue
           }
         }
@@ -296,6 +304,9 @@ export default async function deployToProduction({
         if (isAllowableHerokuError(error)) {
           dynoAcceptableErrorCount += 1
           if (dynoAcceptableErrorCount <= ALLOWED_MISSING_RESPONSE_COUNT) {
+            console.warn(
+              `Ignoring allowable Heroku error #${dynoAcceptableErrorCount}: ${error.statusCode}`
+            )
             continue
           }
         }

script/deployment/deploy-to-staging.js

@@ -10,7 +10,8 @@ const HEROKU_LOG_LINES_TO_SHOW = 25
 
 // Allow for a few 404 (Not Found), 429 (Too Many Requests), etc. responses from
 // the semi-unreliable Heroku API when we're polling for status updates
-const ALLOWED_MISSING_RESPONSE_COUNT = 5
+const ALLOWED_MISSING_RESPONSE_COUNT =
+  parseInt(process.env.ALLOWED_POLLING_FAILURES_PER_PHASE, 10) || 10
 const ALLOWABLE_ERROR_CODES = [404, 429, 500, 503]
 
 export default async function deployToStaging({
@@ -233,7 +234,7 @@ export default async function deployToStaging({
       // A new Build is created as a by-product of creating an AppSetup.
       // Poll until there is a Build object attached to the AppSetup.
       let setupAcceptableErrorCount = 0
-      while (!appSetup || appSetup.status === 'pending' || !build || !build.id) {
+      while (!appSetup || !build || !build.id) {
         await sleep(SLEEP_INTERVAL)
         try {
           appSetup = await heroku.get(`/app-setups/${appSetup.id}`)
@@ -243,6 +244,9 @@ export default async function deployToStaging({
           if (isAllowableHerokuError(error)) {
             setupAcceptableErrorCount += 1
             if (setupAcceptableErrorCount <= ALLOWED_MISSING_RESPONSE_COUNT) {
+              console.warn(
+                `Ignoring allowable Heroku error #${setupAcceptableErrorCount}: ${error.statusCode}`
+              )
               continue
             }
           }
@@ -272,6 +276,7 @@ See Heroku logs for more information:\n${logUrl}`
         )
       }
 
+      console.log('Heroku AppSetup finished', appSetup)
       console.log('Heroku build detected', build)
     } else {
       // If the app does exist, just manually trigger a new build
@@ -317,7 +322,7 @@ See Heroku logs for more information:\n${logUrl}`
 
     // Poll until the Build's status changes from "pending" to "succeeded" or "failed".
     let buildAcceptableErrorCount = 0
-    while (!build || build.status === 'pending' || !build.release || !build.release.id) {
+    while (!build || !build.release || !build.release.id) {
       await sleep(SLEEP_INTERVAL)
       try {
         build = await heroku.get(`/apps/${appName}/builds/${buildId}`)
@@ -326,6 +331,9 @@ See Heroku logs for more information:\n${logUrl}`
         if (isAllowableHerokuError(error)) {
           buildAcceptableErrorCount += 1
           if (buildAcceptableErrorCount <= ALLOWED_MISSING_RESPONSE_COUNT) {
+            console.warn(
+              `Ignoring allowable Heroku error #${buildAcceptableErrorCount}: ${error.statusCode}`
+            )
             continue
           }
         }
@@ -352,6 +360,7 @@ See Heroku logs for more information:\n${logUrl}`
       `Finished Heroku build after ${Math.round((Date.now() - buildStartTime) / 1000)} seconds.`,
       build
     )
+    console.log('Heroku release detected', build.release)
 
     const releaseStartTime = Date.now() // Close enough...
     let releaseId = build.release.id
@@ -379,6 +388,9 @@ See Heroku logs for more information:\n${logUrl}`
         if (isAllowableHerokuError(error)) {
           releaseAcceptableErrorCount += 1
           if (releaseAcceptableErrorCount <= ALLOWED_MISSING_RESPONSE_COUNT) {
+            console.warn(
+              `Ignoring allowable Heroku error #${releaseAcceptableErrorCount}: ${error.statusCode}`
+            )
             continue
           }
         }
@@ -485,6 +497,9 @@ See Heroku logs for more information:\n${logUrl}`
         if (isAllowableHerokuError(error)) {
           dynoAcceptableErrorCount += 1
           if (dynoAcceptableErrorCount <= ALLOWED_MISSING_RESPONSE_COUNT) {
+            console.warn(
+              `Ignoring allowable Heroku error #${dynoAcceptableErrorCount}: ${error.statusCode}`
+            )
             continue
           }
         }