From cdafb46e4613a7c06667382b74c2e1a501ecba8c Mon Sep 17 00:00:00 2001 From: Hector Alfaro Date: Tue, 14 Nov 2023 11:04:46 -0500 Subject: [PATCH] Move script to check state of canary slots to its own file (#46057) Co-authored-by: Peter Bengtsson --- .github/workflows/azure-prod-build-deploy.yml | 65 ++----------------- .../workflows/azure-staging-build-deploy.yml | 44 +------------ src/workflows/check-canary-slots.js | 62 ++++++++++++++++++ 3 files changed, 71 insertions(+), 100 deletions(-) create mode 100755 src/workflows/check-canary-slots.js diff --git a/.github/workflows/azure-prod-build-deploy.yml b/.github/workflows/azure-prod-build-deploy.yml index 0df70dea49..04f7a118f7 100644 --- a/.github/workflows/azure-prod-build-deploy.yml +++ b/.github/workflows/azure-prod-build-deploy.yml @@ -31,6 +31,9 @@ jobs: env: DOCKER_IMAGE: ${{ secrets.PROD_REGISTRY_SERVER }}/${{ github.repository }}:${{ github.sha }} DOCKER_IMAGE_CACHE_REF: ${{ secrets.PROD_REGISTRY_SERVER }}/${{ github.repository }}:main-production + RESOURCE_GROUP_NAME: docs-prod + APP_SERVICE_NAME: ghdocs-prod + SLOT_NAME: canary steps: - name: 'Az CLI login' @@ -97,75 +100,19 @@ jobs: - name: 'Apply updated docker-compose.prod.yaml config to canary slot' run: | - az webapp config container set --multicontainer-config-type COMPOSE --multicontainer-config-file docker-compose.prod.yaml --slot canary -n ghdocs-prod -g docs-prod + az webapp config container set --multicontainer-config-type COMPOSE --multicontainer-config-file docker-compose.prod.yaml --slot ${{ env.SLOT_NAME }} -n ${{ env.APP_SERVICE_NAME }} -g ${{ env.RESOURCE_GROUP_NAME }} # Watch canary slot instances to see when all the instances are ready - name: Check that canary slot is ready - uses: actions/github-script@98814c53be79b1d30f795b907e553d8679345975 env: CHECK_INTERVAL: 10000 EXPECTED_SHA: ${{ github.sha }} CANARY_BUILD_URL: https://ghdocs-prod-canary.azurewebsites.net/_build - with: - script: | - const { execSync } = require('child_process') - - const getBuildSha = (timeoutSeconds = 5) => { - const url = process.env.CANARY_BUILD_URL; - console.log(`Fetching ${url}`); - const t0 = Date.now(); - try { - const o = execSync(`curl --connect-timeout ${timeoutSeconds} ${url}`, { - encoding: "utf8", - }); - console.log(`Fetched ${url}. Took ${Date.now() - t0}ms`); - return o.toString().trim(); - } catch (err) { - console.log(`Error fetching build sha from ${url}`); - return null; - } - }; - - const getStatesForSlot = (slot) => { - return JSON.parse( - execSync( - `az webapp list-instances --slot ${slot} --query "[].state" -n ghdocs-prod -g docs-prod`, - { encoding: 'utf8' } - ) - ) - } - - const waitDuration = parseInt(process.env.CHECK_INTERVAL, 10) || 10000 - let attempts = 0 - async function doCheck() { - attempts++ - console.log('Attempt:', attempts); - const buildSha = getBuildSha(); - console.log("Canary build SHA:", buildSha || "*unknown/failed*", "Expected SHA:", process.env.EXPECTED_SHA) - - const states = getStatesForSlot('canary') - console.log('Instance states:', states) - - const isAllReady = states.every((s) => s === 'READY') - - if (buildSha === process.env.EXPECTED_SHA && isAllReady) { - process.exit(0) // success - } - - if (attempts * waitDuration > 10 * 60 * 1000) { - console.log(`Giving up after a total of ${(attempts * waitDuration)/1000} seconds`) - process.exit(1) // failure - } - - console.log(`checking again in ${waitDuration}ms`) - setTimeout(doCheck, waitDuration) - } - - doCheck() + run: src/workflows/check-canary-slots.js - name: 'Swap canary slot to production' run: | - az webapp deployment slot swap --slot canary --target-slot production -n ghdocs-prod -g docs-prod + az webapp deployment slot swap --slot ${{ env.SLOT_NAME }} --target-slot production -n ${{ env.APP_SERVICE_NAME }} -g ${{ env.RESOURCE_GROUP_NAME }} - uses: ./.github/actions/slack-alert if: ${{ failure() && github.event_name != 'workflow_dispatch' }} diff --git a/.github/workflows/azure-staging-build-deploy.yml b/.github/workflows/azure-staging-build-deploy.yml index 0e19ff7264..e8d302c0aa 100644 --- a/.github/workflows/azure-staging-build-deploy.yml +++ b/.github/workflows/azure-staging-build-deploy.yml @@ -110,49 +110,11 @@ jobs: # Watch deployment slot instances to see when all the instances are ready - name: Check that deployment slot is ready - uses: actions/github-script@98814c53be79b1d30f795b907e553d8679345975 env: CHECK_INTERVAL: 10000 - with: - script: | - const { execSync } = require('child_process') - - const slotName = process.env.SLOT_NAME - const appServiceName = process.env.APP_SERVICE_NAME - const resourceGroupName = process.env.RESOURCE_GROUP_NAME - - const getStatesForSlot = (slot, appService, resourceGroup) => { - return JSON.parse( - execSync( - `az webapp list-instances --slot ${slot} --query "[].state" -n ${appService} -g ${resourceGroup}`, - { encoding: 'utf8' } - ) - ) - } - - let hasStopped = false - const waitDuration = parseInt(process.env.CHECK_INTERVAL, 10) || 10000 - async function doCheck() { - const states = getStatesForSlot(slotName, appServiceName, resourceGroupName) - console.log(`Instance states:`, states) - - // We must wait until at-least 1 instance has STOPPED to know we're looking at the "next" deployment and not the "previous" one - // That way we don't immediately succeed just because all the previous instances were READY - if (!hasStopped) { - hasStopped = states.some((s) => s === 'STOPPED') - } - - const isAllReady = states.every((s) => s === 'READY') - - if (hasStopped && isAllReady) { - process.exit(0) // success - } - - console.log(`checking again in ${waitDuration}ms`) - setTimeout(doCheck, waitDuration) - } - - doCheck() + EXPECTED_SHA: ${{ github.sha }} + CANARY_BUILD_URL: https://ghdocs-staging-canary.azurewebsites.net/_build + run: src/workflows/check-canary-slots.js - name: 'Swap deployment slot to production' run: | diff --git a/src/workflows/check-canary-slots.js b/src/workflows/check-canary-slots.js new file mode 100755 index 0000000000..4474ad0c6d --- /dev/null +++ b/src/workflows/check-canary-slots.js @@ -0,0 +1,62 @@ +#!/usr/bin/env node +import { execSync } from 'child_process' + +const slotName = process.env.SLOT_NAME +const appServiceName = process.env.APP_SERVICE_NAME +const resourceGroupName = process.env.RESOURCE_GROUP_NAME +const url = process.env.CANARY_BUILD_URL +const expectedSHA = process.env.EXPECTED_SHA +const waitDuration = parseInt(process.env.CHECK_INTERVAL, 10) || 10000 +const curlConnectTimeoutSeconds = parseInt(process.env.CURL_CONNECT_TIMEOUT_SECONDS || '5', 10) +const maxWaitingTimeSeconds = parseInt(process.MAX_WAITING_TIME || 10 * 60 * 1000, 10) + +function getBuildSha() { + console.log(`Fetching ${url}`) + const t0 = Date.now() + try { + const o = execSync(`curl --connect-timeout ${curlConnectTimeoutSeconds} ${url}`, { + encoding: 'utf8', + }) + console.log(`Fetched ${url}. Took ${Date.now() - t0}ms`) + return o.toString().trim() + } catch (err) { + console.log(`Error fetching build sha from ${url}`) + return null + } +} + +function getStatesForSlot(slot, appService, resourceGroup) { + return JSON.parse( + execSync( + `az webapp list-instances --slot ${slot} --query "[].state" -n ${appService} -g ${resourceGroup}`, + { encoding: 'utf8' }, + ), + ) +} + +let attempts = 0 +async function doCheck() { + attempts++ + console.log('Attempt:', attempts) + const buildSha = getBuildSha() + console.log('Canary build SHA:', buildSha || '*unknown/failed*', 'Expected SHA:', expectedSHA) + + const states = getStatesForSlot(slotName, appServiceName, resourceGroupName) + console.log('Instance states:', states) + + const isAllReady = states.every((s) => s === 'READY') + + if (buildSha === expectedSHA && isAllReady) { + console.log('Got the expected build SHA and all slots are ready! 🚀') + return + } + + if (attempts * waitDuration > maxWaitingTimeSeconds) { + throw new Error(`Giving up after a total of ${(attempts * waitDuration) / 1000} seconds`) + } + + console.log(`checking again in ${waitDuration}ms`) + setTimeout(doCheck, waitDuration) +} + +doCheck()