diff --git a/.dockerignore b/.dockerignore index c7ac7e4e8e..bf645012d0 100644 --- a/.dockerignore +++ b/.dockerignore @@ -1,6 +1,12 @@ +.devcontainer/ .git/ -node_modules/ .github/ .vscode/ +contributing/ docs/ -script/ \ No newline at end of file +node_modules/ +script/ +tests/ +lib/rest/static/dereferenced +# Folder is cloned during the preview + prod workflows, the assets are merged into other locations for use before the build +docs-early-access/ diff --git a/.github/actions-scripts/merge-early-access.sh b/.github/actions-scripts/merge-early-access.sh new file mode 100755 index 0000000000..fb126ccf3a --- /dev/null +++ b/.github/actions-scripts/merge-early-access.sh @@ -0,0 +1,11 @@ +#!/usr/bin/env bash + +# [start-readme] +# +# This script takes docs-early-access files and merges them into docs-internal +# +# [end-readme] + +mv docs-early-access/assets assets/images/early-access +mv docs-early-access/content content/early-access +mv docs-early-access/data data/early-access diff --git a/.github/actions-scripts/prune-for-preview-env.sh b/.github/actions-scripts/prune-for-preview-env.sh new file mode 100755 index 0000000000..6126c13b7e --- /dev/null +++ b/.github/actions-scripts/prune-for-preview-env.sh @@ -0,0 +1,24 @@ +#!/usr/bin/env bash + +# [start-readme] +# +# This script removes files that are unnecessary for our preview environments. +# This is typically run before a docker build to reduce the size of the build context sent to docker +# +# [end-readme] + +# Remove all but the english search indexes +find lib/search/indexes ! -name '*-en.json.br' ! -name '*-en-records.json.br' -maxdepth 1 -type f -delete + +# Translations are never tested in preview environments +# but let's keep the empty directory. +rm -rf translations +mkdir translations + +# The assumption here is that a preview build will not +# need these legacy redirects. Only the redirects from +# front-matter will be at play. +# These static redirects json files are notoriously large +echo '[]' > lib/redirects/static/archived-frontmatter-fallbacks.json +echo '{}' > lib/redirects/static/developer.json +echo '{}' > lib/redirects/static/archived-redirects-from-213-to-217.json diff --git a/.github/workflows/main-docker-cache.yml b/.github/workflows/main-preview-docker-cache.yml similarity index 62% rename from .github/workflows/main-docker-cache.yml rename to .github/workflows/main-preview-docker-cache.yml index 8f594be9eb..dcfe532f64 100644 --- a/.github/workflows/main-docker-cache.yml +++ b/.github/workflows/main-preview-docker-cache.yml @@ -1,4 +1,4 @@ -name: Build and Push Main Docker Cache +name: Build and Push Main Preview Env Docker Cache # **What it does**: Builds and pushes the `main` Docker cache image # **Why we have it**: It allows PRs using the registry cache to pull a pre-built image, which should speed up the build @@ -23,9 +23,10 @@ jobs: runs-on: ubuntu-latest timeout-minutes: 15 env: + ENABLE_EARLY_ACCESS: ${{ github.repository == 'github/docs-internal' }} NONPROD_REGISTRY_USERNAME: ghdocs NONPROD_REGISTRY_NAME: ghdocs - DOCKER_IMAGE_MAIN_REF: ${{ secrets.NONPROD_REGISTRY_SERVER }}/${{ github.repository }}:main + DOCKER_IMAGE_CACHE_REF: ${{ secrets.NONPROD_REGISTRY_SERVER }}/${{ github.repository }}:main-preview steps: - name: 'Az CLI login' @@ -53,19 +54,29 @@ jobs: - name: Check out LFS objects run: git lfs checkout - - if: ${{ github.repository == 'github/docs-internal' }} - name: Clone early access - env: - DOCUBOT_REPO_PAT: ${{ secrets.DOCUBOT_REPO_PAT }} - GIT_BRANCH: main - run: npm install dotenv && node script/early-access/clone-for-build.js + - if: ${{ env.ENABLE_EARLY_ACCESS }} + name: Clone docs-early-access + uses: actions/checkout@ec3a7ce113134d7a93b817d10a8272cb61118579 + with: + repository: github/docs-early-access + token: ${{ secrets.DOCUBOT_REPO_PAT }} + path: docs-early-access + ref: main + + - if: ${{ env.ENABLE_EARLY_ACCESS }} + name: Merge docs-early-access repo's folders + run: .github/actions-scripts/merge-early-access.sh + + # In addition to making the final image smaller, we also save time by not sending unnecessary files to the docker build context + - name: 'Prune for preview env' + run: .github/actions-scripts/prune-for-preview-env.sh - name: 'Build and push image' uses: docker/build-push-action@a66e35b9cbcf4ad0ea91ffcaf7bbad63ad9e0229 with: context: . push: true - target: ${{ fromJSON('["production", "production_early_access"]')[github.repository == 'github/docs-internal'] }} - tags: ${{ env.DOCKER_IMAGE_MAIN_REF }} - cache-from: type=registry,ref=${{ env.DOCKER_IMAGE_MAIN_REF }} - cache-to: type=registry,mode=max,ref=${{ env.DOCKER_IMAGE_MAIN_REF }} + target: preview + tags: ${{ env.DOCKER_IMAGE_CACHE_REF }} + cache-from: type=registry,ref=${{ env.DOCKER_IMAGE_CACHE_REF }} + cache-to: type=registry,mode=max,ref=${{ env.DOCKER_IMAGE_CACHE_REF }} diff --git a/.github/workflows/prod-build-deploy-azure.yml b/.github/workflows/prod-build-deploy-azure.yml index 3e3b3ee7ed..a44b36eba5 100644 --- a/.github/workflows/prod-build-deploy-azure.yml +++ b/.github/workflows/prod-build-deploy-azure.yml @@ -30,6 +30,7 @@ jobs: url: 'https://docs.github.com' env: DOCKER_IMAGE: ${{ secrets.PROD_REGISTRY_SERVER }}/${{ github.repository }}:${{ github.sha }} + DOCKER_IMAGE_CACHE_REF: ${{ secrets.PROD_REGISTRY_SERVER }}/${{ github.repository }}:main-production steps: - name: 'Az CLI login' @@ -64,21 +65,26 @@ jobs: node-version: 16.13.x cache: npm - - name: Clone early access - run: npm install dotenv && node script/early-access/clone-for-build.js - env: - DOCUBOT_REPO_PAT: ${{ secrets.DOCUBOT_REPO_PAT }} - GIT_BRANCH: main + - name: Clone docs-early-access + uses: actions/checkout@ec3a7ce113134d7a93b817d10a8272cb61118579 + with: + repository: github/docs-early-access + token: ${{ secrets.DOCUBOT_REPO_PAT }} + path: docs-early-access + ref: main + + - name: Merge docs-early-access repo's folders + run: .github/actions-scripts/merge-early-access.sh - name: 'Build and push image' uses: docker/build-push-action@1814d3dfb36d6f84174e61f4a4b05bd84089a4b9 with: context: . push: true - target: 'production_early_access' - tags: ${{ env.DOCKER_IMAGE }} - cache-from: type=gha - cache-to: type=gha,mode=max + target: production + tags: ${{ env.DOCKER_IMAGE }}, ${{ env.DOCKER_IMAGE_CACHE_REF }} + cache-from: type=registry,ref=${{ env.DOCKER_IMAGE_CACHE_REF }} + cache-to: type=registry,mode=max,ref=${{ env.DOCKER_IMAGE_CACHE_REF }} - name: 'Update docker-compose.prod.yaml template file' run: | diff --git a/.github/workflows/staging-build-and-deploy-azure.yml b/.github/workflows/staging-build-and-deploy-azure.yml index 3f7a13498f..2052c63d33 100644 --- a/.github/workflows/staging-build-and-deploy-azure.yml +++ b/.github/workflows/staging-build-and-deploy-azure.yml @@ -40,6 +40,7 @@ jobs: GITHUB_EVENT_NUMBER: ${{ github.event.number }} STAGING_RESOURCE_GROUPS: 4 NONPROD_REGISTRY_USERNAME: ghdocs + ENABLE_EARLY_ACCESS: ${{ github.repository == 'github/docs-internal' }} # Image tag is unique to each workflow run so that it always triggers a new deployment DOCKER_IMAGE: ${{ secrets.NONPROD_REGISTRY_SERVER }}/${{ github.repository }}/pr-${{ github.event.number }}:${{ github.event.pull_request.head.sha }}-${{ github.run_number }}-${{ github.run_attempt }} @@ -87,24 +88,64 @@ jobs: - name: Check out LFS objects run: git lfs checkout - - if: ${{ github.repository == 'github/docs-internal' }} - name: Clone early access + - if: ${{ env.ENABLE_EARLY_ACCESS }} + name: Determine which docs-early-access branch to clone + id: 'check-early-access' + uses: actions/github-script@2b34a689ec86a68d8ab9478298f91d5401337b7d env: - DOCUBOT_REPO_PAT: ${{ secrets.DOCUBOT_REPO_PAT }} - GIT_BRANCH: ${{ github.event.pull_request.head.sha }} - run: npm install dotenv && node script/early-access/clone-for-build.js + BRANCH_NAME: ${{ github.head_ref || github.ref_name }} + with: + github-token: ${{ secrets.DOCUBOT_REPO_PAT }} + result-encoding: string + script: | + const { BRANCH_NAME } = process.env + + try { + const { status } = await github.request('GET /repos/{owner}/{repo}/branches/{branch}', { + owner: 'github', + repo: 'docs-early-access', + branch: BRANCH_NAME, + }) + + if (status !== 200) { + throw new Error('Received non-200 response from branch GET request') + } + + console.log(`Using docs-early-access branch '${BRANCH_NAME}'`) + return BRANCH_NAME + } catch (e) { + console.log(`Failed to get docs-early-access branch '${BRANCH_NAME}', 'main' will be used instead.`) + return 'main' + } + + - if: ${{ env.ENABLE_EARLY_ACCESS }} + name: Clone docs-early-access + uses: actions/checkout@ec3a7ce113134d7a93b817d10a8272cb61118579 + with: + repository: github/docs-early-access + token: ${{ secrets.DOCUBOT_REPO_PAT }} + path: docs-early-access + ref: ${{ steps.check-early-access.outputs.result }} + + - if: ${{ env.ENABLE_EARLY_ACCESS }} + name: Merge docs-early-access repo's folders + run: .github/actions-scripts/merge-early-access.sh + + # In addition to making the final image smaller, we also save time by not sending unnecessary files to the docker build context + - name: 'Prune for preview env' + run: .github/actions-scripts/prune-for-preview-env.sh - name: 'Build and push image' uses: docker/build-push-action@1814d3dfb36d6f84174e61f4a4b05bd84089a4b9 with: context: . push: true - target: ${{ fromJSON('["production", "production_early_access"]')[github.repository == 'github/docs-internal'] }} + target: preview tags: ${{ env.DOCKER_IMAGE }} # we only pull the `main` cache image cache-from: | type=local,src=/tmp/.buildx-cache - type=registry,ref=${{ secrets.NONPROD_REGISTRY_SERVER }}/${{ github.repository }}:main + type=registry,ref=${{ secrets.NONPROD_REGISTRY_SERVER }}/${{ github.repository }}:main-preview # `main-docker-cache.yml` handles updating the remote cache so we don't pollute it with PR specific code cache-to: type=local,dest=/tmp/.buildx-cache-new,mode=max diff --git a/Dockerfile b/Dockerfile index 1be06838ea..88e861a470 100644 --- a/Dockerfile +++ b/Dockerfile @@ -59,12 +59,12 @@ COPY next-env.d.ts ./next-env.d.ts RUN npm run build # -------------------------------------------------------------------------------- -# MAIN IMAGE +# PREVIEW IMAGE - no translations # -------------------------------------------------------------------------------- -FROM base as production +FROM base as preview -# Copy just our prod dependencies +# Copy just prod dependencies COPY --chown=node:node --from=prod_deps $APP_HOME/node_modules $APP_HOME/node_modules # Copy our front-end code @@ -79,14 +79,15 @@ ENV AIRGAP false # By default we typically don't want to run in clustered mode ENV WEB_CONCURRENCY 1 -# This makes sure server.mjs always picks up the preferred port +# Preferred port for server.mjs ENV PORT 4000 +ENV ENABLED_LANGUAGES "en" + # Copy only what's needed to run the server COPY --chown=node:node package.json ./ COPY --chown=node:node assets ./assets COPY --chown=node:node includes ./includes -COPY --chown=node:node translations ./translations COPY --chown=node:node content ./content COPY --chown=node:node lib ./lib COPY --chown=node:node middleware ./middleware @@ -99,13 +100,10 @@ EXPOSE $PORT CMD ["node", "server.mjs"] - # -------------------------------------------------------------------------------- -# MAIN IMAGE WITH EARLY ACCESS +# PRODUCTION IMAGE - includes all translations # -------------------------------------------------------------------------------- +FROM preview as production -FROM production as production_early_access - -COPY --chown=node:node content/early-access ./content/early-access - -CMD ["node", "server.mjs"] +# Copy in all translations +COPY --chown=node:node translations ./translations diff --git a/staging-azure-deploy-template.json b/staging-azure-deploy-template.json index 127f9c58a2..b99cc79cd2 100644 --- a/staging-azure-deploy-template.json +++ b/staging-azure-deploy-template.json @@ -86,7 +86,7 @@ }, { "name": "ENABLED_LANGUAGES", - "value": "en,ja" + "value": "en" } ], "linuxFxVersion": "[parameters('linuxFxVersion')]",