From eed068d0587292c522b4ff92b49cbc2bc27af60d Mon Sep 17 00:00:00 2001 From: Kevin Heis Date: Thu, 20 Feb 2025 10:44:49 -0800 Subject: [PATCH] Shave a minute off of Docker image build (#54478) --- .dockerignore | 3 +- .gitignore | 3 - Dockerfile | 187 +++++++++++++++--------------------- contributing/deployments.md | 4 +- package-lock.json | 45 --------- package.json | 6 -- 6 files changed, 81 insertions(+), 167 deletions(-) diff --git a/.dockerignore b/.dockerignore index 0fabbd3418..213309047b 100644 --- a/.dockerignore +++ b/.dockerignore @@ -8,5 +8,4 @@ node_modules/ tests/ # Folder is cloned during the preview + prod workflows, the assets are merged into other locations for use before the build docs-early-access/ -# During the preview deploy untrusted user code may be cloned into this directory -user-code/ +README.md diff --git a/.gitignore b/.gitignore index 0e177c3f5a..1956484624 100644 --- a/.gitignore +++ b/.gitignore @@ -27,9 +27,6 @@ broken_links.md # still have these files on their disk. lib/redirects/.redirects-cache*.json -# During the preview deploy untrusted user code may be cloned into this directory -# We ignore it from git to keep things deterministic -user-code/ # Logs from scripts */logs/ diff --git a/Dockerfile b/Dockerfile index b228d90113..72ff2924fe 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,44 +1,46 @@ # This Dockerfile is used solely for production deployments to Moda -# For staging deployments, see src/deployments/staging/Dockerfile # For building this file locally, see src/deployments/production/README.md +# Environment variables are set in the Moda configuration: +# config/moda/configuration/*/env.yaml -# -------------------------------------------------------------------------------- -# BASE IMAGE -# -------------------------------------------------------------------------------- +# --------------------------------------------------------------- +# BASE STAGE: Install linux dependencies and set up the node user +# --------------------------------------------------------------- # To update the sha: # https://github.com/github/gh-base-image/pkgs/container/gh-base-image%2Fgh-base-noble FROM ghcr.io/github/gh-base-image/gh-base-noble:20250131-172559-g0fd5a2edc AS base +# Install curl for Node install and determining the early access branch # Install git for cloning docs-early-access & translations repos -# Install curl for determining the early access branch -RUN apt-get -qq update && apt-get -qq install --no-install-recommends git curl - # Install Node.js latest LTS # https://github.com/nodejs/release#release-schedule # Ubuntu's apt-get install nodejs is _very_ outdated -RUN curl -sL https://deb.nodesource.com/setup_22.x | bash - -RUN apt-get install -y nodejs -RUN node --version +# Must run as root +RUN apt-get -qq update && apt-get -qq install --no-install-recommends curl git \ + && curl -sL https://deb.nodesource.com/setup_22.x | bash - \ + && apt-get install -y nodejs \ + && node --version -# This directory is owned by the node user -RUN useradd -ms /bin/bash node -ARG APP_HOME=/home/node/app -RUN mkdir -p $APP_HOME && chown -R node:node $APP_HOME +# Create the node user and home directory +ARG APP_HOME="/home/node/app" # Define in base so all child stages inherit it +RUN useradd -ms /bin/bash node \ + && mkdir -p $APP_HOME && chown -R node:node $APP_HOME + +# ----------------------------------------------------------------- +# CLONES STAGE: Clone docs-internal, early-access, and translations +# ----------------------------------------------------------------- +FROM base AS clones +USER node:node WORKDIR $APP_HOME -# Switch to root to ensure we have permissions to copy, chmod, and install -USER root - -# Copy in build scripts -COPY src/deployments/production/build-scripts/*.sh ./build-scripts/ - -# Make scripts executable -RUN chmod +x build-scripts/*.sh - # We need to copy over content that will be merged with early-access -COPY content ./content -COPY assets ./assets -COPY data ./data +COPY --chown=node:node content content/ +COPY --chown=node:node assets assets/ +COPY --chown=node:node data data/ + +# Copy in build scripts and make them executable +COPY --chown=node:node --chmod=+x \ + src/deployments/production/build-scripts/*.sh build-scripts/ # Use the mounted --secret to: # - 1. Fetch the docs-internal repo @@ -46,111 +48,77 @@ COPY data ./data # - 3. Fetch each translations repo to the repo/translations directory # We use --mount-type=secret to avoid the secret being copied into the image layers for security # The secret passed via --secret can only be used in this RUN command -RUN --mount=type=secret,id=DOCS_BOT_PAT_READPUBLICKEY \ +RUN --mount=type=secret,id=DOCS_BOT_PAT_READPUBLICKEY,mode=0444 \ # We don't cache because Docker can't know if we need to fetch new content from remote repos echo "Don't cache this step by printing date: $(date)" && \ . ./build-scripts/fetch-repos.sh -# Give node user access to the copied content since we cloned as root -RUN chown -R node:node $APP_HOME/content -RUN chown -R node:node $APP_HOME/assets -RUN chown -R node:node $APP_HOME/data -# Give node user access to translations repos -RUN chown -R node:node $APP_HOME/translations - -# Change back to node to make sure we don't run anything as the root user -USER node - -# --------------- -# ALL DEPS Image -# --------------- -FROM base AS all_deps - -ARG APP_HOME=/home/node/app -USER node +# ----------------------------------------- +# DEPENDENCIES STAGE: Install node packages +# ----------------------------------------- +FROM base AS dependencies +USER node:node WORKDIR $APP_HOME # Copy what is needed to run npm ci COPY --chown=node:node package.json package-lock.json ./ -RUN npm ci --no-optional --registry https://registry.npmjs.org/ +RUN npm ci --omit=optional --registry https://registry.npmjs.org/ -# --------------- -# BUILDER Image -# --------------- -FROM all_deps AS builder - -ARG APP_HOME=/home/node/app -USER node +# ----------------------------------------- +# BUILD STAGE: Prepare for production stage +# ----------------------------------------- +FROM base AS build +USER node:node WORKDIR $APP_HOME -# Copy what is needed to: -# 1. Build the app -# 2. run warmup-remotejson script -# 3. run precompute-pageinfo script -# Dependencies -COPY --chown=node:node --from=all_deps $APP_HOME/node_modules $APP_HOME/node_modules -# Content with merged early-access content -COPY --chown=node:node --from=base $APP_HOME/data ./data -COPY --chown=node:node --from=base $APP_HOME/assets ./assets -COPY --chown=node:node --from=base $APP_HOME/content ./content # Source code -COPY --chown=node:node --from=all_deps $APP_HOME/package.json ./ -COPY src ./src -COPY next.config.js ./ -COPY tsconfig.json ./ +COPY --chown=node:node src src/ +COPY --chown=node:node package.json ./ +COPY --chown=node:node next.config.js ./ +COPY --chown=node:node tsconfig.json ./ -# 1. Build -RUN npm run build +# From the clones stage +COPY --chown=node:node --from=clones $APP_HOME/data data/ +COPY --chown=node:node --from=clones $APP_HOME/assets assets/ +COPY --chown=node:node --from=clones $APP_HOME/content content/ +COPY --chown=node:node --from=clones $APP_HOME/translations translations/ -# 2. Warm up the remotejson cache -RUN npm run warmup-remotejson +# From the dependencies stage +COPY --chown=node:node --from=dependencies $APP_HOME/node_modules node_modules/ -# 3. Precompute the pageinfo cache -RUN npm run precompute-pageinfo -- --max-versions 2 +# Generate build files +RUN npm run build \ + && npm run warmup-remotejson \ + && npm run precompute-pageinfo -- --max-versions 2 \ + && npm prune --production -# Prune deps for prod image -RUN npm prune --production - -# -------------------------------------------------------------------------------- -# PRODUCTION IMAGE -# -------------------------------------------------------------------------------- +# ------------------------------------------------- +# PRODUCTION STAGE: What will run on the containers +# ------------------------------------------------- FROM base AS production - -ARG APP_HOME=/home/node/app -USER node +USER node:node WORKDIR $APP_HOME -# Copy the content with merged early-access content -COPY --chown=node:node --from=base $APP_HOME/data ./data -COPY --chown=node:node --from=base $APP_HOME/assets ./assets -COPY --chown=node:node --from=base $APP_HOME/content ./content +# Source code +COPY --chown=node:node src src/ +COPY --chown=node:node package.json ./ +COPY --chown=node:node next.config.js ./ +COPY --chown=node:node tsconfig.json ./ -# Include cloned translations -COPY --chown=node:node --from=base $APP_HOME/translations ./translations +# From clones stage +COPY --chown=node:node --from=clones $APP_HOME/data data/ +COPY --chown=node:node --from=clones $APP_HOME/assets assets/ +COPY --chown=node:node --from=clones $APP_HOME/content content/ +COPY --chown=node:node --from=clones $APP_HOME/translations translations/ -# Copy prod dependencies -COPY --chown=node:node --from=builder $APP_HOME/package.json ./ -COPY --chown=node:node --from=builder $APP_HOME/node_modules $APP_HOME/node_modules +# From dependencies stage (*modified in build stage) +COPY --chown=node:node --from=build $APP_HOME/node_modules node_modules/ -# Copy built artifacts needed at runtime for the server -COPY --chown=node:node --from=builder $APP_HOME/.next $APP_HOME/.next - -# Copy cache files generated during build scripts -COPY --chown=node:node --from=builder $APP_HOME/.remotejson-cache ./.remotejson-cache -COPY --chown=node:node --from=builder $APP_HOME/.pageinfo-cache.json.br* ./.pageinfo-cache.json.br - -# Copy only what's needed to run the server -COPY --chown=node:node --from=builder $APP_HOME/src ./src -COPY --chown=node:node --from=builder $APP_HOME/.remotejson-cache ./.remotejson-cache -COPY --chown=node:node --from=builder $APP_HOME/.pageinfo-cache.json.br* ./.pageinfo-cache.json.br -COPY --chown=node:node --from=builder $APP_HOME/next.config.js ./ -COPY --chown=node:node --from=builder $APP_HOME/tsconfig.json ./ - -# - - - -# Environment variables are set in the Moda -# configuration: config/moda/configuration/*/env.yaml -# - - - +# From build stage +COPY --chown=node:node --from=build $APP_HOME/.next .next/ +COPY --chown=node:node --from=build $APP_HOME/.remotejson-cache ./ +COPY --chown=node:node --from=build $APP_HOME/.pageinfo-cache.json.br* ./ # This makes it possible to set `--build-arg BUILD_SHA=abc123` # and it then becomes available as an environment variable in the docker run. @@ -158,5 +126,6 @@ ARG BUILD_SHA ENV BUILD_SHA=$BUILD_SHA # Entrypoint to start the server -# Note: Currently we have to use tsx because we have a mix of `.ts` and `.js` files with multiple import patterns +# Note: Currently we have to use tsx because +# we have a mix of `.ts` and `.js` files with multiple import patterns CMD ["node_modules/.bin/tsx", "src/frame/server.ts"] diff --git a/contributing/deployments.md b/contributing/deployments.md index 034e052ae9..068a41b8b7 100644 --- a/contributing/deployments.md +++ b/contributing/deployments.md @@ -2,9 +2,9 @@ Staging and production deployments are automated by a deployer service created and maintained by @github/docs-engineering. -### Preview deployments +### Review deployments -When a pull request contains only content changes, it can be previewed without a deployment. Code changes will require a deployment. GitHub Staff can deploy such a PR to a staging environment. +TBD ### Production deployments diff --git a/package-lock.json b/package-lock.json index 6d19814670..18c7449656 100644 --- a/package-lock.json +++ b/package-lock.json @@ -114,7 +114,6 @@ "@types/connect-timeout": "0.0.39", "@types/cookie": "0.6.0", "@types/cookie-parser": "1.4.7", - "@types/elasticsearch": "^5.0.43", "@types/event-to-promise": "^0.7.5", "@types/express": "4.17.21", "@types/imurmurhash": "^0.1.4", @@ -150,18 +149,13 @@ "http-status-code": "^2.1.0", "husky": "^9.1.4", "json-schema-merge-allof": "^0.8.1", - "kill-port": "2.0.1", "lint-staged": "^15.2.9", "markdownlint": "^0.34.0", "markdownlint-rule-search-replace": "^1.2.0", - "mdast-util-gfm-table": "^2.0.0", - "micromark-extension-gfm-table": "^2.0.0", "mkdirp": "^3.0.0", "mockdate": "^3.0.5", "nock": "^14.0.0", "nodemon": "3.1.3", - "npm-merge-driver-install": "^3.0.0", - "nth-check": "2.1.1", "prettier": "^3.3.2", "rimraf": "^6.0.0", "robots-parser": "^3.0.0", @@ -4114,12 +4108,6 @@ "@types/ms": "*" } }, - "node_modules/@types/elasticsearch": { - "version": "5.0.43", - "resolved": "https://registry.npmjs.org/@types/elasticsearch/-/elasticsearch-5.0.43.tgz", - "integrity": "sha512-N+MpzURpDCWd7zaJ7CE1aU+nBSeAABLhDE0lGodQ0LLftx7ku6hjTXLr9OAFZLSXiWL3Xxx8jts485ynrcm5NA==", - "dev": true - }, "node_modules/@types/estree": { "version": "1.0.6", "resolved": "https://registry.npmjs.org/@types/estree/-/estree-1.0.6.tgz", @@ -8244,11 +8232,6 @@ "url": "https://github.com/sponsors/ljharb" } }, - "node_modules/get-them-args": { - "version": "1.3.2", - "dev": true, - "license": "MIT" - }, "node_modules/get-tsconfig": { "version": "4.7.5", "resolved": "https://registry.npmjs.org/get-tsconfig/-/get-tsconfig-4.7.5.tgz", @@ -9933,18 +9916,6 @@ "json-buffer": "3.0.1" } }, - "node_modules/kill-port": { - "version": "2.0.1", - "dev": true, - "license": "MIT", - "dependencies": { - "get-them-args": "1.3.2", - "shell-exec": "1.0.2" - }, - "bin": { - "kill-port": "cli.js" - } - }, "node_modules/kind-of": { "version": "6.0.3", "license": "MIT", @@ -12392,17 +12363,6 @@ "url": "https://github.com/sponsors/sindresorhus" } }, - "node_modules/npm-merge-driver-install": { - "version": "3.0.0", - "dev": true, - "license": "Apache-2.0", - "bin": { - "npm-merge-driver-install": "src/install.js", - "npm-merge-driver-is-installed": "src/is-installed.js", - "npm-merge-driver-merge": "src/merge.js", - "npm-merge-driver-uninstall": "src/uninstall.js" - } - }, "node_modules/npm-run-path": { "version": "4.0.1", "dev": true, @@ -13940,11 +13900,6 @@ "node": ">=8" } }, - "node_modules/shell-exec": { - "version": "1.0.2", - "dev": true, - "license": "MIT" - }, "node_modules/side-channel": { "version": "1.0.6", "resolved": "https://registry.npmjs.org/side-channel/-/side-channel-1.0.6.tgz", diff --git a/package.json b/package.json index 91bc62b84f..01e1793c03 100644 --- a/package.json +++ b/package.json @@ -344,7 +344,6 @@ "@types/connect-timeout": "0.0.39", "@types/cookie": "0.6.0", "@types/cookie-parser": "1.4.7", - "@types/elasticsearch": "^5.0.43", "@types/event-to-promise": "^0.7.5", "@types/express": "4.17.21", "@types/imurmurhash": "^0.1.4", @@ -380,18 +379,13 @@ "http-status-code": "^2.1.0", "husky": "^9.1.4", "json-schema-merge-allof": "^0.8.1", - "kill-port": "2.0.1", "lint-staged": "^15.2.9", "markdownlint": "^0.34.0", "markdownlint-rule-search-replace": "^1.2.0", - "mdast-util-gfm-table": "^2.0.0", - "micromark-extension-gfm-table": "^2.0.0", "mkdirp": "^3.0.0", "mockdate": "^3.0.5", "nock": "^14.0.0", "nodemon": "3.1.3", - "npm-merge-driver-install": "^3.0.0", - "nth-check": "2.1.1", "prettier": "^3.3.2", "rimraf": "^6.0.0", "robots-parser": "^3.0.0",