From 7bb3381a461cbf02520ef38f025365aebac0e3f1 Mon Sep 17 00:00:00 2001 From: Oliver Eyton-Williams Date: Fri, 4 Oct 2024 02:24:32 +0200 Subject: [PATCH] build: add workflow to publish images to DOCR (#54025) --- .dockerignore | 3 +- .github/workflows/build-images.yml | 43 ++++++++++++ .github/workflows/e2e-with-new-api.yml | 25 +------ .../workflows/temporary-container-checks.yml | 7 +- docker-compose.yml | 4 +- docker/api-server/Dockerfile | 63 ++++++++++++++++++ docker/api/Dockerfile | 53 ++++++++------- docker/new-api/Dockerfile | 66 ------------------- 8 files changed, 141 insertions(+), 123 deletions(-) create mode 100644 .github/workflows/build-images.yml create mode 100644 docker/api-server/Dockerfile delete mode 100644 docker/new-api/Dockerfile diff --git a/.dockerignore b/.dockerignore index 09d3362e7f5..ed2e7b45a94 100644 --- a/.dockerignore +++ b/.dockerignore @@ -4,8 +4,7 @@ client/public .git .gitignore .dockerignore -docker/web/Dockerfile -docker/api/Dockerfile +docker/**/Dockerfile **/*docker-compose* **/node_modules .eslintcache diff --git a/.github/workflows/build-images.yml b/.github/workflows/build-images.yml new file mode 100644 index 00000000000..eca923e2767 --- /dev/null +++ b/.github/workflows/build-images.yml @@ -0,0 +1,43 @@ +name: CI - Build Images +on: + workflow_dispatch: + +jobs: + build: + name: Build (Image) + runs-on: ubuntu-22.04 + strategy: + matrix: + node-version: [20.x] + apps: [api] + site_tlds: [dev] + fail-fast: false + + steps: + - name: Checkout Source Files + uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1 + + - name: Create a tagname + id: tagname + run: | + echo "tagname=$(git rev-parse --short HEAD)-$(date +%Y%m%d)-$(date +%H%M)" >> $GITHUB_ENV + + - name: Build & Tag Image + run: | + docker build \ + --tag registry.digitalocean.com/${{ secrets.DOCR_NAME }}/${{ matrix.site_tlds }}/learn-${{ matrix.apps }}:$tagname \ + --tag registry.digitalocean.com/${{ secrets.DOCR_NAME }}/${{ matrix.site_tlds }}/learn-${{ matrix.apps }}:latest \ + --file docker/${{ matrix.apps }}/Dockerfile . + + - name: Install doctl + uses: digitalocean/action-doctl@v2 + with: + token: ${{ secrets.DIGITALOCEAN_ACCESS_TOKEN }} + + - name: Log in to DigitalOcean Container Registry with short-lived credentials + run: doctl registry login --expiry-seconds 1200 + + - name: Push image to DigitalOcean Container Registry + run: | + docker push registry.digitalocean.com/${{ secrets.DOCR_NAME }}/${{ matrix.site_tlds }}/learn-${{ matrix.apps }}:$tagname + docker push registry.digitalocean.com/${{ secrets.DOCR_NAME }}/${{ matrix.site_tlds }}/learn-${{ matrix.apps }}:latest diff --git a/.github/workflows/e2e-with-new-api.yml b/.github/workflows/e2e-with-new-api.yml index 73bc026ea52..8522639b03e 100644 --- a/.github/workflows/e2e-with-new-api.yml +++ b/.github/workflows/e2e-with-new-api.yml @@ -71,8 +71,8 @@ jobs: name: webpack-stats path: client/public/stats.json - build-api: - name: Build Api (Container) + build-new-api: + name: Build New Api (Container) runs-on: ubuntu-22.04 strategy: matrix: @@ -90,27 +90,8 @@ jobs: -t fcc-api \ -f docker/api/Dockerfile . - build-new-api: - name: Build New Api (Container) - runs-on: ubuntu-22.04 - strategy: - matrix: - node-version: [20.x] - - steps: - - name: Checkout Source Files - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1 - with: - submodules: 'recursive' - - - name: Create Image - run: | - docker build \ - -t fcc-new-api \ - -f docker/new-api/Dockerfile . - - name: Save Image - run: docker save fcc-new-api > api-artifact.tar + run: docker save fcc-api > api-artifact.tar - name: Upload Api Artifact uses: actions/upload-artifact@5d5d22a31266ced268874388b861e4b58bb5c2f3 # v4.3.1 diff --git a/.github/workflows/temporary-container-checks.yml b/.github/workflows/temporary-container-checks.yml index 01cbdde8275..68d3d4c9940 100644 --- a/.github/workflows/temporary-container-checks.yml +++ b/.github/workflows/temporary-container-checks.yml @@ -50,7 +50,6 @@ jobs: --build-arg SHOW_NEW_CURRICULUM=false \ --build-arg GROWTHBOOK_URI=api_URI_from_Growthbook_dashboard \ --build-arg FREECODECAMP_NODE_ENV=development \ - -t fcc-client \ -f docker/web/Dockerfile . build-api: @@ -69,8 +68,7 @@ jobs: - name: Create Image run: | docker build \ - -t fcc-api \ - -f docker/api/Dockerfile . + -f docker/api-server/Dockerfile . build-new-api: name: Build New Api (Container) @@ -88,5 +86,4 @@ jobs: - name: Create Image run: | docker build \ - -t fcc-new-api \ - -f docker/new-api/Dockerfile . + -f docker/api/Dockerfile . diff --git a/docker-compose.yml b/docker-compose.yml index 8f1fea467c7..dabaa283986 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -26,7 +26,7 @@ services: depends_on: - mongo - mailhog - image: fcc-new-api + image: fcc-api env_file: - .env environment: @@ -35,6 +35,4 @@ services: - MONGOHQ_URL=mongodb://mongo:27017/freecodecamp?directConnection=true - MAILHOG_HOST=mailhog ports: - # PORT is used by the new api, so we use the less generic API_PORT to - # avoid conflicts. - '3000:3000' diff --git a/docker/api-server/Dockerfile b/docker/api-server/Dockerfile new file mode 100644 index 00000000000..f6cfbeef41c --- /dev/null +++ b/docker/api-server/Dockerfile @@ -0,0 +1,63 @@ +FROM node:20-bookworm AS builder +# global installs need root permissions, so have to happen before we switch to +# the node user +RUN npm i -g pnpm@9 +# node images create a non-root user that we can use +USER node +WORKDIR /home/node/build + +COPY --chown=node:node *.* . +COPY --chown=node:node api-server/ api-server/ +COPY --chown=node:node shared/ shared/ +COPY --chown=node:node tools/ tools/ +COPY --chown=node:node curriculum/ curriculum/ +# TODO: AFAIK it's just the intro translations. Those should be folded into the +# curriculum and then we can remove this. +COPY --chown=node:node client/ client/ + +# We have to prevent pnpm from deduping peer dependencies because otherwise it +# will install all of the packages, not just api-server. Also, pnpm deploy is +# not useful since we need to install more than one package. + +RUN pnpm config set dedupe-peer-dependents false +RUN pnpm -F=api-server -F=tools/scripts/build -F=challenge-parser -F=curriculum -F=shared \ + install --frozen-lockfile --ignore-scripts + +# The api needs to source curriculum.json and build:curriculum relies on the +# following env vars. +ARG SHOW_UPCOMING_CHANGES=false +ENV SHOW_UPCOMING_CHANGES=$SHOW_UPCOMING_CHANGES +ARG SHOW_NEW_CURRICULUM=false +ENV SHOW_NEW_CURRICULUM=$SHOW_NEW_CURRICULUM +RUN pnpm build:curriculum + +RUN pnpm build:server + +FROM node:20-bookworm AS deps + +WORKDIR /home/node/build +COPY --chown=node:node pnpm*.yaml . +COPY --chown=node:node api-server/package.json api-server/package.json +COPY --chown=node:node shared/package.json shared/package.json + +RUN npm i -g pnpm@9 +# Prevent pnpm installing unnecessary packages (see above) +RUN pnpm config set dedupe-peer-dependents false +RUN pnpm -F=api-server -F=shared install --prod --ignore-scripts + +FROM node:20-alpine +RUN npm i -g pm2@4 +USER node +WORKDIR /home/node/fcc +COPY --from=builder --chown=node:node /home/node/build/api-server/config/ api-server/config/ +COPY --from=builder --chown=node:node /home/node/build/api-server/lib/ api-server/lib/ +COPY --from=builder --chown=node:node /home/node/build/api-server/ecosystem.config.js api-server/ecosystem.config.js +COPY --from=builder --chown=node:node /home/node/build/api-server/package.json api-server/package.json +COPY --from=builder --chown=node:node /home/node/build/shared/ shared/ +COPY --from=builder --chown=node:node /home/node/build/package.json package.json +COPY --from=deps --chown=node:node /home/node/build/node_modules/ node_modules/ +COPY --from=deps --chown=node:node /home/node/build/api-server/node_modules/ api-server/node_modules/ +COPY --from=deps --chown=node:node /home/node/build/shared/node_modules/ shared/node_modules/ + +CMD ["pm2-runtime", "start", "api-server/ecosystem.config.js"] + diff --git a/docker/api/Dockerfile b/docker/api/Dockerfile index f6cfbeef41c..406fed30829 100644 --- a/docker/api/Dockerfile +++ b/docker/api/Dockerfile @@ -1,4 +1,5 @@ FROM node:20-bookworm AS builder +RUN apt-get update && apt-get install -y jq # global installs need root permissions, so have to happen before we switch to # the node user RUN npm i -g pnpm@9 @@ -7,7 +8,7 @@ USER node WORKDIR /home/node/build COPY --chown=node:node *.* . -COPY --chown=node:node api-server/ api-server/ +COPY --chown=node:node api/ api/ COPY --chown=node:node shared/ shared/ COPY --chown=node:node tools/ tools/ COPY --chown=node:node curriculum/ curriculum/ @@ -15,13 +16,14 @@ COPY --chown=node:node curriculum/ curriculum/ # curriculum and then we can remove this. COPY --chown=node:node client/ client/ -# We have to prevent pnpm from deduping peer dependencies because otherwise it -# will install all of the packages, not just api-server. Also, pnpm deploy is -# not useful since we need to install more than one package. - RUN pnpm config set dedupe-peer-dependents false -RUN pnpm -F=api-server -F=tools/scripts/build -F=challenge-parser -F=curriculum -F=shared \ - install --frozen-lockfile --ignore-scripts +# While we want to ignore scripts generally, we do need to generate the prisma +# client. Note: npx is the simplest way to generate the client without us having +# to have prisma as a prod dependency. The jq tricks are to ensure we're using +# the right version of prisma. +RUN pnpm install -F=api -F=curriculum -F tools/scripts/build -F challenge-parser \ + --frozen-lockfile --ignore-scripts +RUN cd api && npx prisma@$(jq -r '.devDependencies.prisma' < package.json) generate # The api needs to source curriculum.json and build:curriculum relies on the # following env vars. @@ -29,35 +31,36 @@ ARG SHOW_UPCOMING_CHANGES=false ENV SHOW_UPCOMING_CHANGES=$SHOW_UPCOMING_CHANGES ARG SHOW_NEW_CURRICULUM=false ENV SHOW_NEW_CURRICULUM=$SHOW_NEW_CURRICULUM -RUN pnpm build:curriculum -RUN pnpm build:server +RUN pnpm build:curriculum +RUN pnpm -F=api build FROM node:20-bookworm AS deps +RUN apt-get update && apt-get install -y jq WORKDIR /home/node/build COPY --chown=node:node pnpm*.yaml . -COPY --chown=node:node api-server/package.json api-server/package.json -COPY --chown=node:node shared/package.json shared/package.json - +COPY --chown=node:node api/ api/ +COPY --chown=node:node shared/ shared/ RUN npm i -g pnpm@9 -# Prevent pnpm installing unnecessary packages (see above) -RUN pnpm config set dedupe-peer-dependents false -RUN pnpm -F=api-server -F=shared install --prod --ignore-scripts -FROM node:20-alpine +# Weirdly this config does not seem necessary for the new api (the same number +# of deps are installed in both cases), but I'm including it just for +# consistency. +RUN pnpm config set dedupe-peer-dependents false +RUN pnpm install --prod --ignore-scripts -F=shared -F=api --frozen-lockfile +RUN cd api && npx prisma@$(jq -r '.devDependencies.prisma' < package.json) generate + +FROM node:20-bookworm RUN npm i -g pm2@4 USER node WORKDIR /home/node/fcc -COPY --from=builder --chown=node:node /home/node/build/api-server/config/ api-server/config/ -COPY --from=builder --chown=node:node /home/node/build/api-server/lib/ api-server/lib/ -COPY --from=builder --chown=node:node /home/node/build/api-server/ecosystem.config.js api-server/ecosystem.config.js -COPY --from=builder --chown=node:node /home/node/build/api-server/package.json api-server/package.json -COPY --from=builder --chown=node:node /home/node/build/shared/ shared/ -COPY --from=builder --chown=node:node /home/node/build/package.json package.json +COPY --from=builder --chown=node:node /home/node/build/api/dist/ ./ +COPY --from=builder --chown=node:node /home/node/build/api/package.json api/ +COPY --from=builder --chown=node:node /home/node/build/shared/config/curriculum.json shared/config/ + COPY --from=deps --chown=node:node /home/node/build/node_modules/ node_modules/ -COPY --from=deps --chown=node:node /home/node/build/api-server/node_modules/ api-server/node_modules/ COPY --from=deps --chown=node:node /home/node/build/shared/node_modules/ shared/node_modules/ +COPY --from=deps --chown=node:node /home/node/build/api/node_modules/ api/node_modules/ -CMD ["pm2-runtime", "start", "api-server/ecosystem.config.js"] - +CMD ["pm2-runtime", "start", "-i", "0","api/src/server.js"] diff --git a/docker/new-api/Dockerfile b/docker/new-api/Dockerfile deleted file mode 100644 index 406fed30829..00000000000 --- a/docker/new-api/Dockerfile +++ /dev/null @@ -1,66 +0,0 @@ -FROM node:20-bookworm AS builder -RUN apt-get update && apt-get install -y jq -# global installs need root permissions, so have to happen before we switch to -# the node user -RUN npm i -g pnpm@9 -# node images create a non-root user that we can use -USER node -WORKDIR /home/node/build - -COPY --chown=node:node *.* . -COPY --chown=node:node api/ api/ -COPY --chown=node:node shared/ shared/ -COPY --chown=node:node tools/ tools/ -COPY --chown=node:node curriculum/ curriculum/ -# TODO: AFAIK it's just the intro translations. Those should be folded into the -# curriculum and then we can remove this. -COPY --chown=node:node client/ client/ - -RUN pnpm config set dedupe-peer-dependents false -# While we want to ignore scripts generally, we do need to generate the prisma -# client. Note: npx is the simplest way to generate the client without us having -# to have prisma as a prod dependency. The jq tricks are to ensure we're using -# the right version of prisma. -RUN pnpm install -F=api -F=curriculum -F tools/scripts/build -F challenge-parser \ - --frozen-lockfile --ignore-scripts -RUN cd api && npx prisma@$(jq -r '.devDependencies.prisma' < package.json) generate - -# The api needs to source curriculum.json and build:curriculum relies on the -# following env vars. -ARG SHOW_UPCOMING_CHANGES=false -ENV SHOW_UPCOMING_CHANGES=$SHOW_UPCOMING_CHANGES -ARG SHOW_NEW_CURRICULUM=false -ENV SHOW_NEW_CURRICULUM=$SHOW_NEW_CURRICULUM - -RUN pnpm build:curriculum -RUN pnpm -F=api build - -FROM node:20-bookworm AS deps -RUN apt-get update && apt-get install -y jq - -WORKDIR /home/node/build -COPY --chown=node:node pnpm*.yaml . -COPY --chown=node:node api/ api/ -COPY --chown=node:node shared/ shared/ -RUN npm i -g pnpm@9 - -# Weirdly this config does not seem necessary for the new api (the same number -# of deps are installed in both cases), but I'm including it just for -# consistency. -RUN pnpm config set dedupe-peer-dependents false -RUN pnpm install --prod --ignore-scripts -F=shared -F=api --frozen-lockfile -RUN cd api && npx prisma@$(jq -r '.devDependencies.prisma' < package.json) generate - -FROM node:20-bookworm -RUN npm i -g pm2@4 -USER node -WORKDIR /home/node/fcc -COPY --from=builder --chown=node:node /home/node/build/api/dist/ ./ -COPY --from=builder --chown=node:node /home/node/build/api/package.json api/ -COPY --from=builder --chown=node:node /home/node/build/shared/config/curriculum.json shared/config/ - -COPY --from=deps --chown=node:node /home/node/build/node_modules/ node_modules/ -COPY --from=deps --chown=node:node /home/node/build/shared/node_modules/ shared/node_modules/ -COPY --from=deps --chown=node:node /home/node/build/api/node_modules/ api/node_modules/ - -CMD ["pm2-runtime", "start", "-i", "0","api/src/server.js"]