feat: create Docker images for api-server (#51142)

.github/workflows/temporary-container-checks.yml

@@ -28,23 +28,6 @@ jobs:
       - name: Checkout Source Files
         uses: actions/checkout@f43a0e5ff2bd294095638e18286ca9a3d1956744 # v3
 
-      - name: Checkout client-config
-        uses: actions/checkout@f43a0e5ff2bd294095638e18286ca9a3d1956744 # v3
-        with:
-          repository: freeCodeCamp/client-config
-          path: client-config
-
-      - name: Setup pnpm
-        uses: pnpm/action-setup@v2
-        with:
-          version: 8
-
-      - name: Use Node.js ${{ matrix.node-version }}
-        uses: actions/setup-node@5e21ff4d9bc1a8cf6de233a3057d20ec6b3fb69d # v3
-        with:
-          node-version: ${{ matrix.node-version }}
-          cache: pnpm
-
       - name: Create Image
         run: |
           docker build \
@@ -88,6 +71,32 @@ jobs:
       #     name: webpack-stats
       #     path: client/public/stats.json
 
+  build-api:
+    name: Build Api (Container)
+    runs-on: ubuntu-20.04
+    strategy:
+      matrix:
+        node-version: [18.x]
+
+    steps:
+      - name: Checkout Source Files
+        uses: actions/checkout@f43a0e5ff2bd294095638e18286ca9a3d1956744 # v3
+
+      - name: Create Image
+        run: |
+          docker build \
+            -t fcc-api \
+            -f docker/api/Dockerfile .
+
+      - name: Save Image
+        run: docker save fcc-api > api-artifact.tar
+
+      - name: Upload Api Artifact
+        uses: actions/upload-artifact@0b7f8abb1508181956e8e162db84b466c27e18ce # v3
+        with:
+          name: api-artifact
+          path: api-artifact.tar
+
   build-new-api:
     name: Build New Api (Container)
     runs-on: ubuntu-20.04
@@ -108,7 +117,7 @@ jobs:
   cypress-run:
     name: Test
     runs-on: ubuntu-20.04
-    needs: build-client
+    needs: [build-client, build-api]
     strategy:
       fail-fast: false
       matrix:
@@ -116,17 +125,6 @@ jobs:
         browsers: [chrome]
         node-version: [18.x]
 
-    services:
-      mongodb:
-        image: mongo:4.4
-        ports:
-          - 27017:27017
-      # We need mailhog to catch any emails the api tries to send.
-      mailhog:
-        image: mailhog/mailhog
-        ports:
-          - 1025:1025
-
     steps:
       - name: Set Action Environment Variables
         run: |
@@ -137,13 +135,8 @@ jobs:
         uses: actions/checkout@f43a0e5ff2bd294095638e18286ca9a3d1956744 # v3
 
       - uses: actions/download-artifact@9bc31d5ccc31df68ecc42ccf4149144866c47d8a # v3
-        with:
-          name: client-artifact
-
-      - name: Load Client Image
-        run: |
-          docker load < client-artifact.tar
 
+      # Cypress calls some pnpm scripts, so we need to install pnpm.
       - name: Setup pnpm
         uses: pnpm/action-setup@v2
         with:
@@ -153,34 +146,24 @@ jobs:
         uses: actions/setup-node@5e21ff4d9bc1a8cf6de233a3057d20ec6b3fb69d # v3
         with:
           node-version: ${{ matrix.node-version }}
-          # cypress-io/github-action caches the store, so we should not cache it
-          # here.
 
-      - name: Set freeCodeCamp Environment Variables
-        run: cp sample.env .env
+      - name: Install Dependencies
+        run: pnpm install
 
-      - name: Install and Build
+      - name: Load Images
         run: |
-          pnpm install
-          pnpm run create:shared
-          pnpm run build:curriculum
-          pnpm run build:server
+          docker load < client-artifact/client-artifact.tar
+          docker load < api-artifact/api-artifact.tar
 
-      - name: Seed Database
-        run: pnpm run seed
-
-      # start-ci uses pm2, so it needs to be installed globally
-      - name: Install pm2
-        run: npm i -g pm2
+      - name: Set freeCodeCamp Environment Variables (needed by api)
+        run: cp sample.env .env
 
       - name: Cypress run
         uses: cypress-io/github-action@v4
         with:
           record: ${{ env.CYPRESS_RECORD_KEY != 0 }}
-          start: |
-            pnpm start:server
-            docker compose up -d
-          wait-on: http://localhost:8000
+          start: docker compose up -d
+          wait-on: http://localhost:8000, http://localhost:3000
           wait-on-timeout: 1200
           config: baseUrl=http://localhost:8000
           browser: ${{ matrix.browsers }}

api-server/ecosystem.config.js

@@ -4,7 +4,15 @@ const path = require('path');
 const dotenv = require('dotenv');
 
 const filePath = path.resolve(__dirname, '..', '.env');
-const env = dotenv.parse(fs.readFileSync(filePath));
+let env = {};
+try {
+  env = dotenv.parse(fs.readFileSync(filePath));
+} catch (e) {
+  console.log(
+    "If you're setting the env vars in the shell, it should be fine (you can probably ignore the error)."
+  );
+  console.log(e);
+}
 // without this, loopback cannot find strong-error-handler. Node can, so we know
 // there's no _real_ issue, but loopback is not able to find it.
 const loopbackModuleResolutionHack = path.resolve(

api-server/package.json

@@ -20,7 +20,10 @@
   "main": "none",
   "scripts": {
     "babel-dev-server": "babel-node --inspect=0.0.0.0 ./src/server/index.js",
+    "prebuild": "pnpm common-setup",
     "build": "babel src --out-dir lib --ignore '/**/*.test.js' --copy-files --no-copy-ignored",
+    "common-setup": "pnpm -w run create:shared",
+    "predevelop": "pnpm common-setup",
     "develop": "node src/development-start.js",
     "start": "cross-env DEBUG=fcc* node lib/production-start.js"
   },
@@ -47,6 +50,8 @@
     "express-validator": "6.14.1",
     "helmet": "3.23.3",
     "helmet-csp": "2.10.0",
+    "joi": "17.9.2",
+    "joi-objectid": "3.0.1",
     "jsonwebtoken": "8.5.1",
     "lodash": "4.17.21",
     "loopback": "3.28.0",
@@ -82,8 +87,6 @@
     "@babel/plugin-proposal-optional-chaining": "7.17.12",
     "@babel/preset-env": "7.18.0",
     "@babel/register": "7.17.7",
-    "joi": "17.9.2",
-    "joi-objectid": "3.0.1",
     "loopback-component-explorer": "6.4.0",
     "nodemon": "2.0.16",
     "smee-client": "1.2.3"

cypress/plugins/index.js

@@ -20,6 +20,7 @@ module.exports = (on, config) => {
   // `on` is used to hook into various events Cypress emits
   // `config` is the resolved Cypress config
   config.env = config.env || {};
+  // TODO: once we've containerized the API, we can remove this.
   on('before:run', () => {
     if (!existsSync('../../shared/config/curriculum.json')) {
       execSync('pnpm run build:curriculum');

docker-compose.yml

@@ -1,7 +1,32 @@
 services:
+  mongo:
+    image: mongo
+    ports:
+      - '27017:27017'
+  mailhog:
+    restart: unless-stopped
+    image: mailhog/mailhog
+    ports:
+      - '1025:1025'
+      - '8025:8025'
+  api:
+    depends_on:
+      - mongo
+      - mailhog
+    image: fcc-api
+    env_file:
+      - .env
+    environment:
+      # The api cannot connect to mongodb or mailhog via localhost from inside the
+      # container, so we have to override these variables.
+      - MONGOHQ_URL=mongodb://mongo:27017/freecodecamp?directConnection=true
+      - MAILHOG_HOST=mailhog
+    ports:
+      # PORT is used by the new api, so we use the less generic API_PORT to
+      # avoid conflicts.
+      - '${API_PORT:-3000}:3000'
   client:
     image: fcc-client
     ports:
-      # PORT is used by the new api, so we use the less generic HOST_PORT to avoid
-      # conflicts.
-      - '${HOST_PORT:-8000}:8000'
+      # Same principle as above (avoiding conflicts)
+      - '${CLIENT_PORT:-8000}:8000'

docker/api/Dockerfile

@@ -1,35 +1,61 @@
-FROM node:16-buster AS builder
-# Install doppler CLI
-RUN (curl -Ls --tlsv1.2 --proto "=https" --retry 3 https://cli.doppler.com/install.sh) | sh -s -- --verify-signature
+# bookworm was only released on 10-6-2023, so is a little too new.
+FROM node:18-bullseye AS builder
+ENV PNPM_HOME="/pnpm"
+ENV PATH="$PNPM_HOME:$PATH"
+# global installs need root permissions, so have to happen before we switch to
+# the node user
+RUN npm i -g pnpm@8
 # node images create a non-root user that we can use
 USER node
 WORKDIR /home/node/build
 COPY --chown=node:node . .
-# Pass `DOPPLER_TOKEN` at build time to create an encrypted snapshot for high-availability
-ARG DOPPLER_TOKEN
-RUN \
-  doppler secrets download doppler.encrypted.json &&\
-  pnpm install --no-progress --ignore-scripts &&\
-  doppler run --fallback=doppler.encrypted.json --command="npm run create:config" &&\
-  doppler run --fallback=doppler.encrypted.json --command="npm run build:curriculum" &&\
-  doppler run --fallback=doppler.encrypted.json --command="npm run build:server"
 
-FROM node:16-alpine as depends
-USER node
-WORKDIR /home/node/depends
-COPY --chown=node:node . .
-RUN pnpm install --production --workspace=api-server --no-progress --ignore-scripts
+# TODO: figure out why the cache is getting invalidated. Is it in part because
+# we're not ignoring THIS file? Or do we need corepack?
 
-FROM node:16-alpine
+# We have to prevent pnpm from deduping peer dependencies because otherwise it
+# will install all of the packages, not just api-server. Also, pnpm deploy is
+# not useful since we need to install more than one package.
+
+RUN pnpm config set dedupe-peer-dependents false
+RUN pnpm -F=api-server -F=tools/scripts/build -F=challenge-parser -F=curriculum -F=shared \
+  install --frozen-lockfile --ignore-scripts
+
+# The api needs to source curriculum.json and build:curriculum relies on the
+# following env vars.
+ARG SHOW_UPCOMING_CHANGES=false
+ENV SHOW_UPCOMING_CHANGES=$SHOW_UPCOMING_CHANGES
+ARG SHOW_NEW_CURRICULUM=true
+ENV SHOW_NEW_CURRICULUM=$SHOW_NEW_CURRICULUM
+RUN pnpm build:curriculum
+
+RUN pnpm build:server
+
+FROM node:18-bullseye AS deps
+
+WORKDIR /home/node/build
+COPY --chown=node:node pnpm*.yaml .
+COPY --chown=node:node api-server/package.json api-server/package.json
+COPY --chown=node:node shared/package.json shared/package.json
+
+RUN npm i -g pnpm@8
+# Prevent pnpm installing unnecessary packages (see above)
+RUN pnpm config set dedupe-peer-dependents false
+RUN pnpm -F=api-server -F=shared install --prod --ignore-scripts
+
+FROM node:18-alpine
 RUN npm i -g pm2@4
 USER node
-WORKDIR /home/node/api
+WORKDIR /home/node/fcc
+COPY --from=builder --chown=node:node /home/node/build/api-server/config/ api-server/config/
 COPY --from=builder --chown=node:node /home/node/build/api-server/lib/ api-server/lib/
-COPY --from=builder --chown=node:node /home/node/build/utils/ utils/
-COPY --from=builder --chown=node:node /home/node/build/config/ config/
-COPY --from=depends --chown=node:node /home/node/depends/api-server/node_modules/ api-server/node_modules/
-COPY --from=depends --chown=node:node /home/node/depends/node_modules/ node_modules/
-WORKDIR /home/node/api/api-server
-CMD ["pm2-runtime", "./lib/production-start.js"]
+COPY --from=builder --chown=node:node /home/node/build/api-server/ecosystem.config.js api-server/ecosystem.config.js
+COPY --from=builder --chown=node:node /home/node/build/api-server/package.json api-server/package.json
+COPY --from=builder --chown=node:node /home/node/build/shared/ shared/
+COPY --from=builder --chown=node:node /home/node/build/package.json package.json
+COPY --from=deps --chown=node:node /home/node/build/node_modules/ node_modules/
+COPY --from=deps --chown=node:node /home/node/build/api-server/node_modules/ api-server/node_modules/
+COPY --from=deps --chown=node:node /home/node/build/shared/node_modules/ shared/node_modules/
+
+CMD ["pm2-runtime", "start", "api-server/ecosystem.config.js"]
 
-# TODO: don't copy mocks/fixtures