From a1621ffabb2320f323352a3c3384c57bc24a9544 Mon Sep 17 00:00:00 2001 From: Sarah Schneider Date: Fri, 20 Nov 2020 10:41:51 -0500 Subject: [PATCH 1/9] support LANGUAGE and VERSION env vars as options --- script/sync-algolia-search-indices.js | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/script/sync-algolia-search-indices.js b/script/sync-algolia-search-indices.js index 586eb5cfdd..105e0defa3 100755 --- a/script/sync-algolia-search-indices.js +++ b/script/sync-algolia-search-indices.js @@ -14,7 +14,9 @@ main() async function main () { const sync = require('../lib/algolia/sync') const opts = { - dryRun: 'DRY_RUN' in process.env + dryRun: 'DRY_RUN' in process.env, + language: process.env.LANGUAGE, + version: process.env.VERSION } await sync(opts) } From 180c7c7db1dd5d12b9a1f3fabed8c2f1058fc4de Mon Sep 17 00:00:00 2001 From: Sarah Schneider Date: Fri, 20 Nov 2020 10:43:39 -0500 Subject: [PATCH 2/9] build specified language or version indices only --- lib/algolia/sync.js | 32 +++++++++++++++++++++++++++++--- 1 file changed, 29 insertions(+), 3 deletions(-) diff --git a/lib/algolia/sync.js b/lib/algolia/sync.js index fb46cd17cd..087b87cf05 100644 --- a/lib/algolia/sync.js +++ b/lib/algolia/sync.js @@ -15,17 +15,43 @@ const allVersions = require('../all-versions') // e.g. `github-docs-dotcom-en.json` and `github-docs-2.14-ja.json` module.exports = async function syncAlgoliaIndices (opts = {}) { if (opts.dryRun) { - console.log('This is a dry run! The script will abort without uploading anything.') + console.log('This is a dry run! The script will build the indices locally but not upload anything.\n') rimraf(cacheDir) mkdirp(cacheDir) } + if (opts.language) { + if (!Object.keys(languages).includes(opts.language)) { + console.log(`Error! ${opts.language} not found. You must provide a currently supported two-letter language code.`) + process.exit(1) + } + } + + if (opts.version) { + if (!Object.keys(allVersions).includes(opts.version)) { + console.log(`Error! ${opts.version} not found. You must provide a currently supported version in format.`) + process.exit(1) + } + } + + // build indices for a specific language if provided; otherwise build indices for all languages + const languagesToBuild = opts.language + ? Object.keys(languages).filter(language => language === opts.language) + : Object.keys(languages) + + // build indices for a specific version if provided; otherwise build indices for all veersions + const versionsToBuild = opts.version + ? Object.keys(allVersions).filter(version => version === opts.version) + : Object.keys(allVersions) + + console.log(`Building indices for ${opts.language || 'all languages'} and ${opts.version || 'all versions'}.\n`) + // Exlude WIP pages, hidden pages, index pages, etc const indexablePages = await findIndexablePages() // Build and validate all indices - for (const languageCode in languages) { - for (const pageVersion in allVersions) { + for (const languageCode of languagesToBuild) { + for (const pageVersion of versionsToBuild) { // if GHES, resolves to the release number like 2.21, 2.22, etc. // if FPT, resolves to 'dotcom' // if GHAE, resolves to 'ghae' From 071879e97b9f01e6f030f39718139285087b028b Mon Sep 17 00:00:00 2001 From: Sarah Schneider Date: Fri, 20 Nov 2020 10:45:05 -0500 Subject: [PATCH 3/9] add new label-triggered workflow --- .../sync-single-english-algolia-index.yml | 44 +++++++++++++++++++ 1 file changed, 44 insertions(+) create mode 100644 .github/workflows/sync-single-english-algolia-index.yml diff --git a/.github/workflows/sync-single-english-algolia-index.yml b/.github/workflows/sync-single-english-algolia-index.yml new file mode 100644 index 0000000000..bc2fb38ddc --- /dev/null +++ b/.github/workflows/sync-single-english-algolia-index.yml @@ -0,0 +1,44 @@ +name: Algolia Sync Single English Index + +on: + pull_request: + types: + - opened + - reopened + - synchronize + - ready_for_review + - unlocked + +# This workflow requires a label in the format `sync-english-index-for-` +jobs: + updateIndices: + name: Update English index for single version based on a label's version + if: github.repository == 'github/docs-internal' && github.event.label.name.startsWith('sync-english-index-for-') + runs-on: ubuntu-latest + steps: + - name: checkout + uses: actions/checkout@5a4ac9002d0be2fb38bd78e4b4dbde5606d7042f + - uses: actions/setup-node@56899e050abffc08c2b3b61f3ec6a79a9dc3223d + with: + node-version: 14.x + - name: cache node modules + uses: actions/cache@0781355a23dac32fd3bac414512f4b903437991a + with: + path: ~/.npm + key: ${{ runner.os }}-node-${{ hashFiles('**/package-lock.json') }} + restore-keys: | + ${{ runner.os }}-node- + - name: npm ci + run: npm ci + - name: Get version from label + id: getVersion + - run: | + echo "::set-output name=version::$(github.event.label.name.split('sync-english-index-for-')[1])" + - name: Sync English index for single version + env: + VERSION: ${{ steps.getVersion.outputs.version }} + LANGUAGE: 'en' + ALGOLIA_APPLICATION_ID: ${{ secrets.ALGOLIA_APPLICATION_ID }} + ALGOLIA_API_KEY: ${{ secrets.ALGOLIA_API_KEY }} + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + run: npm run sync-search From 0a195e40eb1ed06617251adc89ab91e9c20b1b10 Mon Sep 17 00:00:00 2001 From: Sarah Schneider Date: Fri, 20 Nov 2020 10:45:23 -0500 Subject: [PATCH 4/9] document the new command options and the new workflow --- contributing/search.md | 35 +++++++++++++++++++++++++++++++++++ 1 file changed, 35 insertions(+) diff --git a/contributing/search.md b/contributing/search.md index 35b240b678..1eee8fb17d 100644 --- a/contributing/search.md +++ b/contributing/search.md @@ -26,6 +26,41 @@ To sync the indices from your development environment: 2. Run `npm run sync-search-dry-run`. This takes a while to complete. It will prepare, test, and validate all the indices without actually uploading anything to Algolia's servers. 3. Run `npm run sync-search` to prepare the indices again and upload them to the Algolia servers. +## Build indices for a specific language and/or version + +You can optionally build indices for a given language and/or a version. This is much faster than building all the indices. + +You can build them as a dry run, which creates the index locally but does not upload them to Algolia. For example: +``` +VERSION='free-pro-team@latest' LANGUAGE='en' npm run sync-search-dry-run +``` +Or you can build and load them to Algolia to make them live: +``` +VERSION='free-pro-team@latest' LANGUAGE='en' npm run sync-search +``` +The `VERSION` env var must be a currently supported version in `` format. The `LANGUAGE` env var must be a currently supported two-letter language code. + +### Label-triggered workflow + +For our daily shipping needs, the indices are synced to Algolia only after a PR is shipped to the default branch, which means the search data isn't available for up to an hour after the content goes live. + +But for GHES releases, we have additional needs. We want: + +1) The search results to be live immediately upon releasing the new content. +2) The search results to be previewable on the staging app the GHES release megabranch. + +Manually running `npm run sync` locally before shipping the megabranch can address #1, but it's cumbersome and hard to get the timing right. And it does not address #2. + +Now, docs team members can use an Actions workflow on GHES megabranch PRs by applying a label in this format: +``` +sync-english-index-for- +``` +This label will run a workflow on every push to **build and upload ONLY** the English index for the specified version. This means: + +* The GHES content will be searchable at the same time the megabranch is shipped, with no delay. +* The GHES content will be searchable on staging throughout content creation. +* No manual steps (unless anyone wants to do a dry-run test as described in the previous section). + ## Files - [.github/workflows/sync-algolia-search-indices.yml](.github/workflows/sync-algolia-search-indices.yml) - the GitHub Actions workflow file that updates search indices whenever the main branch is updated. From db79e8fc56b63780063f9a706e21ab95d5412f7a Mon Sep 17 00:00:00 2001 From: Sarah Schneider Date: Fri, 20 Nov 2020 10:59:18 -0500 Subject: [PATCH 5/9] Update contributing/search.md Co-authored-by: Kevin Heis --- contributing/search.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/contributing/search.md b/contributing/search.md index 1eee8fb17d..491a6919e3 100644 --- a/contributing/search.md +++ b/contributing/search.md @@ -38,7 +38,7 @@ Or you can build and load them to Algolia to make them live: ``` VERSION='free-pro-team@latest' LANGUAGE='en' npm run sync-search ``` -The `VERSION` env var must be a currently supported version in `` format. The `LANGUAGE` env var must be a currently supported two-letter language code. +The `VERSION` environment variable must be a currently supported version in `` format. The `LANGUAGE` environment variable must be a currently supported two-letter language code. ### Label-triggered workflow From 67cc12fca165f312780f719519ed07c38f8235bb Mon Sep 17 00:00:00 2001 From: Sarah Schneider Date: Fri, 20 Nov 2020 11:02:02 -0500 Subject: [PATCH 6/9] remove extra hyphen --- .github/workflows/sync-single-english-algolia-index.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/sync-single-english-algolia-index.yml b/.github/workflows/sync-single-english-algolia-index.yml index bc2fb38ddc..1ac65d8551 100644 --- a/.github/workflows/sync-single-english-algolia-index.yml +++ b/.github/workflows/sync-single-english-algolia-index.yml @@ -32,7 +32,7 @@ jobs: run: npm ci - name: Get version from label id: getVersion - - run: | + run: | echo "::set-output name=version::$(github.event.label.name.split('sync-english-index-for-')[1])" - name: Sync English index for single version env: From 6041ffcfeba004dda2f27854b14932c3803c24d1 Mon Sep 17 00:00:00 2001 From: Sarah Schneider Date: Fri, 20 Nov 2020 11:18:10 -0500 Subject: [PATCH 7/9] Update contributing/search.md --- contributing/search.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/contributing/search.md b/contributing/search.md index 491a6919e3..dabce854aa 100644 --- a/contributing/search.md +++ b/contributing/search.md @@ -47,7 +47,7 @@ For our daily shipping needs, the indices are synced to Algolia only after a PR But for GHES releases, we have additional needs. We want: 1) The search results to be live immediately upon releasing the new content. -2) The search results to be previewable on the staging app the GHES release megabranch. +2) The search results to be viewable on the staging app for the GHES release megabranch. Manually running `npm run sync` locally before shipping the megabranch can address #1, but it's cumbersome and hard to get the timing right. And it does not address #2. From 1a8dedf4d50db56e69d5c6066c553e4a40b68b82 Mon Sep 17 00:00:00 2001 From: Sarah Schneider Date: Fri, 20 Nov 2020 16:50:11 -0500 Subject: [PATCH 8/9] bunch of edits --- contributing/search.md | 92 ++++++++++++++++++++++++++---------------- 1 file changed, 58 insertions(+), 34 deletions(-) diff --git a/contributing/search.md b/contributing/search.md index dabce854aa..30919813b4 100644 --- a/contributing/search.md +++ b/contributing/search.md @@ -1,4 +1,18 @@ -## Search +# Search + +## Table of contents +- [Overview](#overview) +- [How it works](#how-it-works) +- [Manual sync from a checkout](#manual-sync-from-a-checkout) + - [Build without sync (dry run)](#build-without-sync-dry-run) + - [Build and sync](#build-and-sync) +- [Label-triggered workflow](#label-triggered-workflow) +- [Files](#files) + - [GitHub Actions workflow files](#github-actions-workflow-files) + - [Code files](#code-files) + + +## Overview This site's search functionality is powered by [Algolia](https://www.algolia.com), a third-party service. @@ -10,65 +24,75 @@ To see all existing search-related issues and pull requests, visit [github.com/g --- -## How it Works +## How it works The search data is synced automatically using a [GitHub Actions workflow](.github/workflows/sync-algolia-search-indices.yml) that is triggered by pushes to the `main` branch. This process generates structured data for all pages on the site, compares that data to what's currently on Algolia, then adds, updates, or removes indices based on the diff of the local and remote data, being careful not to create duplicate records and avoiding any unnecessary (and costly) indexing operations. -The Actions workflow usually takes about five minutes, and the progress can be viewed (by GitHub employees) in the [Actions tab](https://github.com/github/docs/actions?query=workflow%3AAlgolia) of the repo. +The Actions workflow progress can be viewed (by GitHub employees) in the [Actions tab](https://github.com/github/docs/actions?query=workflow%3AAlgolia) of the repo. -## Development +## Manual sync from a checkout -In cases where a publicity event like GitHub Satellite or GitHub Universe demands a very tight shipping window, it is also possible to manually sync the indices with Algolia's servers from your local checkout of the repo, before your feature branch is merged to main. Manually syncing the indices can also be useful to test an unreleased GitHub Enterprise version or a translated language (Portuguese, Chinese, etc) that is not yet in production. +It is also possible to manually sync the indices to Algolia from your local checkout of the repo, before your branch is merged to `main`. -To sync the indices from your development environment: +**Prerequisite:** Make sure the environment variables `ALGOLIA_APPLICATION_ID` and `ALGOLIA_API_KEY` are set in your `.env` file. You can find these values on [Algolia](https://www.algolia.com/apps/ZI5KPY1HBE/api-keys/all). -1. Make sure the two required environment variables `ALGOLIA_APPLICATION_ID` and `ALGOLIA_API_KEY` are set in your `.env` file. These can be retrieved from the [Algolia site](https://www.algolia.com/apps/ZI5KPY1HBE/api-keys/all). -2. Run `npm run sync-search-dry-run`. This takes a while to complete. It will prepare, test, and validate all the indices without actually uploading anything to Algolia's servers. -3. Run `npm run sync-search` to prepare the indices again and upload them to the Algolia servers. +### Build without sync (dry run) -## Build indices for a specific language and/or version - -You can optionally build indices for a given language and/or a version. This is much faster than building all the indices. - -You can build them as a dry run, which creates the index locally but does not upload them to Algolia. For example: +To build all the indices without uploading them to Algolia's servers (this takes about an hour): ``` -VERSION='free-pro-team@latest' LANGUAGE='en' npm run sync-search-dry-run +npm run sync-search-dry-run ``` -Or you can build and load them to Algolia to make them live: +To build indices for a specific language and/or version (this is much faster): ``` -VERSION='free-pro-team@latest' LANGUAGE='en' npm run sync-search +VERSION= LANGUAGE= npm run sync-search-dry-run ``` -The `VERSION` environment variable must be a currently supported version in `` format. The `LANGUAGE` environment variable must be a currently supported two-letter language code. +You can set `VERSION` and `LANGUAGE` individually, too. -### Label-triggered workflow +Substitute a currently supported version for `` and a currently supported two-letter language code for ``. -For our daily shipping needs, the indices are synced to Algolia only after a PR is shipped to the default branch, which means the search data isn't available for up to an hour after the content goes live. +### Build and sync -But for GHES releases, we have additional needs. We want: +To build all the indices and sync them to Algolia (this also takes about an hour): +``` +npm run sync-search +``` +To build indices for a specific language and/or version and sync them to Algolia: +``` +VERSION= npm run sync-search +``` +You can set `VERSION` and `LANGUAGE` individually, too. -1) The search results to be live immediately upon releasing the new content. -2) The search results to be viewable on the staging app for the GHES release megabranch. +Substitute a currently supported version for `` and a currently supported two-letter language code for ``. -Manually running `npm run sync` locally before shipping the megabranch can address #1, but it's cumbersome and hard to get the timing right. And it does not address #2. +## Label-triggered workflow -Now, docs team members can use an Actions workflow on GHES megabranch PRs by applying a label in this format: +Docs team members can use an Actions workflow on GHES release PRs by applying a label in this format: ``` sync-english-index-for- ``` -This label will run a workflow on every push to **build and upload ONLY** the English index for the specified version. This means: +This label will run a workflow on every push that **builds and uploads ONLY** the English index for the specified version. This means: -* The GHES content will be searchable at the same time the megabranch is shipped, with no delay. +* The GHES content will be searchable at the same time the release PR is shipped, with no delay. * The GHES content will be searchable on staging throughout content creation. -* No manual steps (unless anyone wants to do a dry-run test as described in the previous section). +* No manual steps (unless you want to do a [dry run test](#build-without-sync-dry-run)). + +Why do we need this? For our daily shipping needs, it's tolerable that search updates aren't available for up to an hour after the content goes live. But GHES releases are more time-sensitive, and writers have a greater need to preview search data on staging. ## Files -- [.github/workflows/sync-algolia-search-indices.yml](.github/workflows/sync-algolia-search-indices.yml) - the GitHub Actions workflow file that updates search indices whenever the main branch is updated. -- [javascripts/search.js](javascripts/search.js) - the browser-side code that enables search using Algolia's [InstantSearch.js](https://github.com/algolia/instantsearch.js/) library. -- [lib/algolia/client.js](lib/algolia/client.js) - a thin wrapper around the [algoliasearch](https://ghub.io/algoliasearch) Node.js module for interacting with the Algolia API. -- [lib/algolia/search-index.js](lib/algolia/search-index.js) - a class for generating structured search data from repository content and syncing it with the remote Algolia service. This class has built-in validation to ensure that all records are valid before they're uploaded. This class also takes care of removing deprecated records, and compares existing remote records with the latest local records to avoid uploading records that haven't changed. -- [script/sync-algolia-search-indices.js](script/sync-algolia-search-indices.js) - the script used by the Actions workflow to update search indices on our Algolia account. This can also be [run in the development environment](#development). -- [tests/algolia-search.js](tests/algolia-search.js) - tests! +### GitHub Actions workflow files + +- [`.github/workflows/sync-algolia-search-indices.yml`](.github/workflows/sync-algolia-search-indices.yml) - Builds and syncs search indices whenever the `main` branch is pushed to (that is, on production deploys). +- [`.github/workflows/dry-run-sync-algolia-search-indices.yml`](.github/workflows/dry-run-sync-algolia-search-indices.yml) - This workflow can be run manually (via `workflow_dispatch`) to do a dry run build of all the indices. Useful for confirming that the indices can build without erroring out. +- [`.github/workflows/sync-single-english-algolia-index.yml`](.github/workflows/sync-single-english-algolia-index.yml) - This workflow is run when a label in the right format is applied to a PR. See "[Label-triggered workflow](#label-triggered-workflow)" for details. + +### Code files + +- [javascripts/search.js](javascripts/search.js) - The browser-side code that enables search using Algolia's [InstantSearch.js](https://github.com/algolia/instantsearch.js/) library. +- [lib/algolia/client.js](lib/algolia/client.js) - A thin wrapper around the [algoliasearch](https://ghub.io/algoliasearch) Node.js module for interacting with the Algolia API. +- [lib/algolia/search-index.js](lib/algolia/search-index.js) - A class for generating structured search data from repository content and syncing it with the remote Algolia service. This class has built-in validation to ensure that all records are valid before they're uploaded. This class also takes care of removing deprecated records, and compares existing remote records with the latest local records to avoid uploading records that haven't changed. +- [script/sync-algolia-search-indices.js](script/sync-algolia-search-indices.js) - The script used by the Actions workflow to update search indices on our Algolia account. This can also be [run in the development environment](#development). +- [tests/algolia-search.js](tests/algolia-search.js) - Tests! ## Indices From 0b3594cbb91441ee831f52fb71d3753f3aa7ea35 Mon Sep 17 00:00:00 2001 From: Sarah Schneider Date: Fri, 20 Nov 2020 16:58:02 -0500 Subject: [PATCH 9/9] a few more edits --- contributing/search.md | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/contributing/search.md b/contributing/search.md index 30919813b4..4da0507ebf 100644 --- a/contributing/search.md +++ b/contributing/search.md @@ -2,11 +2,11 @@ ## Table of contents - [Overview](#overview) -- [How it works](#how-it-works) +- [Production deploys](#production-deploys) - [Manual sync from a checkout](#manual-sync-from-a-checkout) - [Build without sync (dry run)](#build-without-sync-dry-run) - [Build and sync](#build-and-sync) -- [Label-triggered workflow](#label-triggered-workflow) +- [Label-triggered Actions workflow](#label-triggered-actions-workflow) - [Files](#files) - [GitHub Actions workflow files](#github-actions-workflow-files) - [Code files](#code-files) @@ -24,12 +24,14 @@ To see all existing search-related issues and pull requests, visit [github.com/g --- -## How it works +## Production deploys -The search data is synced automatically using a [GitHub Actions workflow](.github/workflows/sync-algolia-search-indices.yml) that is triggered by pushes to the `main` branch. This process generates structured data for all pages on the site, compares that data to what's currently on Algolia, then adds, updates, or removes indices based on the diff of the local and remote data, being careful not to create duplicate records and avoiding any unnecessary (and costly) indexing operations. +A [GitHub Actions workflow](.github/workflows/sync-algolia-search-indices.yml) triggered by pushes to the `main` branch syncs the search data to Algolia. This process generates structured data for all pages on the site, compares that data to what's currently on Algolia, then adds, updates, or removes indices based on the diff of the local and remote data, being careful not to create duplicate records and avoiding any unnecessary (and costly) indexing operations. The Actions workflow progress can be viewed (by GitHub employees) in the [Actions tab](https://github.com/github/docs/actions?query=workflow%3AAlgolia) of the repo. +Because the workflow runs after a branch is merged to `main`, there is a slight delay for search data updates to appear on the site. + ## Manual sync from a checkout It is also possible to manually sync the indices to Algolia from your local checkout of the repo, before your branch is merged to `main`. @@ -64,7 +66,7 @@ You can set `VERSION` and `LANGUAGE` individually, too. Substitute a currently supported version for `` and a currently supported two-letter language code for ``. -## Label-triggered workflow +## Label-triggered Actions workflow Docs team members can use an Actions workflow on GHES release PRs by applying a label in this format: ``` @@ -80,11 +82,11 @@ Why do we need this? For our daily shipping needs, it's tolerable that search up ## Files -### GitHub Actions workflow files +### Actions workflow files - [`.github/workflows/sync-algolia-search-indices.yml`](.github/workflows/sync-algolia-search-indices.yml) - Builds and syncs search indices whenever the `main` branch is pushed to (that is, on production deploys). - [`.github/workflows/dry-run-sync-algolia-search-indices.yml`](.github/workflows/dry-run-sync-algolia-search-indices.yml) - This workflow can be run manually (via `workflow_dispatch`) to do a dry run build of all the indices. Useful for confirming that the indices can build without erroring out. -- [`.github/workflows/sync-single-english-algolia-index.yml`](.github/workflows/sync-single-english-algolia-index.yml) - This workflow is run when a label in the right format is applied to a PR. See "[Label-triggered workflow](#label-triggered-workflow)" for details. +- [`.github/workflows/sync-single-english-algolia-index.yml`](.github/workflows/sync-single-english-algolia-index.yml) - This workflow is run when a label in the right format is applied to a PR. See "[Label-triggered Actions workflow](#label-triggered-actions-workflow)" for details. ### Code files