Move search files to src (#35822)

Co-authored-by: Peter Bengtsson <peterbe@github.com>

.github/workflows/sync-search-elasticsearch.yml

@@ -183,7 +183,7 @@ jobs:
           # not exist.
           VERSION: ${{ github.event.inputs.version }}
         run: |
-          ./script/search/index-elasticsearch.js /tmp/records \
+          ./src/search/scripts/index-elasticsearch.js /tmp/records \
             --language ${{ matrix.language }} \
 
       - name: Check created indexes and aliases

.github/workflows/sync-search-pr.yml

@@ -8,7 +8,7 @@ name: Sync search - PR
 on:
   pull_request:
     paths:
-      - 'script/search/**'
+      - 'src/search/**'
       - 'package*.json'
       # Ultimately, for debugging this workflow itself
       - .github/workflows/sync-search-pr.yml
@@ -107,7 +107,7 @@ jobs:
 
       - name: Index into Elasticsearch
         run: |
-          ./script/search/index-elasticsearch.js /tmp/records \
+          ./src/search/scripts/index-elasticsearch.js /tmp/records \
             --language en \
             --version dotcom
 

.github/workflows/test.yml

@@ -53,6 +53,7 @@ jobs:
               { name: 'routing', path: 'tests/routing', },
               { name: 'rendering', path: 'tests/rendering', },
               { name: 'rendering-fixtures', path: 'tests/rendering-fixtures', },
+              { name: 'search', path: 'src/search/tests', },
               context.payload.repository.full_name === 'github/docs-internal' &&
                 { name: 'translations', path: 'tests/translations', },
               { name: 'unit', path: 'tests/unit', },
@@ -76,7 +77,7 @@ jobs:
         uses: actions/checkout@93ea575cb5d8a053eaa0ac8fa3b40d7e05a33cc8
 
       - uses: ./.github/actions/setup-elasticsearch
-        if: ${{ matrix.name == 'content' || matrix.name == 'translations' }}
+        if: ${{ matrix.name == 'search' || matrix.name == 'translations' }}
 
       - uses: ./.github/actions/node-npm-setup
 
@@ -165,7 +166,7 @@ jobs:
       - name: Index fixtures into the local Elasticsearch
         # For the sake of saving time, only run this step if the group
         # is one that will run tests against an Elasticsearch on localhost.
-        if: ${{ matrix.name == 'content' || matrix.name == 'translations' }}
+        if: ${{ matrix.name == 'search' || matrix.name == 'translations' }}
         run: npm run index-test-fixtures
 
       - name: Run tests

components/page-header/Breadcrumbs.tsx

@@ -22,7 +22,7 @@ export const Breadcrumbs = ({ inHeader }: Props) => {
       NOTE: The breadcrumbs class and the nav tag are used by the
       Lunr search scripts. The a tag generated by the Link is also used.
       If these change, please also change
-      updating script/search/parse-page-sections-into-records.js.
+      updating src/search/scripts/parse-page-sections-into-records.js.
     */
     <nav
       data-testid={inHeader ? 'breadcrumbs-header' : 'breadcrumbs-in-article'}

components/page-header/Header.tsx

@@ -19,7 +19,7 @@ import { LanguagePicker } from './LanguagePicker'
 import { HeaderNotifications } from 'components/page-header/HeaderNotifications'
 import { ApiVersionPicker } from 'components/sidebar/ApiVersionPicker'
 import { useTranslation } from 'components/hooks/useTranslation'
-import { Search } from 'components/Search'
+import { Search } from 'src/search/components/Search'
 import { Breadcrumbs } from 'components/page-header/Breadcrumbs'
 import { VersionPicker } from 'components/page-header/VersionPicker'
 import { SidebarNav } from 'components/sidebar/SidebarNav'

middleware/api/index.js

@@ -3,7 +3,7 @@ import { createProxyMiddleware } from 'http-proxy-middleware'
 
 import events from '../../src/events/middleware.js'
 import anchorRedirect from '../../src/rest/api/anchor-redirect.js'
-import search from './search.js'
+import search from '../../src/search/middleware/search.js'
 import webhooks from './webhooks.js'
 
 const router = express.Router()

middleware/context.js

@@ -5,7 +5,7 @@ import { productMap } from '../lib/all-products.js'
 import pathUtils from '../lib/path-utils.js'
 import productNames from '../lib/product-names.js'
 import warmServer from '../lib/warm-server.js'
-import searchVersions from '../lib/search/versions.js'
+import searchVersions from '../src/search/lib/versions.js'
 import nonEnterpriseDefaultVersion from '../lib/non-enterprise-default-version.js'
 import { getDataByLanguage, getUIDataMerged } from '../lib/get-data.js'
 

package.json

@@ -194,7 +194,7 @@
     "dev": "cross-env npm start",
     "fixture-dev": "cross-env ROOT=tests/fixtures npm start",
     "fixture-test": "cross-env ROOT=tests/fixtures npm test -- tests/rendering-fixtures",
-    "index-test-fixtures": "node script/search/index-elasticsearch.js -l en -l ja -V ghae -V dotcom --index-prefix tests -- tests/content/fixtures/search-indexes",
+    "index-test-fixtures": "node src/search/scripts/index-elasticsearch.js -l en -l ja -V ghae -V dotcom --index-prefix tests -- src/search/tests/fixtures/search-indexes",
     "lint": "eslint '**/*.{js,mjs,ts,tsx}'",
     "lint-translation": "cross-env NODE_OPTIONS=--experimental-vm-modules jest tests/linting/lint-files.js",
     "playwright-test": "playwright test --project=\"Google Chrome\"",
@@ -210,7 +210,7 @@
     "start-for-playwright": "cross-env ROOT=tests/fixtures NODE_ENV=test node server.js",
     "sync-search": "cross-env NODE_OPTIONS='--max_old_space_size=8192' start-server-and-test sync-search-server 4002 sync-search-indices",
     "sync-search-ghes-release": "cross-env GHES_RELEASE=1 start-server-and-test sync-search-server 4002 sync-search-indices",
-    "sync-search-indices": "script/search/sync-search-indices.js",
+    "sync-search-indices": "src/search/scripts/sync-search-indices.js",
     "sync-search-server": "cross-env NODE_ENV=production PORT=4002 MINIMAL_RENDER=true CHANGELOG_DISABLED=true node server.js",
     "translation-check": "start-server-and-test translation-check-server 4002 translation-check-test",
     "translation-check-server": "cross-env NODE_ENV=test PORT=4002 node server.js",

pages/[versionId]/search.tsx

@@ -1,4 +1 @@
-import Search from '../search'
+export { default, getServerSideProps } from '../search'
-export { getServerSideProps } from '../search'
-
-export default Search

pages/search.tsx

@@ -1,46 +1 @@
-import type { GetServerSideProps } from 'next'
+export { default, getServerSideProps } from '../src/search/pages/search'
-
-import searchVersions from '../lib/search/versions.js'
-
-import { MainContextT, MainContext, getMainContext } from 'components/context/MainContext'
-import { DefaultLayout } from 'components/DefaultLayout'
-import { Search } from 'components/search/index'
-
-type Props = {
-  mainContext: MainContextT
-}
-
-export default function Page({ mainContext }: Props) {
-  return (
-    <MainContext.Provider value={mainContext}>
-      <DefaultLayout>
-        <Search />
-      </DefaultLayout>
-    </MainContext.Provider>
-  )
-}
-
-export const getServerSideProps: GetServerSideProps<Props> = async (context) => {
-  const req = context.req as any
-  const res = context.res as any
-
-  const version = req.context.currentVersion
-
-  const searchVersion = searchVersions[Array.isArray(version) ? version[0] : version]
-  if (!searchVersion) {
-    // E.g. someone loaded `/en/enterprisy-server@2.99/search`
-    // That's going to 404 in the XHR later but it simply shouldn't be
-    // a valid starting page.
-    return {
-      notFound: true,
-    }
-  }
-
-  const mainContext = await getMainContext(req, res)
-
-  return {
-    props: {
-      mainContext,
-    },
-  }
-}

script/README.md

@@ -632,7 +632,7 @@ See how a piece of text gets turned into tokens by the different analyzers. Requ
 
 Example:
 
-   ./script/search/analyze-text.js my words to tokenize
+   ./src/scripts/search/analyze-text.js my words to tokenize
 
 ---
 

src/search/README.md

@@ -15,9 +15,9 @@ To see all existing search-related issues and pull requests, visit [github.com/g
 ## How to search
 
 The site search is part of every version of docs.github.com. This endpoint responds in JSON format, and fronts our search querying functionality. We recommend using this endpoint, as the endpoint will be more stable. On any page, you can use the search box to search the documents we've indexed.
-You can also query our search endpoint directly at: 
+You can also query our search endpoint directly at:
 `https://docs.github.com/search?version=<VERSION>&language=<LANGUAGE CODE>&filters=topics:<TOPIC>&query=<QUERY>`
- 
+
 - The VERSION can be any numbered GitHub Enterprise Server version (e.g., `2.22`, `3.0`), GitHub AE (`ghae`), or the Free pro team plan (`dotcom`).
 - The LANGUAGE CODE can be one of: `zh`, `es`, `pt`, `ru`, `ja`, `fr`, `de`, `ko`
 - TOPIC can be any topics in [the allowed list of topics](/data/allowed-topics.js). The values in the `topics` attribute are **not** case sensitive, so filtering on `GitHub actions` or `github actions` will return the same result. **Note:** Currently, the topics filter only works for the dotcom version in the English language. We plan to expand this search query to other languages and versions in the future.
@@ -43,7 +43,7 @@ The Actions workflow progress can be viewed (by GitHub employees) in the [Action
 
 ## Manually triggering the search index update workflow
 
-You can manually run the workflow to generate the indexes after you push your changes to `main` to speed up the indexing when needed. It's recommended to do this for only the `free-pro-team@latest` version and the `en` language because running all languages and versions takes about 40 minutes. To run it manually, click "Run workflow" button in the [Actions tab](https://github.com/github/docs-internal/actions/workflows/sync-search-indices.yml). Enter the language and version you'd like to generate the indexes for as inputs to the workflow. By default, all languages and versions are generated.
+You can manually run the workflow to generate the indexes after you push your changes to `main` to speed up the indexing when needed. It's recommended to do this for only the `free-pro-team@latest` version and the `en` language because running all languages and versions takes about 40 minutes. To run it manually, click "Run workflow" button in the Actions tab. Enter the language and version you'd like to generate the indexes for as inputs to the workflow. By default, all languages and versions are generated.
 
 ### Build and sync
 
@@ -77,7 +77,7 @@ Why do we need this? For our daily shipping needs, it's tolerable that search up
 
 ### Actions workflow files
 
-- [`.github/workflows/sync-search-indices.yml`](.github/workflows/sync-search-indices.yml) - Builds and syncs search indices on the `main` branch every four hours. Search indices are committed directly to the `main` branch on both the `github/docs-internal` and `github/docs` repositories. It can also be run manually. To run it manually, click "Run workflow" button in the [Actions tab](https://github.com/github/docs-internal/actions/workflows/sync-search-indices.yml).
+- [`.github/workflows/sync-search-indices.yml`](.github/workflows/sync-search-indices.yml) - Builds and syncs search indices on the `main` branch every four hours. Search indices are committed directly to the `main` branch on both the `github/docs-internal` and `github/docs` repositories. It can also be run manually. To run it manually, click "Run workflow" button in the Actions tab.
 - [`.github/workflows/sync-single-english-index.yml`](.github/workflows/sync-single-english-index.yml) - This workflow is run when a label in the right format is applied to a PR. See "[Label-triggered Actions workflow](#label-triggered-actions-workflow)" for details.
 
 ### Code files

src/search/components/Search.tsx

@@ -6,7 +6,7 @@ import { SearchIcon } from '@primer/octicons-react'
 import { useTranslation } from 'components/hooks/useTranslation'
 import { DEFAULT_VERSION, useVersion } from 'components/hooks/useVersion'
 import { useQuery } from 'components/hooks/useQuery'
-import { useBreakpoint } from './hooks/useBreakpoint'
+import { useBreakpoint } from 'components/hooks/useBreakpoint'
 
 export function Search() {
   const router = useRouter()

src/search/components/index.tsx

@@ -7,11 +7,11 @@ import { sendEvent, EventType } from 'src/events/browser'
 import { useTranslation } from 'components/hooks/useTranslation'
 import { DEFAULT_VERSION, useVersion } from 'components/hooks/useVersion'
 import { useNumberFormatter } from 'components/hooks/useNumberFormatter'
-import type { SearchResultsT } from 'components/search/types'
+import type { SearchResultsT } from 'src/search/components/types'
-import { SearchResults } from 'components/search/SearchResults'
+import { SearchResults } from 'src/search/components/SearchResults'
-import { SearchError } from 'components/search/SearchError'
+import { SearchError } from 'src/search/components/SearchError'
-import { NoQuery } from 'components/search/NoQuery'
+import { NoQuery } from 'src/search/components/NoQuery'
-import { Loading } from 'components/search/Loading'
+import { Loading } from 'src/search/components/Loading'
 import { useQuery } from 'components/hooks/useQuery'
 import { usePage } from 'components/hooks/usePage'
 import { useMainContext } from 'components/context/MainContext'

src/search/lib/versions.js

@@ -1,4 +1,4 @@
-import { allVersions } from '../all-versions.js'
+import { allVersions } from '../../../lib/all-versions.js'
 
 export default Object.fromEntries(
   Object.entries(allVersions).map(([versionStr, versionObject]) => [

src/search/middleware/search.js

@@ -1,12 +1,12 @@
 import express from 'express'
 
-import FailBot from '../../lib/failbot.js'
+import FailBot from '../../../lib/failbot.js'
-import languages from '../../lib/languages.js'
+import languages from '../../../lib/languages.js'
-import { allVersions } from '../../lib/all-versions.js'
+import { allVersions } from '../../../lib/all-versions.js'
-import statsd from '../../lib/statsd.js'
+import statsd from '../../../lib/statsd.js'
-import { searchCacheControl } from '../cache-control.js'
+import { searchCacheControl } from '../../../middleware/cache-control.js'
-import catchMiddlewareError from '../catch-middleware-error.js'
+import catchMiddlewareError from '../../../middleware/catch-middleware-error.js'
-import { setFastlySurrogateKey } from '../set-fastly-surrogate-key.js'
+import { setFastlySurrogateKey } from '../../../middleware/set-fastly-surrogate-key.js'
 import {
   getSearchResults,
   POSSIBLE_HIGHLIGHT_FIELDS,

src/search/pages/search.tsx

@@ -0,0 +1,44 @@
+import type { GetServerSideProps } from 'next'
+
+import { MainContextT, MainContext, getMainContext } from 'components/context/MainContext'
+import { DefaultLayout } from 'components/DefaultLayout'
+import { Search } from 'src/search/components/index'
+
+type Props = {
+  mainContext: MainContextT
+}
+
+export default function Page({ mainContext }: Props) {
+  return (
+    <MainContext.Provider value={mainContext}>
+      <DefaultLayout>
+        <Search />
+      </DefaultLayout>
+    </MainContext.Provider>
+  )
+}
+
+export const getServerSideProps: GetServerSideProps<Props> = async (context) => {
+  const req = context.req as any
+  const res = context.res as any
+
+  const version = req.context.currentVersion
+
+  const searchVersion = req.context.searchVersions[Array.isArray(version) ? version[0] : version]
+  if (!searchVersion) {
+    // E.g. someone loaded `/en/enterprisy-server@2.99/search`
+    // That's going to 404 in the XHR later but it simply shouldn't be
+    // a valid starting page.
+    return {
+      notFound: true,
+    }
+  }
+
+  const mainContext = await getMainContext(req, res)
+
+  return {
+    props: {
+      mainContext,
+    },
+  }
+}

src/search/scripts/analyze-text.js

@@ -8,7 +8,7 @@
 //
 // Example:
 //
-//    ./script/search/analyze-text.js my words to tokenize
+//    ./src/search/scripts/analyze-text.js my words to tokenize
 //
 // [end-readme]
 
@@ -17,8 +17,8 @@ import { program, Option } from 'commander'
 import chalk from 'chalk'
 import dotenv from 'dotenv'
 
-import { languageKeys } from '../../lib/languages.js'
+import { languageKeys } from '../../../lib/languages.js'
-import { allVersions } from '../../lib/all-versions.js'
+import { allVersions } from '../../../lib/all-versions.js'
 
 // Now you can optionally have set the ELASTICSEARCH_URL in your .env file.
 dotenv.config()

src/search/scripts/build-records.js

@@ -7,8 +7,8 @@ import { HTTPError } from 'got'
 
 import parsePageSectionsIntoRecords from './parse-page-sections-into-records.js'
 import getPopularPages from './popular-pages.js'
-import languages from '../../lib/languages.js'
+import languages from '../../../lib/languages.js'
-import domwaiter from '../domwaiter.js'
+import domwaiter from '../../../script/domwaiter.js'
 
 const pageMarker = chalk.green('|')
 const recordMarker = chalk.grey('.')

src/search/scripts/find-indexable-pages.js

@@ -1,5 +1,5 @@
 #!/usr/bin/env node
-import { loadPages } from '../../lib/page-data.js'
+import { loadPages } from '../../../lib/page-data.js'
 
 export default async function findIndexablePages(match = '') {
   const allPages = await loadPages()

src/search/scripts/index-elasticsearch.js

@@ -15,10 +15,10 @@ import { program, Option } from 'commander'
 import chalk from 'chalk'
 import dotenv from 'dotenv'
 
-import { retryOnErrorTest } from '../helpers/retry-on-error-test.js'
+import { retryOnErrorTest } from '../../../script/helpers/retry-on-error-test.js'
-import { languageKeys } from '../../lib/languages.js'
+import { languageKeys } from '../../../lib/languages.js'
-import { allVersions } from '../../lib/all-versions.js'
+import { allVersions } from '../../../lib/all-versions.js'
-import statsd from '../../lib/statsd.js'
+import statsd from '../../../lib/statsd.js'
 
 // Now you can optionally have set the ELASTICSEARCH_URL in your .env file.
 dotenv.config()

src/search/scripts/sync-search-indices.js

@@ -12,8 +12,8 @@ import { existsSync } from 'fs'
 import assert from 'assert'
 import { program, Option } from 'commander'
 
-import { languageKeys } from '../../lib/languages.js'
+import { languageKeys } from '../../../lib/languages.js'
-import { allVersions } from '../../lib/all-versions.js'
+import { allVersions } from '../../../lib/all-versions.js'
 import searchSync from './sync.js'
 
 const shortNames = Object.fromEntries(

src/search/scripts/sync.js

@@ -1,11 +1,11 @@
 #!/usr/bin/env node
 import chalk from 'chalk'
 
-import languages from '../../lib/languages.js'
+import languages from '../../../lib/languages.js'
 import buildRecords from './build-records.js'
 import findIndexablePages from './find-indexable-pages.js'
-import { allVersions } from '../../lib/all-versions.js'
+import { allVersions } from '../../../lib/all-versions.js'
-import { namePrefix } from '../../lib/search/config.js'
+import { namePrefix } from '../../../src/search/lib/config.js'
 import { writeIndexRecords } from './search-index-records.js'
 
 // Build a search data file for every combination of product version and language

src/search/tests/api-search.js

@@ -13,8 +13,8 @@
 
 import { jest, test, expect } from '@jest/globals'
 
-import { describeIfElasticsearchURL } from '../helpers/conditional-runs.js'
+import { describeIfElasticsearchURL } from '../../../tests/helpers/conditional-runs.js'
-import { get } from '../helpers/e2etest.js'
+import { get } from '../../../tests/helpers/e2etest.js'
 
 if (!process.env.ELASTICSEARCH_URL) {
   console.warn(
@@ -30,7 +30,7 @@ describeIfElasticsearchURL('search v1 middleware', () => {
   test('basic search', async () => {
     const sp = new URLSearchParams()
     // To see why this will work,
-    // see tests/content/fixtures/search-indexes/github-docs-dotcom-en-records.json
+    // see src/search/tests/fixtures/search-indexes/github-docs-dotcom-en-records.json
     // which clearly has a record with the title "Foo"
     sp.set('query', 'foo')
     const res = await get('/api/search/v1?' + sp)

src/search/tests/parse-page-sections-into-records.js

@@ -5,7 +5,7 @@ import fs from 'fs/promises'
 import cheerio from 'cheerio'
 import { expect, test } from '@jest/globals'
 
-import parsePageSectionsIntoRecords from '../../../script/search/parse-page-sections-into-records.js'
+import parsePageSectionsIntoRecords from '../scripts/parse-page-sections-into-records'
 const __dirname = path.dirname(fileURLToPath(import.meta.url))
 
 const fixtures = {

src/search/tests/search.js

@@ -1,6 +1,6 @@
 import { expect, jest } from '@jest/globals'
 
-import { getDOM } from '../helpers/e2etest.js'
+import { getDOM } from '../../../tests/helpers/e2etest.js'
 
 describe('search results page', () => {
   jest.setTimeout(5 * 60 * 1000)

tests/browser/browser.js

@@ -23,7 +23,7 @@ describe('homepage', () => {
 })
 
 // Note: we can only test Elasticsearch searches on things we have indexed
-// in the fixtures. See the contents of /tests/content/fixtures/search-indexes/
+// in the fixtures. See the contents of /src/search/tests/fixtures/search-indexes
 describe('browser search', () => {
   jest.setTimeout(60 * 1000)
 

tests/translations/api-search.js

@@ -10,7 +10,7 @@ describeIfElasticsearchURL('search v1 middleware in non-English', () => {
   test('basic search in Japanese', async () => {
     const sp = new URLSearchParams()
     // To see why this will work,
-    // see tests/content/fixtures/search-indexes/github-docs-dotcom-en-records.json
+    // see src/search/tests/fixtures/search-indexes/github-docs-dotcom-en-records.json
     // which clearly has a record with the title "Foo"
     sp.set('query', 'foo')
     sp.set('language', 'ja')