diff --git a/.github/workflows/js-lint.yml b/.github/workflows/js-lint.yml index 24b0305b9a..22415aa9fe 100644 --- a/.github/workflows/js-lint.yml +++ b/.github/workflows/js-lint.yml @@ -29,3 +29,6 @@ jobs: - name: Run linter run: npm run lint + + - name: Run TypeScript + run: npm run tsc diff --git a/assets/images/help/teams/review-assignment-button.png b/assets/images/help/teams/review-assignment-button.png deleted file mode 100644 index 412fa4bc78..0000000000 Binary files a/assets/images/help/teams/review-assignment-button.png and /dev/null differ diff --git a/assets/images/help/teams/review-assignment-notifications.png b/assets/images/help/teams/review-assignment-notifications.png index c32a5723b3..150d348004 100644 Binary files a/assets/images/help/teams/review-assignment-notifications.png and b/assets/images/help/teams/review-assignment-notifications.png differ diff --git a/assets/images/help/teams/review-button.png b/assets/images/help/teams/review-button.png new file mode 100644 index 0000000000..8595920551 Binary files /dev/null and b/assets/images/help/teams/review-button.png differ diff --git a/components/Search.module.scss b/components/Search.module.scss index 9f045b5149..d4f73490b9 100644 --- a/components/Search.module.scss +++ b/components/Search.module.scss @@ -11,6 +11,10 @@ border-bottom: 1px solid currentColor; } +.searchResultContent { + max-height: 4rem; +} + .searchResultContent mark { text-decoration: none; border-bottom: 1px dotted currentColor; diff --git a/components/Search.tsx b/components/Search.tsx index 2342a66bdb..3c9b4c89fc 100644 --- a/components/Search.tsx +++ b/components/Search.tsx @@ -14,9 +14,10 @@ import styles from './Search.module.scss' type SearchResult = { url: string breadcrumbs: string - heading: string title: string content: string + score: number + popularity: number } type Props = { @@ -56,6 +57,10 @@ export function Search({ } }, []) + useEffect(() => { + closeSearch() + }, [currentVersion, language]) + // Search with your keyboard useEffect(() => { document.addEventListener('keydown', searchWithYourKeyboard) @@ -191,6 +196,7 @@ export function Search({ activeHit={activeHit} setActiveHit={setActiveHit} onGotoResult={onGotoResult} + debug={'debug' in router.query} /> {/* eslint-disable-next-line jsx-a11y/click-events-have-key-events, jsx-a11y/no-static-element-interactions */} @@ -260,6 +266,7 @@ function ShowSearchResults({ activeHit, setActiveHit, onGotoResult, + debug, }: { isOverlay: boolean isLoading: boolean @@ -267,6 +274,7 @@ function ShowSearchResults({ activeHit: number setActiveHit: (index: number) => void onGotoResult: (url: string, index: number) => void + debug: boolean }) { const { t } = useTranslation('search') @@ -290,7 +298,7 @@ function ShowSearchResults({ // When there are search results, it doesn't matter if this is overlay or not. return (
    - {results.map(({ url, breadcrumbs, heading, title, content }, index) => { + {results.map(({ url, breadcrumbs, title, content, score, popularity }, index) => { const isActive = index === activeHit return (
  1. + {debug && ( + + score: {score.toFixed(4)} popularity: {popularity.toFixed(4)} + + )}
    -
    + {content ? ( +
    + ) : ( +
    + {t('no_content')} +
    + )}
  2. diff --git a/components/article/ArticlePage.tsx b/components/article/ArticlePage.tsx index 40474ba91c..2e0b8d339c 100644 --- a/components/article/ArticlePage.tsx +++ b/components/article/ArticlePage.tsx @@ -72,7 +72,11 @@ export const ArticlePage = () => { )} - {intro && {intro}} + {intro && ( + + {intro} + + )} {permissions && (
    diff --git a/components/landing/LandingHero.tsx b/components/landing/LandingHero.tsx index e6c96cf934..253a5c08d3 100644 --- a/components/landing/LandingHero.tsx +++ b/components/landing/LandingHero.tsx @@ -28,7 +28,7 @@ export const LandingHero = () => { {beta_product && Beta} - {intro && {intro}} + {intro && {intro}} {introLinks && Object.entries(introLinks) diff --git a/components/landing/TocLanding.tsx b/components/landing/TocLanding.tsx index 767312dabb..9d02636d99 100644 --- a/components/landing/TocLanding.tsx +++ b/components/landing/TocLanding.tsx @@ -29,7 +29,7 @@ export const TocLanding = () => { {title} - {introPlainText && {introPlainText}} + {introPlainText && {introPlainText}} {productCallout && ( diff --git a/components/page-header/Header.tsx b/components/page-header/Header.tsx index 7623943f62..316834d35c 100644 --- a/components/page-header/Header.tsx +++ b/components/page-header/Header.tsx @@ -53,12 +53,15 @@ export const Header = () => { className="d-none d-lg-flex flex-justify-end flex-items-center flex-wrap flex-xl-nowrap" data-testid="desktop-header" > -
    +
    - +
    diff --git a/components/page-header/VersionPicker.tsx b/components/page-header/VersionPicker.tsx index e78853efdc..8568c212cf 100644 --- a/components/page-header/VersionPicker.tsx +++ b/components/page-header/VersionPicker.tsx @@ -9,7 +9,7 @@ import { useVersion } from 'components/hooks/useVersion' import { useTranslation } from 'components/hooks/useTranslation' type Props = { - variant?: 'inline' + variant?: 'inline' | 'compact' } export const VersionPicker = ({ variant }: Props) => { const router = useRouter() diff --git a/components/sublanding/SubLandingHero.tsx b/components/sublanding/SubLandingHero.tsx index dcf5e4e22e..c0b33c8747 100644 --- a/components/sublanding/SubLandingHero.tsx +++ b/components/sublanding/SubLandingHero.tsx @@ -46,7 +46,7 @@ export const SubLandingHero = () => {

    {title} guides

    - {intro && {intro}} + {intro && {intro}}
    {featuredTrack && ( diff --git a/content/account-and-profile/setting-up-and-managing-your-github-profile/customizing-your-profile/personalizing-your-profile.md b/content/account-and-profile/setting-up-and-managing-your-github-profile/customizing-your-profile/personalizing-your-profile.md index 2a8be7f8f7..060afa3657 100644 --- a/content/account-and-profile/setting-up-and-managing-your-github-profile/customizing-your-profile/personalizing-your-profile.md +++ b/content/account-and-profile/setting-up-and-managing-your-github-profile/customizing-your-profile/personalizing-your-profile.md @@ -109,7 +109,7 @@ When you set your status, you can also let people know that you have limited ava ![Requested reviewer shows "busy" note next to username](/assets/images/help/profile/request-a-review-limited-availability-status.png) -If you select the "Busy" option, when people @mention your username, assign you an issue or pull request, or request a pull request review from you, a note next to your username will show that you're busy. You will also be excluded from automatic review assignment for pull requests assigned to any teams you belong to. For more information, see "[Managing code review assignment for your team](/organizations/organizing-members-into-teams/managing-code-review-assignment-for-your-team)." +If you select the "Busy" option, when people @mention your username, assign you an issue or pull request, or request a pull request review from you, a note next to your username will show that you're busy. You will also be excluded from automatic review assignment for pull requests assigned to any teams you belong to. For more information, see "[Managing code review settings for your team](/organizations/organizing-members-into-teams/managing-code-review-settings-for-your-team)." 1. In the top right corner of {% ifversion fpt or ghec %}{% data variables.product.prodname_dotcom_the_website %}{% else %}{% data variables.product.product_name %}{% endif %}, click your profile photo, then click **Set your status** or, if you already have a status set, click your current status. ![Button on profile to set your status](/assets/images/help/profile/set-status-on-profile.png) diff --git a/content/organizations/managing-peoples-access-to-your-organization-with-roles/roles-in-an-organization.md b/content/organizations/managing-peoples-access-to-your-organization-with-roles/roles-in-an-organization.md index cd610f190a..f45688150f 100644 --- a/content/organizations/managing-peoples-access-to-your-organization-with-roles/roles-in-an-organization.md +++ b/content/organizations/managing-peoples-access-to-your-organization-with-roles/roles-in-an-organization.md @@ -185,7 +185,7 @@ Some of the features listed below are limited to organizations using {% data var | Reinstate former members to the organization | **X** | | | | Add and remove people from **all teams** | **X** | | | Promote organization members to *team maintainer* | **X** | | -| Configure code review assignments (see "[Managing code review assignment for your team](/organizations/organizing-members-into-teams/managing-code-review-assignment-for-your-team)") | **X** | | +| Configure code review assignments (see "[Managing code review settings for your team](/organizations/organizing-members-into-teams/managing-code-review-settings-for-your-team)")) | **X** | | | Add collaborators to **all repositories** | **X** | | | Access the organization audit log | **X** | | | Edit the organization's profile page (see "[About your organization's profile](/github/setting-up-and-managing-your-github-profile/customizing-your-profile/about-your-organizations-profile)" for details) | **X** | | |{% ifversion ghes > 3.1 %} diff --git a/content/organizations/organizing-members-into-teams/index.md b/content/organizations/organizing-members-into-teams/index.md index 801d81f87b..e3c495fc00 100644 --- a/content/organizations/organizing-members-into-teams/index.md +++ b/content/organizations/organizing-members-into-teams/index.md @@ -26,7 +26,7 @@ children: - /adding-organization-members-to-a-team - /assigning-the-team-maintainer-role-to-a-team-member - /setting-your-teams-profile-picture - - /managing-code-review-assignment-for-your-team + - /managing-code-review-settings-for-your-team - /renaming-a-team - /changing-team-visibility - /synchronizing-a-team-with-an-identity-provider-group diff --git a/content/organizations/organizing-members-into-teams/managing-code-review-assignment-for-your-team.md b/content/organizations/organizing-members-into-teams/managing-code-review-settings-for-your-team.md similarity index 61% rename from content/organizations/organizing-members-into-teams/managing-code-review-assignment-for-your-team.md rename to content/organizations/organizing-members-into-teams/managing-code-review-settings-for-your-team.md index 423580ca3b..37e0b02865 100644 --- a/content/organizations/organizing-members-into-teams/managing-code-review-assignment-for-your-team.md +++ b/content/organizations/organizing-members-into-teams/managing-code-review-settings-for-your-team.md @@ -1,8 +1,9 @@ --- -title: Managing code review assignment for your team -intro: Code review assignments clearly indicate which members of a team are expected to submit a review for a pull request. +title: Managing code review settings for your team +intro: You can decrease noise for your team by limiting notifications when your team is requested to review a pull request. redirect_from: - /github/setting-up-and-managing-organizations-and-teams/managing-code-review-assignment-for-your-team + - /organizations/organizing-members-into-teams/managing-code-review-assignment-for-your-team product: '{% data reusables.gated-features.code-review-assignment %}' versions: fpt: '*' @@ -12,13 +13,26 @@ versions: topics: - Organizations - Teams -shortTitle: Code review assignment -permissions: Team maintainers and organization owners can configure code review assignments. +shortTitle: Code review settings +permissions: Team maintainers and organization owners can configure code review settings. --- -## About code review assignments +## About code review settings -By using code review assignments, any time your team has been requested to review a pull request, the team is removed as a reviewer and a specified subset of team members are assigned in the team's place. Code review assignments allow you to decide whether the whole team or just a subset of team members are notified when a team is requested for review. +{% if only-notify-requested-members %} +To reduce noise for your team and clarify individual responsibility for pull request reviews, you can configure code review settings. + +- Team notifications +- Auto assignment + +## About team notifications + +When you choose to only notify requested team members, you disable sending notifications to the entire team when the team is requested to review a pull request if a specific member of that team is also requested for review. This is especially useful when a repository is configured with teams as code owners, but contributors to the repository often know a specific individual that would be the correct reviewer for their pull request. For more information, see "[About code owners](/github/creating-cloning-and-archiving-repositories/about-code-owners)." + +## About auto assignment +{% endif %} + +When you enable auto assignment, any time your team has been requested to review a pull request, the team is removed as a reviewer and a specified subset of team members are assigned in the team's place. Code review assignments allow you to decide whether the whole team or just a subset of team members are notified when a team is requested for review. When code owners are automatically requested for review, the team is still removed and replaced with individuals. The individual approvals don't satisfy the requirement for code owner approval in a protected branch. For more information, see "[About code owners](/github/creating-cloning-and-archiving-repositories/about-code-owners)." @@ -26,7 +40,7 @@ When code owners are automatically requested for review, the team is still remov To further enhance your team's collaboration abilities, you can upgrade to {% data variables.product.prodname_ghe_cloud %}, which includes features like protected branches and code owners on private repositories. {% data reusables.enterprise.link-to-ghec-trial %} {% endif %} -## Routing algorithms +### Routing algorithms Code review assignments automatically choose and assign reviewers based on one of two possible algorithms. @@ -36,29 +50,45 @@ The load balance algorithm chooses reviewers based on each member's total number Any team members that have set their status to "Busy" will not be selected for review. If all team members are busy, the pull request will remain assigned to the team itself. For more information about user statuses, see "[Setting a status](/account-and-profile/setting-up-and-managing-your-github-profile/customizing-your-profile/personalizing-your-profile#setting-a-status)." -## Configuring code review assignment +{% if only-notify-requested-members %} +## Configuring team notifications + {% data reusables.profile.access_org %} {% data reusables.user_settings.access_org %} {% data reusables.organizations.specific_team %} {% data reusables.organizations.team_settings %} -5. In the left sidebar, click **Code review assignment** -![Code review assignment button](/assets/images/help/teams/review-assignment-button.png) +5. In the left sidebar, click **Code review** +![Code review button](/assets/images/help/teams/review-button.png) +2. Select **Only notify requested team members.** +![Code review team notifications](/assets/images/help/teams/review-assignment-notifications.png) +3. Click **Save changes**. +{% endif %} + +## Configuring auto assignment +{% data reusables.profile.access_org %} +{% data reusables.user_settings.access_org %} +{% data reusables.organizations.specific_team %} +{% data reusables.organizations.team_settings %} +5. In the left sidebar, click **Code review** +![Code review button](/assets/images/help/teams/review-button.png) 6. Select **Enable auto assignment**. -![Code review assignment button](/assets/images/help/teams/review-assignment-enable.png) +![Auto-assignment button](/assets/images/help/teams/review-assignment-enable.png) 7. Under "How many team members should be assigned to review?", use the drop-down menu and choose a number of reviewers to be assigned to each pull request. ![Number of reviewers dropdown](/assets/images/help/teams/review-assignment-number.png) 8. Under "Routing algorithm", use the drop-down menu and choose which algorithm you'd like to use. For more information, see "[Routing algorithms](#routing-algorithms)." ![Routing algorithm dropdown](/assets/images/help/teams/review-assignment-algorithm.png) 9. Optionally, to always skip certain members of the team, select **Never assign certain team members**. Then, select one or more team members you'd like to always skip. ![Never assign certain team members checkbox and dropdown](/assets/images/help/teams/review-assignment-skip-members.png) -10. Optionally, to only notify the team members chosen by code review assignment for each pull review request, under "Notifications" select **If assigning team members, don't notify the entire team.** -![Code review assignment notifications](/assets/images/help/teams/review-assignment-notifications.png){% ifversion fpt or ghae or ghes > 3.2 or ghec %} +{% ifversion fpt or ghec or ghae-next or ghes > 3.2 %} 11. Optionally, to include members of child teams as potential reviewers when assigning requests, select **Child team members**. 12. Optionally, to count any members whose review has already been requested against the total number of members to assign, select **Count existing requests**. -13. Optionally, to remove the review request from the team when assigning team members, select **Team review request**.{% endif %} +13. Optionally, to remove the review request from the team when assigning team members, select **Team review request**. +{%- else %} +10. Optionally, to only notify the team members chosen by code review assignment for each pull review request, under "Notifications" select **If assigning team members, don't notify the entire team.** +{%- endif %} 14. Click **Save changes**. -## Disabling code review assignment +## Disabling auto assignment {% data reusables.profile.access_org %} {% data reusables.user_settings.access_org %} {% data reusables.organizations.specific_team %} diff --git a/content/pull-requests/collaborating-with-pull-requests/proposing-changes-to-your-work-with-pull-requests/requesting-a-pull-request-review.md b/content/pull-requests/collaborating-with-pull-requests/proposing-changes-to-your-work-with-pull-requests/requesting-a-pull-request-review.md index 91a9d3e146..d791c95923 100644 --- a/content/pull-requests/collaborating-with-pull-requests/proposing-changes-to-your-work-with-pull-requests/requesting-a-pull-request-review.md +++ b/content/pull-requests/collaborating-with-pull-requests/proposing-changes-to-your-work-with-pull-requests/requesting-a-pull-request-review.md @@ -17,7 +17,7 @@ shortTitle: Request a PR review --- Owners and collaborators on a repository owned by a user account can assign pull request reviews. Organization members with triage permissions to a repository can assign a pull request review. -Owners or collaborators can assign a pull request review to any person that has been explicitly granted [read access](/articles/access-permissions-on-github) to a user-owned repository. Organization members can assign a pull request review to any person or team with read access to a repository. The requested reviewer or team will receive a notification that you asked them to review the pull request. {% ifversion fpt or ghae or ghes or ghec %}If you request a review from a team and code review assignment is enabled, specific members will be requested and the team will be removed as a reviewer. For more information, see "[Managing code review assignment for your team](/organizations/organizing-members-into-teams/managing-code-review-assignment-for-your-team)."{% endif %} +Owners or collaborators can assign a pull request review to any person that has been explicitly granted [read access](/articles/access-permissions-on-github) to a user-owned repository. Organization members can assign a pull request review to any person or team with read access to a repository. The requested reviewer or team will receive a notification that you asked them to review the pull request. {% ifversion fpt or ghae or ghes or ghec %}If you request a review from a team and code review assignment is enabled, specific members will be requested and the team will be removed as a reviewer. For more information, see "[Managing code review settings for your team](/organizations/organizing-members-into-teams/managing-code-review-settings-for-your-team)."{% endif %} {% note %} diff --git a/content/pull-requests/collaborating-with-pull-requests/reviewing-changes-in-pull-requests/about-pull-request-reviews.md b/content/pull-requests/collaborating-with-pull-requests/reviewing-changes-in-pull-requests/about-pull-request-reviews.md index ad3e4d5f2f..41d86814c4 100644 --- a/content/pull-requests/collaborating-with-pull-requests/reviewing-changes-in-pull-requests/about-pull-request-reviews.md +++ b/content/pull-requests/collaborating-with-pull-requests/reviewing-changes-in-pull-requests/about-pull-request-reviews.md @@ -19,7 +19,7 @@ shortTitle: About PR reviews After a pull request is opened, anyone with *read* access can review and comment on the changes it proposes. You can also suggest specific changes to lines of code, which the author can apply directly from the pull request. For more information, see "[Reviewing proposed changes in a pull request](/pull-requests/collaborating-with-pull-requests/reviewing-changes-in-pull-requests/reviewing-proposed-changes-in-a-pull-request)." -Repository owners and collaborators can request a pull request review from a specific person. Organization members can also request a pull request review from a team with read access to the repository. For more information, see "[Requesting a pull request review](/pull-requests/collaborating-with-pull-requests/proposing-changes-to-your-work-with-pull-requests/requesting-a-pull-request-review)." {% ifversion fpt or ghae or ghes or ghec %}You can specify a subset of team members to be automatically assigned in the place of the whole team. For more information, see "[Managing code review assignment for your team](/organizations/organizing-members-into-teams/managing-code-review-assignment-for-your-team)."{% endif %} +Repository owners and collaborators can request a pull request review from a specific person. Organization members can also request a pull request review from a team with read access to the repository. For more information, see "[Requesting a pull request review](/pull-requests/collaborating-with-pull-requests/proposing-changes-to-your-work-with-pull-requests/requesting-a-pull-request-review)." {% ifversion fpt or ghae or ghes or ghec %}You can specify a subset of team members to be automatically assigned in the place of the whole team. For more information, see "[Managing code review settings for your team](/organizations/organizing-members-into-teams/managing-code-review-settings-for-your-team)."{% endif %} Reviews allow for discussion of proposed changes and help ensure that the changes meet the repository's contributing guidelines and other quality standards. You can define which individuals or teams own certain types or areas of code in a CODEOWNERS file. When a pull request modifies code that has a defined owner, that individual or team will automatically be requested as a reviewer. For more information, see "[About code owners](/articles/about-code-owners/)." diff --git a/content/repositories/managing-your-repositorys-settings-and-features/customizing-your-repository/about-code-owners.md b/content/repositories/managing-your-repositorys-settings-and-features/customizing-your-repository/about-code-owners.md index 49bd83ee09..1cc57f9028 100644 --- a/content/repositories/managing-your-repositorys-settings-and-features/customizing-your-repository/about-code-owners.md +++ b/content/repositories/managing-your-repositorys-settings-and-features/customizing-your-repository/about-code-owners.md @@ -25,7 +25,7 @@ Code owners are automatically requested for review when someone opens a pull req When someone with admin or owner permissions has enabled required reviews, they also can optionally require approval from a code owner before the author can merge a pull request in the repository. For more information, see "[About protected branches](/github/administering-a-repository/about-protected-branches#require-pull-request-reviews-before-merging)." -{% ifversion fpt or ghae or ghes or ghec %}If a team has enabled code review assignments, the individual approvals won't satisfy the requirement for code owner approval in a protected branch. For more information, see "[Managing code review assignment for your team](/organizations/organizing-members-into-teams/managing-code-review-assignment-for-your-team)."{% endif %} +{% ifversion fpt or ghae or ghes or ghec %}If a team has enabled code review assignments, the individual approvals won't satisfy the requirement for code owner approval in a protected branch. For more information, see "[Managing code review settings for your team](/organizations/organizing-members-into-teams/managing-code-review-settings-for-your-team)."{% endif %} If a file has a code owner, you can see who the code owner is before you open a pull request. In the repository, you can browse to the file and hover over {% octicon "shield-lock" aria-label="The edit icon" %}. diff --git a/data/features/only-notify-requested-members.yml b/data/features/only-notify-requested-members.yml new file mode 100644 index 0000000000..c60dc7e6b8 --- /dev/null +++ b/data/features/only-notify-requested-members.yml @@ -0,0 +1,7 @@ +# Issue #5108 +# Documentation for the "Only notify requested team members" option in the code review settings +versions: + fpt: '*' + ghec: '*' + ghes: '>=3.4' + ghae: '*' diff --git a/data/reusables/gated-features/code-review-assignment.md b/data/reusables/gated-features/code-review-assignment.md index 54941c9a54..7fbd17b9a2 100644 --- a/data/reusables/gated-features/code-review-assignment.md +++ b/data/reusables/gated-features/code-review-assignment.md @@ -1 +1 @@ -Code review assignment is available with {% data variables.product.prodname_team %}, {% data variables.product.prodname_ghe_server %} 2.20+,{% ifversion ghae %} {% data variables.product.prodname_ghe_managed %},{% endif %} and {% data variables.product.prodname_ghe_cloud %}. For more information, see "[GitHub's products](/articles/githubs-products)." +Code review settings are available with {% data variables.product.prodname_team %}, {% data variables.product.prodname_ghe_server %} 2.20+,{% ifversion ghae %} {% data variables.product.prodname_ghe_managed %},{% endif %} and {% data variables.product.prodname_ghe_cloud %}. For more information, see "[GitHub's products](/articles/githubs-products)." diff --git a/data/reusables/organizations/team_maintainers_can.md b/data/reusables/organizations/team_maintainers_can.md index 5bd403a0ba..7399688abc 100644 --- a/data/reusables/organizations/team_maintainers_can.md +++ b/data/reusables/organizations/team_maintainers_can.md @@ -11,5 +11,5 @@ Members with team maintainer permissions can: - [Remove organization members from the team](/articles/removing-organization-members-from-a-team) - [Promote an existing team member to team maintainer](/organizations/organizing-members-into-teams/assigning-the-team-maintainer-role-to-a-team-member) - Remove the team's access to repositories{% ifversion fpt or ghes or ghae or ghec %} -- [Manage code review assignment for the team](/organizations/organizing-members-into-teams/managing-code-review-assignment-for-your-team){% endif %}{% ifversion fpt or ghec %} +- [Manage code review settings for the team](/organizations/organizing-members-into-teams/managing-code-review-settings-for-your-team){% endif %}{% ifversion fpt or ghec %} - [Manage scheduled reminders for pull requests](/github/setting-up-and-managing-organizations-and-teams/managing-scheduled-reminders-for-pull-requests){% endif %} diff --git a/data/ui.yml b/data/ui.yml index e5723c4bc2..de1f6c6a99 100644 --- a/data/ui.yml +++ b/data/ui.yml @@ -31,6 +31,7 @@ search: placeholder: Search topics, products... loading: Loading no_results: No results found + no_content: No content homepage: explore_by_product: Explore by product version_picker: Version diff --git a/lib/search/config.js b/lib/search/config.js index da32abf1d9..ce43059f8c 100644 --- a/lib/search/config.js +++ b/lib/search/config.js @@ -1,10 +1,9 @@ -export const maxRecordLength = 8000 -export const maxContentLength = 5000 export const namePrefix = 'github-docs' +export const maxContentLength = 5000 export default { - // records must be truncated to avoid going over 10K limit - maxRecordLength, + // to reduce the size of our enormous search index, we limit the + // content and record size for translated articles maxContentLength, namePrefix, } diff --git a/lib/search/indexes/github-docs-3.0-en-records.json.br b/lib/search/indexes/github-docs-3.0-en-records.json.br index 9a6b46079f..726f5833f3 100644 --- a/lib/search/indexes/github-docs-3.0-en-records.json.br +++ b/lib/search/indexes/github-docs-3.0-en-records.json.br @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:3997f6be9ef84754b70f8921d292d2220d76c5a4b073529eaaab7c08ddfee3a7 -size 471841 +oid sha256:2aafd75d5dae7cec3ae63c284f5977c36029d81f9f3375cab4cacfeab202df7b +size 942825 diff --git a/lib/search/indexes/github-docs-3.0-en.json.br b/lib/search/indexes/github-docs-3.0-en.json.br index 042f8610f9..97311b8154 100644 --- a/lib/search/indexes/github-docs-3.0-en.json.br +++ b/lib/search/indexes/github-docs-3.0-en.json.br @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:53004022f4105b939a96862e97311e791350a73894491cfa4407eac67ea10ac1 -size 1845876 +oid sha256:7259b02adc5d0a6d3703f92b8d8fd168b104711b16c7b044c4c107da56e7d234 +size 3850716 diff --git a/lib/search/indexes/github-docs-3.1-en-records.json.br b/lib/search/indexes/github-docs-3.1-en-records.json.br index 9f9aadc759..62ed7b0075 100644 --- a/lib/search/indexes/github-docs-3.1-en-records.json.br +++ b/lib/search/indexes/github-docs-3.1-en-records.json.br @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:3e69db2a34f928de36ad82a9743d2ee743c566f53cd5e1045818fd36360a1e94 -size 481950 +oid sha256:7e984150e0931c807499d0fc8684d2db172af244e1d9b3fc979758f61dc80a55 +size 967385 diff --git a/lib/search/indexes/github-docs-3.1-en.json.br b/lib/search/indexes/github-docs-3.1-en.json.br index 2d982d5eb8..b38c922411 100644 --- a/lib/search/indexes/github-docs-3.1-en.json.br +++ b/lib/search/indexes/github-docs-3.1-en.json.br @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:e98eca40aa6b9eb1730c916776c2f3a9ac28b85a2c3ca8aa0c665ce07ea90bcd -size 1890939 +oid sha256:170a921904f8e81c25d6baec900636e2b1e2e50bf5d65f16817ff49cb633e14e +size 3941037 diff --git a/lib/search/indexes/github-docs-3.2-en-records.json.br b/lib/search/indexes/github-docs-3.2-en-records.json.br index 1ab4991aba..58cf9bd7c8 100644 --- a/lib/search/indexes/github-docs-3.2-en-records.json.br +++ b/lib/search/indexes/github-docs-3.2-en-records.json.br @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:b597fc467e9976b8f07aa2a9d3fe6e78e2f883c9e2d559235a676e7a4329c4d2 -size 489703 +oid sha256:54149a8b44478ae7e25a719a4fc6e622af4cf24f7c042871dada0ae86bfd5640 +size 998126 diff --git a/lib/search/indexes/github-docs-3.2-en.json.br b/lib/search/indexes/github-docs-3.2-en.json.br index 90d650c044..699a7645fa 100644 --- a/lib/search/indexes/github-docs-3.2-en.json.br +++ b/lib/search/indexes/github-docs-3.2-en.json.br @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:8b0a6d7329dffabe781630709aed7f77e5f425d738adde86b32cb6de720f6930 -size 1922283 +oid sha256:3da720ef2982c41ea0531ec97b48630fabf7f20d188af94a26b00c7203195554 +size 4061756 diff --git a/lib/search/indexes/github-docs-3.3-en-records.json.br b/lib/search/indexes/github-docs-3.3-en-records.json.br index 26c5bbd5da..3d1b23b173 100644 --- a/lib/search/indexes/github-docs-3.3-en-records.json.br +++ b/lib/search/indexes/github-docs-3.3-en-records.json.br @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:2b63ab97b5108aaa47d289f7a1a1e0f64cda61ae54d343864748eccee727064e -size 505826 +oid sha256:6319078d6a57f4e22edd04aaf24c7d274dbf1e313e12750aabf4b864083a1c4e +size 1030307 diff --git a/lib/search/indexes/github-docs-3.3-en.json.br b/lib/search/indexes/github-docs-3.3-en.json.br index 839deefe97..748548f863 100644 --- a/lib/search/indexes/github-docs-3.3-en.json.br +++ b/lib/search/indexes/github-docs-3.3-en.json.br @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:a80e1d538dc5470ca284e68a68f91885e7c1f60936005c9ef33006f036a94944 -size 1985776 +oid sha256:453ad24d930c87b2b9cd78193f4ccb900f269e95c3f785ef1a0b7895d15be71e +size 4156442 diff --git a/lib/search/indexes/github-docs-dotcom-en-records.json.br b/lib/search/indexes/github-docs-dotcom-en-records.json.br index 461c3a001b..f527e49e78 100644 --- a/lib/search/indexes/github-docs-dotcom-en-records.json.br +++ b/lib/search/indexes/github-docs-dotcom-en-records.json.br @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:d4344ec9cc27cb32972e88b2bc1b8aaedf29c9f0219283d9f5ee8a05f3007734 -size 654101 +oid sha256:2f91a89d03b025f6856824e14f740fa6699f798bfc66974836b1d0875346e701 +size 1317291 diff --git a/lib/search/indexes/github-docs-dotcom-en.json.br b/lib/search/indexes/github-docs-dotcom-en.json.br index e3c5f56bd4..a7adbf20c9 100644 --- a/lib/search/indexes/github-docs-dotcom-en.json.br +++ b/lib/search/indexes/github-docs-dotcom-en.json.br @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:63285afaf385e920fecb359c0445f4bbbdeb85f85e0a5280cd183745820eabed -size 2460133 +oid sha256:3c3a14dc48e59dc5b312e71f6fa17747fc66526e568c270ec585365854c1d49f +size 5051232 diff --git a/lib/search/indexes/github-docs-ghae-en-records.json.br b/lib/search/indexes/github-docs-ghae-en-records.json.br index a3a2478dba..15fab70b8b 100644 --- a/lib/search/indexes/github-docs-ghae-en-records.json.br +++ b/lib/search/indexes/github-docs-ghae-en-records.json.br @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:177fe1c1e65bceeb00e849a5b3918965959bf7208dea6babfe6add07d04e732b -size 378669 +oid sha256:af643733dea26ff845274b23d560e09d7c7e71e3ec4d37f5453ac40c58e6a879 +size 795513 diff --git a/lib/search/indexes/github-docs-ghae-en.json.br b/lib/search/indexes/github-docs-ghae-en.json.br index 407ee56211..9448652b63 100644 --- a/lib/search/indexes/github-docs-ghae-en.json.br +++ b/lib/search/indexes/github-docs-ghae-en.json.br @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:04c0a814e72ce3183e71ae1bc6eb3226706802a65cd2e28604856cd92fb1b552 -size 1419518 +oid sha256:21800b71002cbeb123bf22d0235975f6146967ffece9f01b9a5e3329d1a9c75e +size 3194909 diff --git a/lib/search/indexes/github-docs-ghec-en-records.json.br b/lib/search/indexes/github-docs-ghec-en-records.json.br index 16b318b6f9..ecb5da0035 100644 --- a/lib/search/indexes/github-docs-ghec-en-records.json.br +++ b/lib/search/indexes/github-docs-ghec-en-records.json.br @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:1d6ba67865923e02b5e6bf103085b25ba3b53f6a6378fa4f07d2effe49131791 -size 593367 +oid sha256:e98188cef351a0702d6e22b645581194ed0ada6450a733443375b2c6d24f7d8c +size 1162569 diff --git a/lib/search/indexes/github-docs-ghec-en.json.br b/lib/search/indexes/github-docs-ghec-en.json.br index a69b12aa4b..fe95afed70 100644 --- a/lib/search/indexes/github-docs-ghec-en.json.br +++ b/lib/search/indexes/github-docs-ghec-en.json.br @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:f2351d9f2222dafbf6479c46a818b560be34c7e2bb785ac94da91fd88b8f6a82 -size 2339325 +oid sha256:b7441c7b87e8ac058a2173bc6e0c3874f29626759a84b247694c2a7440460d23 +size 4704034 diff --git a/lib/search/lunr-search.js b/lib/search/lunr-search.js index 0746e1beae..f533e91860 100644 --- a/lib/search/lunr-search.js +++ b/lib/search/lunr-search.js @@ -11,6 +11,13 @@ import readFileAsync from '../readfile-async.js' import { namePrefix } from './config.js' import { decompress } from './compress.js' const __dirname = path.dirname(fileURLToPath(import.meta.url)) + +// By default Lunr considers the `-` character to be a word boundary. +// This allows hypens to be included in the query. +// If you change this, remember to make it match the indexing separator +// in script/search/lunr-search-index.js so the query is tokenized +// identically to the way it was indexed. +lunr.QueryLexer.termSeparator = /[\s]+/ lunrStemmerSupport(lunr) tinyseg(lunr) lunrJa(lunr) @@ -21,29 +28,194 @@ const LUNR_DIR = './indexes' const lunrIndexes = new Map() const lunrRecords = new Map() +// Max size of the `.content` record included in the JSON payload that the +// middleware server will serve. +// The reason we're worrying about that here and not in the middleware +// is because what we're *ultimately* sending is HTML so we can't let +// the consumer of this module, slice it as a regular string because +// they might cut off an HTML tag in the middle. +// As of Oct 2021, with the way the CSS works inside components/Search.tsx +// roughly 450-650 characters is contained. Let's just make sure we're +// well within limit. So no visual difference, but smaller JSON payloads. +const MAX_CONTENT_LENGTH = 1000 + export default async function loadLunrResults({ version, language, query, limit }) { const indexName = `${namePrefix}-${version}-${language}` if (!lunrIndexes.has(indexName) || !lunrRecords.has(indexName)) { lunrIndexes.set(indexName, await loadLunrIndex(indexName)) lunrRecords.set(indexName, await loadLunrRecords(indexName)) } - const results = lunrIndexes - .get(indexName) - .search(query) - .slice(0, limit) - .map((result) => { - const record = lunrRecords.get(indexName)[result.ref] - return { - url: result.ref, - breadcrumbs: field(result, record, 'breadcrumbs'), - heading: field(result, record, 'heading'), - title: field(result, record, 'title'), - content: field(result, record, 'content'), - // don't highlight the topics array - topics: record.topics, + const index = lunrIndexes.get(indexName) + const records = lunrRecords.get(indexName) + const queryLength = query.trim().length + + // A search results /combined/ score is: + // + // normalizedScore + POPULARITY_FACTOR * record.popularity + // + // where the "normalizedScore" is the ratio of its Lunr score divided + // by the highest score of all found in Lunr. That means, that the record + // Lunr thinks matches the most becomes 1.0. + // + // It's the number we sort on. The `record.popularity` is always a + // number between (and including) 0-1. + // If the Lunr score is, say, 5.0 and the popularity is 0.1, and + // the POPULARITY_FACTOR is 10, the combined score is 5.0 + 10 * 0.1 = 6.0 + // If you make this too large, the Lunr score becomes insignificant and + // any single match anywhere will always favor the popular documents. + // The best way to adjust this number is to get a feeling for what + // kinds of Lunr score numbers we're usually getting and adjust + // accordingly. + // Short queries are bound to be very ambigous and the more ambiguous + // the more relevant the popularity is. + const POPULARITY_FACTOR = queryLength <= 2 ? 25 : queryLength <= 6 ? 10 : 5 + + // This number determines how much more we favor the title search first. + // It's a multiplier. We do 2 searches: one on title, one on all other fields. + // Then, we compare all scores. But the scores in the results from the title + // we multiply that with this number. + // The effect is that we favor matches in the title more than we favor + // matches that were not in the title. + // If you search for 'foobar' and it appears in the title of one + // not-so-popular record, but also appears in the content of a + // very popular record, you want to give the title-matching one a + // leg up. + // Note that the Lunr scores from the content is usually much higher + // than scores on the title. E.g. the word `codespaces` might appear + // 10 times on a page that is actually about something else. If there's + // a record whose title includes `codespaces` it might get a very low + // Lunr score but since title matches are generally a "better", we + // want to make sure this number accounts for that. + const TITLE_FIRST = queryLength <= 2 ? 45 : queryLength <= 6 ? 25 : 10 + + // Imagine that we have 1,000 documents. 100 of them contain the word + // 'foobar'. Of those 100, we want to display the top 10 "best". + // But if we only do `lunrindex.search('foobar').slice(0, 10)` we + // would slice prematurely. Instead, we do + // `lunrindex.search('foobar').slice(0, 100)` first, sort those, + // and in the final step, after any custom sorting, we `.slice(0, 10)`. + // This number decides how many to extract from Lunr in the first place + // that we're going to do our custom sorting on. + // This number can be allowed to be pretty big because we're only ever + // going to do the more time-consuming highlighting on the `limit` + // records that we finally return. + const PRE_LIMIT = 500 + + let titleQuery = query.trim() + if (titleQuery.length <= 3 && !titleQuery.endsWith('*s')) { + // When the search input is really short, force it to search with + // the "forward wild card". I.e. you typed `go` we turn it into a + // search for `go*` which means it can find things like `Google`. + titleQuery += '*' + } + + let highestTitleScore = 0.0 + const titleResults = index + .query((q) => { + if (/['"]/.test(titleQuery)) { + // If the query contains a quotation marks, you can't easily + // enough break it up into individual words. + q.term(titleQuery, { fields: ['title'] }) + } else { + // This is the structured way of doing turning 'foo bar' + // into `title:foo title:bar'. + titleQuery.split(/ /g).forEach((part) => { + q.term(part, { fields: ['title'] }) + }) } }) - return results + .slice(0, PRE_LIMIT) + .map((result) => { + const { popularity } = records[result.ref] + if (result.score > highestTitleScore) { + highestTitleScore = result.score + } + const score = result.score / highestTitleScore + return { + result, + _score: TITLE_FIRST * (score + POPULARITY_FACTOR * (popularity || 0.0)), + } + }) + + let allQuery = query.trim() + + // Unfortunately, Lunr currently doesn't support phrase matching + // so you always end up with 0 results if you search for `"foo bar"`. + // In this case it's better to do a search for `foo` and `bar`. + if ( + allQuery.startsWith('"') && + allQuery.endsWith('"') && + (allQuery.match(/"/g) || []).length === 2 + ) { + allQuery = allQuery.slice(1, -1) + } + + let highestAllScore = 0.0 + const allResults = index + .search(allQuery) + .slice(0, PRE_LIMIT) + .map((result) => { + const { popularity } = records[result.ref] + if (result.score > highestAllScore) { + highestAllScore = result.score + } + const score = result.score / highestAllScore + return { + result, + score, + _score: score + POPULARITY_FACTOR * (popularity || 0.0), + } + }) + + const _unique = new Set() + const combinedMatchData = {} + const results = [] + for (const matches of [titleResults, allResults]) { + for (const match of matches) { + const { result } = match + // We need to loop over all results (both from title searches and + // from all-field searches) but we can only keep one. + // But before we do that filtering (i.e. omitting previous kept) + // we need to merge all the matchData from each result. + // That's because the `result.matchData` from the title search + // will have Lunr match positions for 'title' but the `result.matchData` + // from the all-field search, will have positions for other things + // such as 'content' and 'breadcrumbs'. + + combinedMatchData[result.ref] = Object.assign( + combinedMatchData[result.ref] || {}, + result.matchData + ) + + if (_unique.has(result.ref)) continue + _unique.add(result.ref) + + results.push(match) + } + } + + // Highest score first + results.sort((a, b) => b._score - a._score) + + // We might have found much more than `limit` number of matches and we've + // taken them all out for our custom sorting. Now, once that's done, + // of the ones we're going to return we apply the highlighting. + // The reasonsing is that the highlighting work isn't free and it'd + // be a waste to do it on results we're not going to return anyway. + return results.slice(0, limit).map(({ result }) => { + const record = records[result.ref] + const matchData = combinedMatchData[result.ref] + return { + url: result.ref, + breadcrumbs: field(matchData, record, 'breadcrumbs'), + title: field(matchData, record, 'title'), + content: smartSlice(field(matchData, record, 'content'), MAX_CONTENT_LENGTH), + // don't highlight the topics array + topics: record.topics, + score: result.score, + popularity: record.popularity || 0.0, + } + }) } async function loadLunrIndex(indexName) { @@ -59,12 +231,12 @@ async function loadLunrRecords(indexName) { } // Highlight a match within an attribute field -function field(result, record, name) { +function field(matchData, record, name) { const text = record[name] if (!text) return text // First, get a list of all the positions of the matching tokens - const positions = Object.values(result.matchData.metadata) + const positions = Object.values(matchData.metadata) .map((fields) => get(fields, [name, 'position'])) .filter(Boolean) .flat() @@ -90,3 +262,34 @@ function field(result, record, name) { function mark(text) { return `${text}` } + +// Give a long string, "slice" it in a safe way so as to not chop any +// HTML tags in half. +// The resulting string will only be at *least* as long as the `length` +// provided. Possibly longer. +function smartSlice(text, length, needleTag = '') { + // If the needleTag isn't present at all, we can dare to use a + // very basic crude string slice because the text won't have any + // other HTML tags we might cut in half. + if (!text.includes(needleTag)) { + return text.slice(0, length) + } + + // The algorithm is simple, split the text by lines. Loop over them, + // and only include them if we've encountered the first needleTag + // and bail early if we've buffered enough in the array of lines. + const lines = [] + let sum = 0 + let started = false + for (const line of text.split('\n')) { + if (line.indexOf(needleTag) > -1) started = true + if (started) { + lines.push(line) + sum += line.length + if (sum > length) { + break + } + } + } + return lines.join('\n') +} diff --git a/package-lock.json b/package-lock.json index 4365b5c8b7..e0564520be 100644 --- a/package-lock.json +++ b/package-lock.json @@ -145,7 +145,6 @@ "http-status-code": "^2.1.0", "husky": "^7.0.4", "image-size": "^1.0.0", - "is-url": "^1.2.4", "japanese-characters": "^1.1.0", "javascript-stringify": "^2.1.0", "jest": "^27.3.1", @@ -3233,66 +3232,6 @@ } } }, - "node_modules/@next/swc-darwin-arm64": { - "version": "11.1.2", - "resolved": "https://registry.npmjs.org/@next/swc-darwin-arm64/-/swc-darwin-arm64-11.1.2.tgz", - "integrity": "sha512-hZuwOlGOwBZADA8EyDYyjx3+4JGIGjSHDHWrmpI7g5rFmQNltjlbaefAbiU5Kk7j3BUSDwt30quJRFv3nyJQ0w==", - "cpu": [ - "arm64" - ], - "optional": true, - "os": [ - "darwin" - ], - "engines": { - "node": ">= 10" - } - }, - "node_modules/@next/swc-darwin-x64": { - "version": "11.1.2", - "resolved": "https://registry.npmjs.org/@next/swc-darwin-x64/-/swc-darwin-x64-11.1.2.tgz", - "integrity": "sha512-PGOp0E1GisU+EJJlsmJVGE+aPYD0Uh7zqgsrpD3F/Y3766Ptfbe1lEPPWnRDl+OzSSrSrX1lkyM/Jlmh5OwNvA==", - "cpu": [ - "x64" - ], - "optional": true, - "os": [ - "darwin" - ], - "engines": { - "node": ">= 10" - } - }, - "node_modules/@next/swc-linux-x64-gnu": { - "version": "11.1.2", - "resolved": "https://registry.npmjs.org/@next/swc-linux-x64-gnu/-/swc-linux-x64-gnu-11.1.2.tgz", - "integrity": "sha512-YcDHTJjn/8RqvyJVB6pvEKXihDcdrOwga3GfMv/QtVeLphTouY4BIcEUfrG5+26Nf37MP1ywN3RRl1TxpurAsQ==", - "cpu": [ - "x64" - ], - "optional": true, - "os": [ - "linux" - ], - "engines": { - "node": ">= 10" - } - }, - "node_modules/@next/swc-win32-x64-msvc": { - "version": "11.1.2", - "resolved": "https://registry.npmjs.org/@next/swc-win32-x64-msvc/-/swc-win32-x64-msvc-11.1.2.tgz", - "integrity": "sha512-e/pIKVdB+tGQYa1cW3sAeHm8gzEri/HYLZHT4WZojrUxgWXqx8pk7S7Xs47uBcFTqBDRvK3EcQpPLf3XdVsDdg==", - "cpu": [ - "x64" - ], - "optional": true, - "os": [ - "win32" - ], - "engines": { - "node": ">= 10" - } - }, "node_modules/@node-rs/helper": { "version": "1.2.1", "resolved": "https://registry.npmjs.org/@node-rs/helper/-/helper-1.2.1.tgz", @@ -6029,15 +5968,6 @@ "node": ">=8" } }, - "node_modules/bindings": { - "version": "1.5.0", - "resolved": "https://registry.npmjs.org/bindings/-/bindings-1.5.0.tgz", - "integrity": "sha512-p2q/t/mhvuOj/UeLlV6566GD/guowlr0hHxClI0W9m7MWYkL1F0hLo+0Aexs9HSPCtR1SXQ0TD3MMKrXZajbiQ==", - "optional": true, - "dependencies": { - "file-uri-to-path": "1.0.0" - } - }, "node_modules/bl": { "version": "4.1.0", "resolved": "https://registry.npmjs.org/bl/-/bl-4.1.0.tgz", @@ -8496,16 +8426,6 @@ "node": ">= 0.8.0" } }, - "node_modules/escodegen/node_modules/source-map": { - "version": "0.6.1", - "resolved": "https://registry.npmjs.org/source-map/-/source-map-0.6.1.tgz", - "integrity": "sha512-UjgapumWlbMhkBgzT7Ykc5YXUT46F0iKu8SGXq0bcwP5dz/h0Plj6enJqjz1Zbq2l5WaqYnrVbwWOWMyF3F47g==", - "dev": true, - "optional": true, - "engines": { - "node": ">=0.10.0" - } - }, "node_modules/escodegen/node_modules/type-check": { "version": "0.3.2", "resolved": "https://registry.npmjs.org/type-check/-/type-check-0.3.2.tgz", @@ -9463,12 +9383,6 @@ "node": ">=6" } }, - "node_modules/file-uri-to-path": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/file-uri-to-path/-/file-uri-to-path-1.0.0.tgz", - "integrity": "sha512-0Zt+s3L7Vf1biwWZ29aARiVYLx7iMGnEUl9x33fbB/j3jR81u/O2LbqK+Bm1CDSNDKVtJ/YjwY7TUd5SkeLQLw==", - "optional": true - }, "node_modules/file-url": { "version": "3.0.0", "resolved": "https://registry.npmjs.org/file-url/-/file-url-3.0.0.tgz", @@ -9635,9 +9549,9 @@ } }, "node_modules/find-process": { - "version": "1.4.5", - "resolved": "https://registry.npmjs.org/find-process/-/find-process-1.4.5.tgz", - "integrity": "sha512-v11rJYYISUWn+s8qZzgGnBvlzRKf3bOtlGFM8H0kw56lGQtOmLuLCzuclA5kehA2j7S5sioOWdI4woT3jDavAw==", + "version": "1.4.4", + "resolved": "https://registry.npmjs.org/find-process/-/find-process-1.4.4.tgz", + "integrity": "sha512-rRSuT1LE4b+BFK588D2V8/VG9liW0Ark1XJgroxZXI0LtwmQJOb490DvDYvbm+Hek9ETFzTutGfJ90gumITPhQ==", "optional": true, "dependencies": { "chalk": "^4.0.0", @@ -9844,19 +9758,6 @@ "integrity": "sha1-FQStJSMVjKpA20onh8sBQRmU6k8=", "devOptional": true }, - "node_modules/fsevents": { - "version": "2.3.2", - "resolved": "https://registry.npmjs.org/fsevents/-/fsevents-2.3.2.tgz", - "integrity": "sha512-xiqMQR4xAeHTuB9uWm+fFRcIOgKBMiOBP+eXiyT7jsgVCq1bkVygt00oASowB7EdtpOHaaPgKt812P9ab+DDKA==", - "hasInstallScript": true, - "optional": true, - "os": [ - "darwin" - ], - "engines": { - "node": "^8.16.0 || ^10.6.0 || >=11.0.0" - } - }, "node_modules/function-bind": { "version": "1.1.1", "resolved": "https://registry.npmjs.org/function-bind/-/function-bind-1.1.1.tgz", @@ -14906,12 +14807,6 @@ "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.2.tgz", "integrity": "sha512-sGkPx+VjMtmA6MX27oA4FBFELFCZZ4S4XqeGOXCv68tT+jb3vk/RyaKWP0PTKyWtmLSM0b+adUTEvbs1PEaH2w==" }, - "node_modules/nan": { - "version": "2.15.0", - "resolved": "https://registry.npmjs.org/nan/-/nan-2.15.0.tgz", - "integrity": "sha512-8ZtvEnA2c5aYCZYd1cvgdnU6cqwixRoYg70xPLWUws5ORTa/lnw+u4amixRS/Ac5U5mQVgp9pnlSUnbNWFaWZQ==", - "optional": true - }, "node_modules/nanoid": { "version": "3.1.30", "resolved": "https://registry.npmjs.org/nanoid/-/nanoid-3.1.30.tgz", @@ -16267,9 +16162,9 @@ } }, "node_modules/pa11y-ci/node_modules/node-fetch": { - "version": "2.6.5", - "resolved": "https://registry.npmjs.org/node-fetch/-/node-fetch-2.6.5.tgz", - "integrity": "sha512-mmlIVHJEu5rnIxgEgez6b9GgWXbkZj5YZ7fx+2r94a2E+Uirsp6HsPTPlomfdHtpt/B0cdKviwkoaM6pyvUOpQ==", + "version": "2.6.6", + "resolved": "https://registry.npmjs.org/node-fetch/-/node-fetch-2.6.6.tgz", + "integrity": "sha512-Z8/6vRlTUChSdIgMa51jxQ4lrw/Jy5SOW10ObaA47/RElsAN2c5Pn8bTgFGWn/ibwzXTE8qwr1Yzx28vsecXEA==", "optional": true, "dependencies": { "whatwg-url": "^5.0.0" @@ -16893,9 +16788,9 @@ } }, "node_modules/parse-headers": { - "version": "2.0.4", - "resolved": "https://registry.npmjs.org/parse-headers/-/parse-headers-2.0.4.tgz", - "integrity": "sha512-psZ9iZoCNFLrgRjZ1d8mn0h9WRqJwFxM9q3x7iUjN/YT2OksthDJ5TiPCu2F38kS4zutqfW+YdVVkBZZx3/1aw==", + "version": "2.0.3", + "resolved": "https://registry.npmjs.org/parse-headers/-/parse-headers-2.0.3.tgz", + "integrity": "sha512-QhhZ+DCCit2Coi2vmAKbq5RGTRcQUOE2+REgv8vdyu7MnYx2eZztegqtTx99TZ86GTIwqiy3+4nQTWZ2tgmdCA==", "optional": true }, "node_modules/parse-json": { @@ -17507,9 +17402,9 @@ } }, "node_modules/puppeteer/node_modules/node-fetch": { - "version": "2.6.5", - "resolved": "https://registry.npmjs.org/node-fetch/-/node-fetch-2.6.5.tgz", - "integrity": "sha512-mmlIVHJEu5rnIxgEgez6b9GgWXbkZj5YZ7fx+2r94a2E+Uirsp6HsPTPlomfdHtpt/B0cdKviwkoaM6pyvUOpQ==", + "version": "2.6.6", + "resolved": "https://registry.npmjs.org/node-fetch/-/node-fetch-2.6.6.tgz", + "integrity": "sha512-Z8/6vRlTUChSdIgMa51jxQ4lrw/Jy5SOW10ObaA47/RElsAN2c5Pn8bTgFGWn/ibwzXTE8qwr1Yzx28vsecXEA==", "optional": true, "dependencies": { "whatwg-url": "^5.0.0" @@ -17600,7 +17495,6 @@ "version": "0.2.1", "resolved": "https://registry.npmjs.org/querystring/-/querystring-0.2.1.tgz", "integrity": "sha512-wkvS7mL/JMugcup3/rMitHmd9ecIGd2lhFhK9N3UUQ450h66d1r3Y9nvXzQAW1Lq+wyx61k/1pfKS5KuKiyEbg==", - "deprecated": "The querystring API is considered Legacy. new code should use the URLSearchParams API instead.", "engines": { "node": ">=0.4.x" } @@ -20095,15 +19989,6 @@ "node": ">= 0.8.0" } }, - "node_modules/static-eval/node_modules/source-map": { - "version": "0.6.1", - "resolved": "https://registry.npmjs.org/source-map/-/source-map-0.6.1.tgz", - "integrity": "sha512-UjgapumWlbMhkBgzT7Ykc5YXUT46F0iKu8SGXq0bcwP5dz/h0Plj6enJqjz1Zbq2l5WaqYnrVbwWOWMyF3F47g==", - "optional": true, - "engines": { - "node": ">=0.10.0" - } - }, "node_modules/static-eval/node_modules/type-check": { "version": "0.3.2", "resolved": "https://registry.npmjs.org/type-check/-/type-check-0.3.2.tgz", @@ -20229,15 +20114,6 @@ "util-deprecate": "~1.0.1" } }, - "node_modules/static-module/node_modules/source-map": { - "version": "0.6.1", - "resolved": "https://registry.npmjs.org/source-map/-/source-map-0.6.1.tgz", - "integrity": "sha512-UjgapumWlbMhkBgzT7Ykc5YXUT46F0iKu8SGXq0bcwP5dz/h0Plj6enJqjz1Zbq2l5WaqYnrVbwWOWMyF3F47g==", - "optional": true, - "engines": { - "node": ">=0.10.0" - } - }, "node_modules/static-module/node_modules/string_decoder": { "version": "1.1.1", "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-1.1.1.tgz", @@ -21804,20 +21680,6 @@ "node": ">= 4.0.0" } }, - "node_modules/unix-dgram": { - "version": "2.0.4", - "resolved": "https://registry.npmjs.org/unix-dgram/-/unix-dgram-2.0.4.tgz", - "integrity": "sha512-7tpK6x7ls7J7pDrrAU63h93R0dVhRbPwiRRCawR10cl+2e1VOvF3bHlVJc6WI1dl/8qk5He673QU+Ogv7bPNaw==", - "hasInstallScript": true, - "optional": true, - "dependencies": { - "bindings": "^1.3.0", - "nan": "^2.13.2" - }, - "engines": { - "node": ">=0.10.48" - } - }, "node_modules/unpipe": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/unpipe/-/unpipe-1.0.0.tgz", @@ -21931,7 +21793,6 @@ "version": "0.2.0", "resolved": "https://registry.npmjs.org/querystring/-/querystring-0.2.0.tgz", "integrity": "sha1-sgmEkgO7Jd+CDadW50cAWHhSFiA=", - "deprecated": "The querystring API is considered Legacy. new code should use the URLSearchParams API instead.", "engines": { "node": ">=0.4.x" } @@ -25233,30 +25094,6 @@ "integrity": "sha512-hsoJmPfhVqjZ8w4IFzoo8SyECVnN+8WMnImTbTKrRUHOVJcYMmKLL7xf7T0ft00tWwAl/3f3Q3poWIN2Ueql/Q==", "requires": {} }, - "@next/swc-darwin-arm64": { - "version": "11.1.2", - "resolved": "https://registry.npmjs.org/@next/swc-darwin-arm64/-/swc-darwin-arm64-11.1.2.tgz", - "integrity": "sha512-hZuwOlGOwBZADA8EyDYyjx3+4JGIGjSHDHWrmpI7g5rFmQNltjlbaefAbiU5Kk7j3BUSDwt30quJRFv3nyJQ0w==", - "optional": true - }, - "@next/swc-darwin-x64": { - "version": "11.1.2", - "resolved": "https://registry.npmjs.org/@next/swc-darwin-x64/-/swc-darwin-x64-11.1.2.tgz", - "integrity": "sha512-PGOp0E1GisU+EJJlsmJVGE+aPYD0Uh7zqgsrpD3F/Y3766Ptfbe1lEPPWnRDl+OzSSrSrX1lkyM/Jlmh5OwNvA==", - "optional": true - }, - "@next/swc-linux-x64-gnu": { - "version": "11.1.2", - "resolved": "https://registry.npmjs.org/@next/swc-linux-x64-gnu/-/swc-linux-x64-gnu-11.1.2.tgz", - "integrity": "sha512-YcDHTJjn/8RqvyJVB6pvEKXihDcdrOwga3GfMv/QtVeLphTouY4BIcEUfrG5+26Nf37MP1ywN3RRl1TxpurAsQ==", - "optional": true - }, - "@next/swc-win32-x64-msvc": { - "version": "11.1.2", - "resolved": "https://registry.npmjs.org/@next/swc-win32-x64-msvc/-/swc-win32-x64-msvc-11.1.2.tgz", - "integrity": "sha512-e/pIKVdB+tGQYa1cW3sAeHm8gzEri/HYLZHT4WZojrUxgWXqx8pk7S7Xs47uBcFTqBDRvK3EcQpPLf3XdVsDdg==", - "optional": true - }, "@node-rs/helper": { "version": "1.2.1", "resolved": "https://registry.npmjs.org/@node-rs/helper/-/helper-1.2.1.tgz", @@ -27649,15 +27486,6 @@ "resolved": "https://registry.npmjs.org/binary-extensions/-/binary-extensions-2.2.0.tgz", "integrity": "sha512-jDctJ/IVQbZoJykoeHbhXpOlNBqGNcwXJKJog42E5HDPUwQTSdjCHdihjj0DlnheQ7blbT6dHOafNAiS8ooQKA==" }, - "bindings": { - "version": "1.5.0", - "resolved": "https://registry.npmjs.org/bindings/-/bindings-1.5.0.tgz", - "integrity": "sha512-p2q/t/mhvuOj/UeLlV6566GD/guowlr0hHxClI0W9m7MWYkL1F0hLo+0Aexs9HSPCtR1SXQ0TD3MMKrXZajbiQ==", - "optional": true, - "requires": { - "file-uri-to-path": "1.0.0" - } - }, "bl": { "version": "4.1.0", "resolved": "https://registry.npmjs.org/bl/-/bl-4.1.0.tgz", @@ -29607,13 +29435,6 @@ "integrity": "sha1-IZMqVJ9eUv/ZqCf1cOBL5iqX2lQ=", "dev": true }, - "source-map": { - "version": "0.6.1", - "resolved": "https://registry.npmjs.org/source-map/-/source-map-0.6.1.tgz", - "integrity": "sha512-UjgapumWlbMhkBgzT7Ykc5YXUT46F0iKu8SGXq0bcwP5dz/h0Plj6enJqjz1Zbq2l5WaqYnrVbwWOWMyF3F47g==", - "dev": true, - "optional": true - }, "type-check": { "version": "0.3.2", "resolved": "https://registry.npmjs.org/type-check/-/type-check-0.3.2.tgz", @@ -30357,12 +30178,6 @@ "integrity": "sha512-Qe/5NJrgIOlwijpq3B7BEpzPFcgzggOTagZmkXQY4LA6bsXKTUstK7Wp12lEJ/mLKTpvIZxmIuRcLYWT6ov9lw==", "optional": true }, - "file-uri-to-path": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/file-uri-to-path/-/file-uri-to-path-1.0.0.tgz", - "integrity": "sha512-0Zt+s3L7Vf1biwWZ29aARiVYLx7iMGnEUl9x33fbB/j3jR81u/O2LbqK+Bm1CDSNDKVtJ/YjwY7TUd5SkeLQLw==", - "optional": true - }, "file-url": { "version": "3.0.0", "resolved": "https://registry.npmjs.org/file-url/-/file-url-3.0.0.tgz", @@ -30491,9 +30306,9 @@ } }, "find-process": { - "version": "1.4.5", - "resolved": "https://registry.npmjs.org/find-process/-/find-process-1.4.5.tgz", - "integrity": "sha512-v11rJYYISUWn+s8qZzgGnBvlzRKf3bOtlGFM8H0kw56lGQtOmLuLCzuclA5kehA2j7S5sioOWdI4woT3jDavAw==", + "version": "1.4.4", + "resolved": "https://registry.npmjs.org/find-process/-/find-process-1.4.4.tgz", + "integrity": "sha512-rRSuT1LE4b+BFK588D2V8/VG9liW0Ark1XJgroxZXI0LtwmQJOb490DvDYvbm+Hek9ETFzTutGfJ90gumITPhQ==", "optional": true, "requires": { "chalk": "^4.0.0", @@ -30643,12 +30458,6 @@ "integrity": "sha1-FQStJSMVjKpA20onh8sBQRmU6k8=", "devOptional": true }, - "fsevents": { - "version": "2.3.2", - "resolved": "https://registry.npmjs.org/fsevents/-/fsevents-2.3.2.tgz", - "integrity": "sha512-xiqMQR4xAeHTuB9uWm+fFRcIOgKBMiOBP+eXiyT7jsgVCq1bkVygt00oASowB7EdtpOHaaPgKt812P9ab+DDKA==", - "optional": true - }, "function-bind": { "version": "1.1.1", "resolved": "https://registry.npmjs.org/function-bind/-/function-bind-1.1.1.tgz", @@ -34417,12 +34226,6 @@ "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.2.tgz", "integrity": "sha512-sGkPx+VjMtmA6MX27oA4FBFELFCZZ4S4XqeGOXCv68tT+jb3vk/RyaKWP0PTKyWtmLSM0b+adUTEvbs1PEaH2w==" }, - "nan": { - "version": "2.15.0", - "resolved": "https://registry.npmjs.org/nan/-/nan-2.15.0.tgz", - "integrity": "sha512-8ZtvEnA2c5aYCZYd1cvgdnU6cqwixRoYg70xPLWUws5ORTa/lnw+u4amixRS/Ac5U5mQVgp9pnlSUnbNWFaWZQ==", - "optional": true - }, "nanoid": { "version": "3.1.30", "resolved": "https://registry.npmjs.org/nanoid/-/nanoid-3.1.30.tgz", @@ -35577,9 +35380,9 @@ } }, "node-fetch": { - "version": "2.6.5", - "resolved": "https://registry.npmjs.org/node-fetch/-/node-fetch-2.6.5.tgz", - "integrity": "sha512-mmlIVHJEu5rnIxgEgez6b9GgWXbkZj5YZ7fx+2r94a2E+Uirsp6HsPTPlomfdHtpt/B0cdKviwkoaM6pyvUOpQ==", + "version": "2.6.6", + "resolved": "https://registry.npmjs.org/node-fetch/-/node-fetch-2.6.6.tgz", + "integrity": "sha512-Z8/6vRlTUChSdIgMa51jxQ4lrw/Jy5SOW10ObaA47/RElsAN2c5Pn8bTgFGWn/ibwzXTE8qwr1Yzx28vsecXEA==", "optional": true, "requires": { "whatwg-url": "^5.0.0" @@ -35988,9 +35791,9 @@ } }, "parse-headers": { - "version": "2.0.4", - "resolved": "https://registry.npmjs.org/parse-headers/-/parse-headers-2.0.4.tgz", - "integrity": "sha512-psZ9iZoCNFLrgRjZ1d8mn0h9WRqJwFxM9q3x7iUjN/YT2OksthDJ5TiPCu2F38kS4zutqfW+YdVVkBZZx3/1aw==", + "version": "2.0.3", + "resolved": "https://registry.npmjs.org/parse-headers/-/parse-headers-2.0.3.tgz", + "integrity": "sha512-QhhZ+DCCit2Coi2vmAKbq5RGTRcQUOE2+REgv8vdyu7MnYx2eZztegqtTx99TZ86GTIwqiy3+4nQTWZ2tgmdCA==", "optional": true }, "parse-json": { @@ -36473,9 +36276,9 @@ } }, "node-fetch": { - "version": "2.6.5", - "resolved": "https://registry.npmjs.org/node-fetch/-/node-fetch-2.6.5.tgz", - "integrity": "sha512-mmlIVHJEu5rnIxgEgez6b9GgWXbkZj5YZ7fx+2r94a2E+Uirsp6HsPTPlomfdHtpt/B0cdKviwkoaM6pyvUOpQ==", + "version": "2.6.6", + "resolved": "https://registry.npmjs.org/node-fetch/-/node-fetch-2.6.6.tgz", + "integrity": "sha512-Z8/6vRlTUChSdIgMa51jxQ4lrw/Jy5SOW10ObaA47/RElsAN2c5Pn8bTgFGWn/ibwzXTE8qwr1Yzx28vsecXEA==", "optional": true, "requires": { "whatwg-url": "^5.0.0" @@ -38474,12 +38277,6 @@ "resolved": "https://registry.npmjs.org/prelude-ls/-/prelude-ls-1.1.2.tgz", "integrity": "sha1-IZMqVJ9eUv/ZqCf1cOBL5iqX2lQ=" }, - "source-map": { - "version": "0.6.1", - "resolved": "https://registry.npmjs.org/source-map/-/source-map-0.6.1.tgz", - "integrity": "sha512-UjgapumWlbMhkBgzT7Ykc5YXUT46F0iKu8SGXq0bcwP5dz/h0Plj6enJqjz1Zbq2l5WaqYnrVbwWOWMyF3F47g==", - "optional": true - }, "type-check": { "version": "0.3.2", "resolved": "https://registry.npmjs.org/type-check/-/type-check-0.3.2.tgz", @@ -38579,12 +38376,6 @@ "util-deprecate": "~1.0.1" } }, - "source-map": { - "version": "0.6.1", - "resolved": "https://registry.npmjs.org/source-map/-/source-map-0.6.1.tgz", - "integrity": "sha512-UjgapumWlbMhkBgzT7Ykc5YXUT46F0iKu8SGXq0bcwP5dz/h0Plj6enJqjz1Zbq2l5WaqYnrVbwWOWMyF3F47g==", - "optional": true - }, "string_decoder": { "version": "1.1.1", "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-1.1.1.tgz", @@ -39776,16 +39567,6 @@ "integrity": "sha512-rBJeI5CXAlmy1pV+617WB9J63U6XcazHHF2f2dbJix4XzpUF0RS3Zbj0FGIOCAva5P/d/GBOYaACQ1w+0azUkg==", "dev": true }, - "unix-dgram": { - "version": "2.0.4", - "resolved": "https://registry.npmjs.org/unix-dgram/-/unix-dgram-2.0.4.tgz", - "integrity": "sha512-7tpK6x7ls7J7pDrrAU63h93R0dVhRbPwiRRCawR10cl+2e1VOvF3bHlVJc6WI1dl/8qk5He673QU+Ogv7bPNaw==", - "optional": true, - "requires": { - "bindings": "^1.3.0", - "nan": "^2.13.2" - } - }, "unpipe": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/unpipe/-/unpipe-1.0.0.tgz", diff --git a/package.json b/package.json index d9d5f13e67..db608ae458 100644 --- a/package.json +++ b/package.json @@ -147,7 +147,6 @@ "http-status-code": "^2.1.0", "husky": "^7.0.4", "image-size": "^1.0.0", - "is-url": "^1.2.4", "japanese-characters": "^1.1.0", "javascript-stringify": "^2.1.0", "jest": "^27.3.1", @@ -216,7 +215,7 @@ "rest-dev": "script/rest/update-files.js && npm run dev", "start": "cross-env NODE_ENV=development ENABLED_LANGUAGES='en,ja' nodemon server.mjs", "start-all-languages": "cross-env NODE_ENV=development nodemon server.mjs", - "sync-search": "start-server-and-test sync-search-server 4002 sync-search-indices", + "sync-search": "cross-env NODE_OPTIONS='--max_old_space_size=8192' start-server-and-test sync-search-server 4002 sync-search-indices", "sync-search-ghes-release": "cross-env GHES_RELEASE=1 start-server-and-test sync-search-server 4002 sync-search-indices", "sync-search-indices": "script/sync-search-indices.js", "sync-search-server": "cross-env NODE_ENV=production WEB_CONCURRENCY=1 PORT=4002 node server.mjs", @@ -224,7 +223,8 @@ "translation-check-server": "cross-env NODE_ENV=test PORT=4002 node server.mjs", "translation-check-test": "script/i18n/test-html-pages.js", "test": "cross-env NODE_OPTIONS='--max_old_space_size=8192 --experimental-vm-modules' jest", - "test-watch": "cross-env NODE_OPTIONS=--experimental-vm-modules jest --watch --notify --notifyMode=change --coverage" + "test-watch": "cross-env NODE_OPTIONS=--experimental-vm-modules jest --watch --notify --notifyMode=change --coverage", + "tsc": "tsc --noEmit" }, "lint-staged": { "*.{js,mjs,ts,tsx}": "eslint --cache --fix", diff --git a/script/search/build-records.js b/script/search/build-records.js index 3bf7d23e4b..269318d07a 100644 --- a/script/search/build-records.js +++ b/script/search/build-records.js @@ -61,19 +61,13 @@ export default async function buildRecords( const waiter = domwaiter(permalinks, { maxConcurrent: MAX_CONCURRENT, minTime: MIN_TIME }) .on('page', (page) => { process.stdout.write(pageMarker) - const newRecords = parsePageSectionsIntoRecords(page.href, page.$) + const newRecord = parsePageSectionsIntoRecords(page) const hrefWithoutLocale = page.href.split('/').slice(2).join('/') const popularity = (hasPopularPages && popularPages[hrefWithoutLocale]) || 0.0 - - for (const newRecord of newRecords) { - newRecord.popularity = popularity - } - if (!newRecords.length) { - console.log(chalk.red(`\nno records found: ${page.href}`)) - } - process.stdout.write(recordMarker.repeat(newRecords.length)) - records.push(...newRecords) + newRecord.popularity = popularity + process.stdout.write(recordMarker) + records.push(newRecord) }) .on('error', (err) => { console.error(err) diff --git a/script/search/find-indexable-pages.js b/script/search/find-indexable-pages.js index 43458dba81..72ab2dc73d 100644 --- a/script/search/find-indexable-pages.js +++ b/script/search/find-indexable-pages.js @@ -8,8 +8,8 @@ export default async function findIndexablePages() { .filter((page) => !page.hidden) // exclude pages that are part of WIP or hidden products .filter((page) => !page.parentProduct || !page.parentProduct.wip || page.parentProduct.hidden) - // exclude index homepages - .filter((page) => !page.relativePath.endsWith('index.md')) + // exclude absolute home page (e.g. /en or /ja) + .filter((page) => page.relativePath !== 'index.md') console.log('total pages', allPages.length) console.log('indexable pages', indexablePages.length) diff --git a/script/search/lunr-search-index.js b/script/search/lunr-search-index.js index 7dbd79e22b..a697395455 100755 --- a/script/search/lunr-search-index.js +++ b/script/search/lunr-search-index.js @@ -48,12 +48,18 @@ export default class LunrIndex { this.use(lunr[language]) } + // By default Lunr considers the `-` character to be a word boundary. + // This allows hyphens to be included in the search index. + // If you change this, remember to make it match the indexing separator + // in lib/search/lunr-search.js so the query is tokenized + // identically to the way it was indexed. + this.tokenizer.separator = /[\s]+/ + this.ref('objectID') this.field('url') - this.field('slug') this.field('breadcrumbs') - this.field('heading') - this.field('title') + this.field('headings', { boost: 3 }) + this.field('title', { boost: 5 }) this.field('content') this.field('topics') this.field('customRanking') diff --git a/script/search/parse-page-sections-into-records.js b/script/search/parse-page-sections-into-records.js index 8542d3e3be..da94072408 100644 --- a/script/search/parse-page-sections-into-records.js +++ b/script/search/parse-page-sections-into-records.js @@ -1,18 +1,18 @@ #!/usr/bin/env node -import { chain } from 'lodash-es' import { maxContentLength } from '../../lib/search/config.js' + // This module takes cheerio page object and divides it into sections // using H1,H2 heading elements as section delimiters. The text // that follows each heading becomes the content of the search record. -const urlPrefix = 'https://docs.github.com' -const ignoredHeadingSlugs = ['in-this-article', 'further-reading'] +const ignoredHeadingSlugs = ['in-this-article', 'further-reading', 'prerequisites'] -export default function parsePageSectionsIntoRecords(href, $) { +export default function parsePageSectionsIntoRecords(page) { + const { href, $, languageCode } = page const title = $('h1').text().trim() - const breadcrumbsArray = $('nav.breadcrumbs a') + const breadcrumbsArray = $('[data-search=breadcrumbs] nav.breadcrumbs a') .map((i, el) => { - return $(el).text().trim().replace(/\n/g, ' ').replace(/\s+/g, ' ') + return $(el).text().trim().replace('/', '').replace(/\s+/g, ' ') }) .get() .slice(0, -1) @@ -28,66 +28,109 @@ export default function parsePageSectionsIntoRecords(href, $) { topics.push(productName.replace('GitHub ', '')) } - let records + const objectID = href - const $sections = $('[data-search=article-content] h2') + const rootSelector = '[data-search=article-body]' + const $root = $(rootSelector) + + const $sections = $('h2', $root) .filter('[id]') .filter((i, el) => { return !ignoredHeadingSlugs.includes($(el).attr('id')) }) - if ($sections.length > 0) { - records = $sections - .map((i, el) => { - const heading = $(el).text().trim() - const slug = $(el).attr('id') - const objectID = [href, slug].join('#') - const url = [urlPrefix, objectID].join('') - const content = $(el) - // Platform-specific content is nested in a DIV - // GraphQL content in nested in two DIVS - .nextUntil('h2, div > h2, div > div > h2') - .map((i, el) => $(el).text()) - .get() - .join(' ') - .trim() - .slice(0, maxContentLength) - return { - objectID, - url, - slug, - breadcrumbs, - heading, - title, - content, - topics, - } - }) - .get() - } else { - // There are no sections. Treat the entire article as the record. - const objectID = href - const url = [urlPrefix, objectID].join('') - const content = $( - '[data-search=article-body] p, [data-search=article-body] ul, [data-search=article-body] ol, [data-search=article-body] table' - ) - .map((i, el) => $(el).text()) - .get() - .join(' ') - .trim() - .slice(0, maxContentLength) + const headings = $sections + .map((i, el) => $(el).text()) + .get() + .join(' ') + .trim() - records = [ - { - objectID, - url, - breadcrumbs, - title, - content, - topics, - }, - ] + const intro = $('[data-search=lead] p').text().trim() + + let body = '' + // Typical example pages with no `$root` are: + // https://docs.github.com/en/code-security/guides or + // https://docs.github.com/en/graphql/overview/explorer + // + // We need to avoid these because if you use `getAllText()` on these + // pages, it will extract *everything* from the page, which will + // include the side bar and footer. + // TODO: Come up a custom solution to extract some text from these + // pages that yields some decent content to be searched on, because + // when you view these pages in a browser, there's clearly text there. + if ($root.length > 0) { + body = getAllText($, $root) } - return chain(records).uniqBy('objectID').value() + if (!body && !intro) { + console.warn(`${objectID} has no body and no intro.`) + } + + if (languageCode !== 'en' && body.length > maxContentLength) { + body = body.slice(0, maxContentLength) + } + + const content = `${intro}\n${body}`.trim() + + return { + objectID, + breadcrumbs, + title, + headings, + content, + topics, + } +} + +function getAllText($, $root) { + let text = '' + + // We need this so we can know if we processed, for example, + // a followed by a

    because if that's the case, don't use + // a ' ' to concatenate the texts together but a '\n' instead. + // That means, given this input: + // + //

    Bla

    FooBar

    Hi again

    + // + // we can produce this outcome: + // + // 'Bla\nFoo Bar\nHi again' + // + let previousTagName = '' + + $('p, h2, h3, td, pre, li', $root).each((i, element) => { + const $element = $(element) + if (previousTagName === 'td' && element.tagName !== 'td') { + text += '\n' + } + // Because our cheerio selector is all the block level tags, + // what you might end up with is, from: + // + //
  3. Text

  4. + //
  5. Code
  6. + // + // ['Text', 'Text', 'Code', 'Code'] + // + // because it will spot both the
  7. and the

    . + // If all HTML was exactly like that, you could omit the

  8. selector, + // but a lot of HTML is like this: + // + //
  9. Bare text
  10. + // + // So we need to bail if we're inside a block level element whose parent + // already was a
  11. . + if ((element.tagName === 'p' || element.tagName === 'pre') && element.parent.tagName === 'li') { + return + } + text += $element.text() + if (element.tagName === 'td') { + text += ' ' + } else { + text += '\n' + } + previousTagName = element.tagName + }) + text = text.trim().replace(/\s*[\r\n]+/g, '\n') + + return text } diff --git a/script/search/rank.js b/script/search/rank.js index 3ca7348198..1e20a1091a 100644 --- a/script/search/rank.js +++ b/script/search/rank.js @@ -10,7 +10,7 @@ const rankings = ['/rest', '/graphql', '/site-policy'].reverse() export default function rank(record) { for (const index in rankings) { const pattern = rankings[index] - if (record.url.includes(pattern)) return Number(index) + if (record.objectID.includes(pattern)) return Number(index) } // Set the default ranking to the highest possible diff --git a/script/search/validate-records.js b/script/search/validate-records.js index f252f6a5a9..209e74f749 100644 --- a/script/search/validate-records.js +++ b/script/search/validate-records.js @@ -1,9 +1,7 @@ #!/usr/bin/env node import assert from 'assert' import { isArray, isString, inRange } from 'lodash-es' -import isURL from 'is-url' import countArrayValues from 'count-array-values' -import { maxRecordLength } from '../../lib/search/config.js' export default function validateRecords(name, records) { assert(isString(name) && name.length, '`name` is required') @@ -27,21 +25,10 @@ export default function validateRecords(name, records) { `title must be a string. received: ${record.title}, ${JSON.stringify(record)}` ) - assert( - isURL(record.url), - `url must be a fully qualified URL. received: ${record.url}, ${JSON.stringify(record)}` - ) - assert( inRange(record.customRanking, 0, 4), `customRanking must be an in-range number. received: ${record.customRanking}, (record: ${record.url})` ) - - const recordLength = JSON.stringify(record).length - assert( - recordLength <= maxRecordLength, - `record ${record.url} is too long! ${recordLength} (max: ${maxRecordLength})` - ) }) return true diff --git a/tests/unit/search/fixtures/page-with-sections.html b/tests/unit/search/fixtures/page-with-sections.html index 32f8dc8289..801e8bb927 100644 --- a/tests/unit/search/fixtures/page-with-sections.html +++ b/tests/unit/search/fixtures/page-with-sections.html @@ -1,30 +1,59 @@ -
    + -

    In this article

    -

    This should be ignored.

    +

    I am the page title

    -

    First heading

    -

    Here's a paragraph.

    +
    +

    This is an introduction to the article.

    +
    -

    And another.

    +
    +

    In this article

    +

    This won't be ignored.

    -

    Second heading

    -

    Here's a paragraph in the second section.

    +

    First heading

    -

    And another.

    + +

    Here's a paragraph.

    And another.

    -

    Further reading

    -

    This should be ignored.

    +

    Second heading

    +

    Here's a paragraph in the second section.

    + +

    And another.

    + +

    Table heading

    + + + + + + +
    PeterHuman
    + +
      +
    • Bullet

    • +
    • Point

    • +
    + +
      +
    1. Numbered
    2. +
    3. List
    4. +
    + +

    Further reading

    +

    This won't be ignored.

    This is out the article tag so it should be ignored

    diff --git a/tests/unit/search/fixtures/page-without-body.html b/tests/unit/search/fixtures/page-without-body.html new file mode 100644 index 0000000000..2ce35a0b4f --- /dev/null +++ b/tests/unit/search/fixtures/page-without-body.html @@ -0,0 +1,18 @@ + + + +

    I am outside the article and should not be included

    + + + +

    A page without body

    + +
    +

    This is an introduction to the article.

    +
    diff --git a/tests/unit/search/fixtures/page-without-sections.html b/tests/unit/search/fixtures/page-without-sections.html index 04f619d566..d337a9deb1 100644 --- a/tests/unit/search/fixtures/page-without-sections.html +++ b/tests/unit/search/fixtures/page-without-sections.html @@ -3,16 +3,22 @@

    I am outside the article and should not be included

    -
    +
    - -

    A page without sections

    - -

    First paragraph.

    - -

    Second paragraph.

    +
    + +

    A page without sections

    + +
    +

    This is an introduction to the article.

    +
    + +
    +

    First paragraph.

    + +

    Second paragraph.

    diff --git a/tests/unit/search/parse-page-sections-into-records.js b/tests/unit/search/parse-page-sections-into-records.js index befa1d15b8..5f4f8026a9 100644 --- a/tests/unit/search/parse-page-sections-into-records.js +++ b/tests/unit/search/parse-page-sections-into-records.js @@ -4,6 +4,7 @@ import fs from 'fs/promises' import cheerio from 'cheerio' import parsePageSectionsIntoRecords from '../../../script/search/parse-page-sections-into-records.js' const __dirname = path.dirname(fileURLToPath(import.meta.url)) + const fixtures = { pageWithSections: await fs.readFile( path.join(__dirname, 'fixtures/page-with-sections.html'), @@ -13,6 +14,10 @@ const fixtures = { path.join(__dirname, 'fixtures/page-without-sections.html'), 'utf8' ), + pageWithoutBody: await fs.readFile( + path.join(__dirname, 'fixtures/page-without-body.html'), + 'utf8' + ), } describe('search parsePageSectionsIntoRecords module', () => { @@ -20,52 +25,57 @@ describe('search parsePageSectionsIntoRecords module', () => { const html = fixtures.pageWithSections const $ = cheerio.load(html) const href = '/example/href' - const records = parsePageSectionsIntoRecords(href, $) - expect(Array.isArray(records)).toBe(true) - expect(records.length).toBe(2) - const expected = [ - { - objectID: '/example/href#first', - url: 'https://docs.github.com/example/href#first', - slug: 'first', - breadcrumbs: 'GitHub Actions / actions learning path', - heading: 'First heading', - title: 'I am the page title', - content: "Here's a paragraph. And another.", - topics: ['topic1', 'topic2', 'GitHub Actions', 'Actions'], - }, - { - objectID: '/example/href#second', - url: 'https://docs.github.com/example/href#second', - slug: 'second', - breadcrumbs: 'GitHub Actions / actions learning path', - heading: 'Second heading', - title: 'I am the page title', - content: "Here's a paragraph in the second section. And another.", - topics: ['topic1', 'topic2', 'GitHub Actions', 'Actions'], - }, - ] + const record = parsePageSectionsIntoRecords({ href, $, languageCode: 'en' }) + const expected = { + objectID: '/example/href', + breadcrumbs: 'GitHub Actions / actions learning path', + title: 'I am the page title', + headings: 'First heading Second heading Table heading', + content: + 'This is an introduction to the article.\n' + + "In this article\nThis won't be ignored.\nFirst heading\n" + + "Here's a paragraph.\nAnd another.\nSecond heading\n" + + "Here's a paragraph in the second section.\nAnd another.\n" + + 'Table heading\nPeter Human\n' + + 'Bullet\nPoint\nNumbered\nList\n' + + "Further reading\nThis won't be ignored.", + topics: ['topic1', 'topic2', 'GitHub Actions', 'Actions'], + } - expect(records).toEqual(expected) + expect(record).toEqual(expected) }) test('works for pages without sections', () => { const html = fixtures.pageWithoutSections const $ = cheerio.load(html) const href = '/example/href' - const records = parsePageSectionsIntoRecords(href, $) - expect(Array.isArray(records)).toBe(true) - expect(records.length).toBe(1) - const expected = [ - { - objectID: '/example/href', - url: 'https://docs.github.com/example/href', - breadcrumbs: 'Education / map topic', - title: 'A page without sections', - content: 'First paragraph. Second paragraph.', - topics: ['key1', 'key2', 'key3', 'Education'], - }, - ] - expect(records).toEqual(expected) + const record = parsePageSectionsIntoRecords({ href, $, languageCode: 'en' }) + const expected = { + objectID: '/example/href', + breadcrumbs: 'Education / map topic', + title: 'A page without sections', + headings: '', + content: 'This is an introduction to the article.\nFirst paragraph.\nSecond paragraph.', + topics: ['key1', 'key2', 'key3', 'Education'], + } + + expect(record).toEqual(expected) + }) + + test('works for pages without content', () => { + const html = fixtures.pageWithoutBody + const $ = cheerio.load(html) + const href = '/example/href' + const record = parsePageSectionsIntoRecords({ href, $, languageCode: 'en' }) + const expected = { + objectID: '/example/href', + breadcrumbs: 'Education / map topic', + title: 'A page without body', + headings: '', + content: 'This is an introduction to the article.', + topics: ['key1', 'key2', 'key3', 'Education'], + } + + expect(record).toEqual(expected) }) }) diff --git a/tests/unit/search/rank.js b/tests/unit/search/rank.js index 72907da83f..e174865306 100644 --- a/tests/unit/search/rank.js +++ b/tests/unit/search/rank.js @@ -2,14 +2,14 @@ import rank from '../../../script/search/rank.js' test('search custom rankings', () => { const expectedRankings = [ - ['https://docs.github.com/en/github/actions', 3], - ['https://docs.github.com/en/rest/reference', 2], - ['https://docs.github.com/en/graphql', 1], - ['https://docs.github.com/en/github/site-policy', 0], + ['/en/github/actions', 3], + ['/en/rest/reference', 2], + ['/en/graphql', 1], + ['/en/github/site-policy', 0], ] - expectedRankings.forEach(([url, expectedRanking]) => { - const expectationMessage = `expected ${url} to have a custom ranking of ${expectedRanking}` - expect(rank({ url }), expectationMessage).toBe(expectedRanking) + expectedRankings.forEach(([objectID, expectedRanking]) => { + const expectationMessage = `expected ${objectID} to have a custom ranking of ${expectedRanking}` + expect(rank({ objectID }), expectationMessage).toBe(expectedRanking) }) })