From b3af722730b24bd8c91bbd4e7fadcc95ed432cfa Mon Sep 17 00:00:00 2001
From: Robert Sese <734194+rsese@users.noreply.github.com>
Date: Thu, 9 Jun 2022 16:48:02 -0500
Subject: [PATCH] Fix daily link checker (#28346)

* reduce linkinator concurrency from 300 to 100

* reduce it and use self-hosted

* try again

* actually do concurrency of 10

* move the override to the Action workflow

* Trying `DISABLE_RENDERING_CACHE`

* back to 100?

* super low concurrency

* 3??

* 1??

* try self-hosted + 100 concurrency

Co-authored-by: Peter Bengtsson <mail@peterbe.com>
---
 .github/workflows/check-all-english-links.yml | 17 ++++++++++++++---
 script/check-english-links.js                 |  4 +++-
 2 files changed, 17 insertions(+), 4 deletions(-)

diff --git a/.github/workflows/check-all-english-links.yml b/.github/workflows/check-all-english-links.yml
index 59ab0da2d1..75294e403d 100644
--- a/.github/workflows/check-all-english-links.yml
+++ b/.github/workflows/check-all-english-links.yml
@@ -17,7 +17,7 @@ jobs:
   check_all_english_links:
     name: Check all links
     if: github.repository == 'github/docs-internal'
-    runs-on: ubuntu-latest
+    runs-on: ${{ fromJSON('["ubuntu-latest", "self-hosted"]')[github.repository == 'github/docs-internal'] }}
     env:
       GITHUB_TOKEN: ${{ secrets.DOCUBOT_READORG_REPO_WORKFLOW_SCOPES }}
       FIRST_RESPONDER_PROJECT: Docs content first responder
@@ -56,10 +56,21 @@ jobs:
           # The default is 10s. But because this runs overnight, we can
           # be a lot more patient.
           REQUEST_TIMEOUT: 20000
+          # The default is 300 which works OK on a fast macbook pro
+          # but so well in Actions.
+          LINKINATOR_CONCURRENCY: 100
         run: |
-          node server.mjs &
+          node server.mjs > /tmp/stdout.log 2> /tmp/stderr.log &
           sleep 5
-          curl --retry-connrefused --retry 3 -I http://localhost:4000/
+          curl --retry-connrefused --retry 4 -I http://localhost:4000/
+
+      - if: ${{ failure() }}
+        name: Debug server outputs on errors
+        run: |
+          echo "____STDOUT____"
+          cat /tmp/stdout.log
+          echo "____STDERR____"
+          cat /tmp/stderr.log
 
       - name: Run script
         run: |
diff --git a/script/check-english-links.js b/script/check-english-links.js
index 05eb2b5f31..e594ee00f2 100755
--- a/script/check-english-links.js
+++ b/script/check-english-links.js
@@ -43,6 +43,8 @@ const DISPLAY_MAX_LENGTH = parseInt(process.env.DISPLAY_MAX_LENGTH || '30000', 1
 // Links with these codes may or may not really be broken.
 const retryStatusCodes = [429, 503, 'Invalid']
 
+const LINKINATOR_CONCURRENCY = parseInt(process.env.LINKINATOR_CONCURRENCY || '300')
+
 program
   .description('Check all links in the English docs.')
   .option(
@@ -73,7 +75,7 @@ const enterpriseReleasesToSkip = new RegExp(`${root}.+?[/@](${deprecated.join('|
 
 const config = {
   path: program.opts().path || englishRoot,
-  concurrency: 300,
+  concurrency: LINKINATOR_CONCURRENCY,
   // If this is a dry run, turn off recursion.
   recurse: !program.opts().dryRun,
   silent: true,