From 0971a6ebf719752cacd70baf5d5b76f955529b52 Mon Sep 17 00:00:00 2001
From: Peter Bengtsson <peterbe@github.com>
Date: Mon, 3 Oct 2022 20:38:32 +0200
Subject: [PATCH] cope with odd 404 errors on /_cat/indices (#31375)

---
 .../workflows/sync-search-elasticsearch.yml   |  7 ++-
 script/helpers/retry-on-error-test.js         | 43 +++++++++++++++++++
 script/search/index-elasticsearch.js          | 23 +++++++++-
 3 files changed, 69 insertions(+), 4 deletions(-)
 create mode 100644 script/helpers/retry-on-error-test.js

diff --git a/.github/workflows/sync-search-elasticsearch.yml b/.github/workflows/sync-search-elasticsearch.yml
index 086ac8ca62..f39ffd5b81 100644
--- a/.github/workflows/sync-search-elasticsearch.yml
+++ b/.github/workflows/sync-search-elasticsearch.yml
@@ -108,8 +108,11 @@ jobs:
 
       - name: Check created indexes and aliases
         run: |
-          curl --fail --retry-connrefused --retry 5 ${{ env.ELASTICSEARCH_URL }}/_cat/indices?v
-          curl --fail --retry-connrefused --retry 5 ${{ env.ELASTICSEARCH_URL }}/_cat/indices?v
+          # Not using `--fail` here because I've observed that it can fail
+          # with a rather cryptic 404 error when it should, if anything, be
+          # a 200 OK with a list of no indices.
+          curl --retry-connrefused --retry 5 ${{ env.ELASTICSEARCH_URL }}/_cat/indices?v
+          curl --retry-connrefused --retry 5 ${{ env.ELASTICSEARCH_URL }}/_cat/indices?v
 
       - name: Send Slack notification if workflow fails
         uses: someimportantcompany/github-actions-slack-message@f8d28715e7b8a4717047d23f48c39827cacad340
diff --git a/script/helpers/retry-on-error-test.js b/script/helpers/retry-on-error-test.js
new file mode 100644
index 0000000000..f929d4b80a
--- /dev/null
+++ b/script/helpers/retry-on-error-test.js
@@ -0,0 +1,43 @@
+// [start-readme]
+//
+// Return a function that you can use to run any code within and if it
+// throws you get a chance to say whether to sleep + retry.
+// Example:
+//
+//   async function mainFunction() {
+//     if (Math.random() > 0.9) throw new Error('too large')
+//     return 'OK'
+//   }
+//
+//   const errorTest = (err) => err instanceof Error && err.message.includes('too large')
+//   const config = {  // all optional
+//     attempts: 3,
+//     sleepTime: 800,
+//     onError: (err, attempts) => console.warn(`Failed ${attempts} attempts`)
+//   }
+//   const ok = await retry(errorTest, mainFunction, config)
+//
+//
+// [end-readme]
+
+const sleep = (ms) => new Promise((resolve) => setTimeout(resolve, ms))
+
+export async function retryOnErrorTest(
+  errorTest,
+  callback,
+  { attempts = 10, sleepTime = 1000, onError = () => {} } = {}
+) {
+  while (--attempts) {
+    try {
+      return await callback()
+    } catch (error) {
+      if (errorTest(error)) {
+        // console.warn('Sleeping on', error.message, { attempts })
+        if (onError) onError(error, attempts)
+        await sleep(sleepTime)
+      } else {
+        throw error
+      }
+    }
+  }
+}
diff --git a/script/search/index-elasticsearch.js b/script/search/index-elasticsearch.js
index 1a72b58fa0..25154d22e5 100755
--- a/script/search/index-elasticsearch.js
+++ b/script/search/index-elasticsearch.js
@@ -15,6 +15,7 @@ import { program, Option } from 'commander'
 import chalk from 'chalk'
 import dotenv from 'dotenv'
 
+import { retryOnErrorTest } from '../helpers/retry-on-error-test.js'
 import { languageKeys } from '../../lib/languages.js'
 import { allVersions } from '../../lib/all-versions.js'
 import { decompress } from '../../lib/search/compress.js'
@@ -351,14 +352,32 @@ async function indexVersion(
   console.log(`Alias ${indexName} -> ${thisAlias}`)
 
   // const indices = await client.cat.indices({ format: 'json' })
-  const { body: indices } = await client.cat.indices({ format: 'json' })
+  const { body: indices } = await retryOnErrorTest(
+    (error) => {
+      return error instanceof errors.ResponseError && error.statusCode === 404
+    },
+    () => client.cat.indices({ format: 'json' }),
+    {
+      // Combined, this is a total of 30 seconds which is not long
+      // for an Action that runs based on automation.
+      attempts: 10,
+      sleepTime: 3000,
+      onError: (error, attempts) => {
+        console.warn(
+          chalk.yellow(
+            `Failed to get a list of indexes for '${indexName}' (${error.message}). Will attempt ${attempts} more times.`
+          )
+        )
+      },
+    }
+  )
   for (const index of indices) {
     if (index.index !== thisAlias && index.index.startsWith(indexName)) {
       aliasUpdates.push({ remove_index: { index: index.index } })
       console.log('Deleting index', index.index)
     }
   }
-  console.log('Updating alias actions:', aliasUpdates)
+  if (verbose) console.log('Updating alias actions:', aliasUpdates)
   await client.indices.updateAliases({ body: { actions: aliasUpdates } })
 }