Retry improvements on bulk indexing (#45059)

Co-authored-by: Robert Sese <734194+rsese@users.noreply.github.com>

script/helpers/retry-on-error-test.js

@@ -17,6 +17,9 @@
 //   }
 //   const ok = await retry(errorTest, mainFunction, config)
 //
+// Note that, by default, the sleep time is "exponential" by a factor of
+// 1.5. So the first sleep will, in the above example,
+// be 800ms. Then 1,200ms, Then 1,800ms. etc.
 //
 // [end-readme]
 
@@ -25,19 +28,49 @@ const sleep = (ms) => new Promise((resolve) => setTimeout(resolve, ms))
 export async function retryOnErrorTest(
   errorTest,
   callback,
-  { attempts = 10, sleepTime = 1000, onError = () => {} } = {},
+  {
+    attempts = 4,
+    sleepTime = 1000,
+    exponential = 1.5,
+    jitterPercent = 25,
+    onError = () => {},
+  } = {},
 ) {
-  while (--attempts) {
+  while (true) {
     try {
       return await callback()
     } catch (error) {
-      if (errorTest(error)) {
-        // console.warn('Sleeping on', error.message, { attempts })
-        if (onError) onError(error, attempts)
-        await sleep(sleepTime)
+      if (error instanceof Error && attempts > 0 && errorTest(error)) {
+        if (onError) onError(error, attempts, sleepTime)
+        attempts--
+        // The reason for the jitter is to avoid a thundering herd problem.
+        // Suppose two independent processes/threads start at the same time.
+        // They both fail, perhaps due to rate limiting. Now, if they both
+        // sleep for 30 seconds in the first retry attempt, it'll just
+        // clash again 30 seconds later. But if you add a bit of jitter, at
+        // the next attempt these independent processes/threads will now
+        // start at slightly different times.
+
+        // According to the Oxford English dictionary, they define "jitter" as:
+        //
+        //    slight irregular movement, variation, or unsteadiness,
+        //    especially in an electrical signal or electronic device.
+        //
+        await sleep(addJitter(sleepTime, jitterPercent))
+        if (exponential) {
+          sleepTime *= 2
+        }
       } else {
         throw error
       }
     }
   }
 }
+
+function addJitter(num, percent) {
+  // Return the number plus between 0 and $percent of that number.
+  // For example, for 1,000 with a 20% jitter you might get 1133.4
+  // because you start with 1,000 and 13.4% is a random number between
+  // 0 and 20%.
+  return num + Math.random() * percent * 0.01 * num
+}