build: retry attemps for tunnel connections (#6246)

On the CI it happens pretty often that the tunnel connection to the different providers like Browserstack or Saucelabs is timing out.

Instead of always restarting manually, the script should automatically retry to run the connection establishing script.

scripts/ci/sources/retry-call.sh

@@ -0,0 +1,25 @@
+#!/bin/bash
+
+# Function that can be used to re-call a specific command a specific amount of times if it fails.
+#   @param ${1} Amount of retry calls
+#   @param ${2} Code block that will be invoked
+function retryCall() {
+  retries=0
+
+  until [ ${retries} -gt ${1} ]; do
+    # Call the command and break the loop if it exits without any errors.
+    (${2}) && return 0
+
+    # Increase the counter of the invoked retries.
+    retries=$[${retries} + 1]
+
+    # Sleep a few seconds and log an info message if there are still remaining retries.
+    if [ ${retries} -le ${1} ]; then
+      echo "Script didn't exit without errors. Retrying $[${1} - retries + 1] times..."
+      sleep 1
+    fi
+  done
+
+  echo "Script exited with errors. Exiting process.."
+  exit 1
+}

scripts/ci/sources/tunnel.sh

@@ -1,5 +1,10 @@
 #!/usr/bin/env bash
 
+# Load the retry-call utility function.
+source ./scripts/ci/sources/retry-call.sh
+
+# Variable the specifies how often the wait script should be invoked if it fails.
+WAIT_RETRIES=1
 
 start_tunnel() {
   case "$MODE" in
@@ -17,15 +22,14 @@ start_tunnel() {
 wait_for_tunnel() {
   case "$MODE" in
     e2e*|saucelabs*)
-      ./scripts/saucelabs/wait-tunnel.sh
+      retryCall ${WAIT_RETRIES} ./scripts/saucelabs/wait-tunnel.sh
       ;;
     browserstack*)
-      ./scripts/browserstack/wait-tunnel.sh
+      retryCall ${WAIT_RETRIES} ./scripts/browserstack/wait-tunnel.sh
       ;;
     *)
       ;;
   esac
-  sleep 10
 }
 
 teardown_tunnel() {