Add incremenal backoff and random delay to retry tool

images/base/retry-tool.sh

@@ -2,10 +2,12 @@
 set -xe
 
 RETRY_COUNTER=0
-MAX_RETRY=15
+MAX_RETRY=10
+CUR_TIMEOUT=120
 while [[ $RETRY_COUNTER -lt $MAX_RETRY ]]; do
-    timeout 120 "$@" && break || sleep 10
+    timeout $CUR_TIMEOUT "$@" && break || sleep $(shuf -i 5-90 -n 1)
     RETRY_COUNTER=$(( $RETRY_COUNTER + 1 ))
+    CUR_TIMEOUT=$(( $CUR_TIMEOUT + 60 ))
     echo "Retry $RETRY_COUNTER..."
 done
 if [[ $RETRY_COUNTER -ge $MAX_RETRY ]]; then