Merge pull request #3 from envato/aw/retry-when-cache-invalidation-fails

Add an until loop to retry cache invalidation, with exponential backoff
envato · Mar 18, 2022 · e6572a7 · e6572a7
2 parents 1210a13 + 4960219
commit e6572a7
Show file tree

Hide file tree

Showing 2 changed files with 64 additions and 8 deletions.
diff --git a/hooks/post-command b/hooks/post-command
@@ -11,14 +11,33 @@ if [[ ${BUILDKITE_COMMAND_EXIT_STATUS:-0} != '0' ]]; then
 fi
 
 distribution_id="$(plugin_read_config DISTRIBUTION_ID)"
+
+# This tries to create an invalidation with an empty path list
+#
+# - If we have permission to call create-invalidation for this distribution,
+#   we'll recieve an InvalidArgument error, and we can proceed with a real invalidation
+# - Otherwise, either an AccessDenied or NoSuchDistribution error are the most likely results,
+#   in which case we exit with error
+if ! aws cloudfront create-invalidation --distribution-id "$distribution_id" --invalidation-batch 'Paths={Quantity=0,Items=[]},CallerReference=cloudfront-invalidation-buildkite-plugin' 2>&1 | grep InvalidArgument > /dev/null; then
+  echo "Unable to invalidate cloudfront cache: create-invalidation not possible for distribution ($distribution_id) with current credentials"
+  exit 1
+fi
+
 paths=()
 while read -r line ; do
   [[ -n "$line" ]] && paths+=("$line")
 done <<< "$(plugin_read_list PATHS)"
 
 echo "~~~ :cloudfront: Creating Cloudfront invalidation for distribution $distribution_id on paths" "${paths[@]}"
-aws cloudfront create-invalidation \
-  --distribution-id "$distribution_id" \
-  --paths "${paths[@]}" \
-  --query Invalidation.Id \
-  --output text
+SLEEP_PERIOD=15
+PERIOD_LIMIT=480
+until aws cloudfront create-invalidation --distribution-id "$distribution_id" --paths "${paths[@]}" --query Invalidation.Id --output text
+do
+  if [ $SLEEP_PERIOD == $PERIOD_LIMIT ]; then
+    echo "Maximum retries reached - giving up..."
+    exit 1
+  fi
+  echo "Invalidation failed - retrying in ${SLEEP_PERIOD}"
+  sleep ${SLEEP_PERIOD}
+  SLEEP_PERIOD=$((SLEEP_PERIOD+SLEEP_PERIOD))
+done
diff --git a/tests/post-command.bats b/tests/post-command.bats
@@ -10,7 +10,9 @@ load '/usr/local/lib/bats/load.bash'
   export BUILDKITE_PLUGIN_AWS_CLOUDFRONT_INVALIDATION_DISTRIBUTION_ID=test_id
   export BUILDKITE_PLUGIN_AWS_CLOUDFRONT_INVALIDATION_PATHS=/something/*
 
-  stub aws "cloudfront create-invalidation --distribution-id test_id --paths /something/* --query Invalidation.Id --output text : echo cloudfront invalidated"
+  stub aws \
+    "cloudfront create-invalidation --distribution-id test_id --invalidation-batch 'Paths={Quantity=0,Items=[]},CallerReference=cloudfront-invalidation-buildkite-plugin' : echo InvalidArgument" \
+    "cloudfront create-invalidation --distribution-id test_id --paths /something/* --query Invalidation.Id --output text : echo cloudfront invalidated"
 
   run $PWD/hooks/post-command
 
@@ -24,7 +26,9 @@ load '/usr/local/lib/bats/load.bash'
   export BUILDKITE_PLUGIN_AWS_CLOUDFRONT_INVALIDATION_DISTRIBUTION_ID=test_id
   export BUILDKITE_PLUGIN_AWS_CLOUDFRONT_INVALIDATION_PATHS_0=/something/*
 
-  stub aws "cloudfront create-invalidation --distribution-id test_id --paths /something/* --query Invalidation.Id --output text : echo cloudfront invalidated"
+  stub aws \
+    "cloudfront create-invalidation --distribution-id test_id --invalidation-batch 'Paths={Quantity=0,Items=[]},CallerReference=cloudfront-invalidation-buildkite-plugin' : echo InvalidArgument" \
+    "cloudfront create-invalidation --distribution-id test_id --paths /something/* --query Invalidation.Id --output text : echo cloudfront invalidated"
 
   run $PWD/hooks/post-command
 
@@ -39,7 +43,9 @@ load '/usr/local/lib/bats/load.bash'
   export BUILDKITE_PLUGIN_AWS_CLOUDFRONT_INVALIDATION_PATHS_0=/something/*
   export BUILDKITE_PLUGIN_AWS_CLOUDFRONT_INVALIDATION_PATHS_1=/something-else/*
 
-  stub aws "cloudfront create-invalidation --distribution-id test_id --paths /something/* /something-else/* --query Invalidation.Id --output text : echo cloudfront invalidated"
+  stub aws \
+    "cloudfront create-invalidation --distribution-id test_id --invalidation-batch 'Paths={Quantity=0,Items=[]},CallerReference=cloudfront-invalidation-buildkite-plugin' : echo InvalidArgument" \
+    "cloudfront create-invalidation --distribution-id test_id --paths /something/* /something-else/* --query Invalidation.Id --output text : echo cloudfront invalidated"
 
   run $PWD/hooks/post-command
 
@@ -57,3 +63,34 @@ load '/usr/local/lib/bats/load.bash'
 
   assert_success
 }
+
+@test "Retries after unsuccessfully submitting a validation request" {
+  export BUILDKITE_COMMAND_EXIT_STATUS=0
+  export BUILDKITE_PLUGIN_AWS_CLOUDFRONT_INVALIDATION_DISTRIBUTION_ID=test_id
+  export BUILDKITE_PLUGIN_AWS_CLOUDFRONT_INVALIDATION_PATHS_0=/something/*
+  export BUILDKITE_PLUGIN_AWS_CLOUDFRONT_INVALIDATION_PATHS_1=/something-else/*
+
+  stub aws \
+    "cloudfront create-invalidation --distribution-id test_id --invalidation-batch 'Paths={Quantity=0,Items=[]},CallerReference=cloudfront-invalidation-buildkite-plugin' : echo InvalidArgument" \
+    "cloudfront create-invalidation --distribution-id test_id --paths /something/* /something-else/* --query Invalidation.Id --output text : return 1" \
+    "cloudfront create-invalidation --distribution-id test_id --paths /something/* /something-else/* --query Invalidation.Id --output text : echo cloudfront invalidated"
+  stub sleep "15 : echo sleeping"
+
+  run $PWD/hooks/post-command
+
+  assert_success
+  assert_output --partial "sleeping"
+  assert_output --partial "cloudfront invalidated"
+  unstub aws
+  unstub sleep
+}
+
+@test "Stops executing if a credential problem is detected" {
+  export BUILDKITE_COMMAND_EXIT_STATUS=0
+
+  stub aws "cloudfront create-invalidation --distribution-id test_id --invalidation-batch 'Paths={Quantity=0,Items=[]},CallerReference=cloudfront-invalidation-buildkite-plugin' : echo AccessDenied" \
+
+  run $PWD/hooks/post-command
+
+  assert_failure
+}