Skip to content

Commit

Permalink
fix: Add retry logic for data downloader (#508)
Browse files Browse the repository at this point in the history
**Reason for Change**:
Retry url download
  • Loading branch information
ishaansehgal99 authored Jul 11, 2024
1 parent 5778600 commit eb0b4f5
Showing 1 changed file with 15 additions and 4 deletions.
19 changes: 15 additions & 4 deletions pkg/tuning/preset-tuning.go
Original file line number Diff line number Diff line change
Expand Up @@ -412,10 +412,21 @@ func handleURLDataSource(ctx context.Context, workspaceObj *kaitov1alpha1.Worksp
for url in $DATA_URLS; do
filename=$(basename "$url" | sed 's/[?=&]/_/g')
echo "Downloading $url to $DATA_VOLUME_PATH/$filename"
curl -sSL $url -o $DATA_VOLUME_PATH/$filename
if [ $? -ne 0 ]; then
echo "Failed to download $url"
fi
retry_count=0
while [ $retry_count -lt 3 ]; do
curl -sSL $url -o $DATA_VOLUME_PATH/$filename
if [ $? -eq 0 ] && [ -s $DATA_VOLUME_PATH/$filename ]; then
echo "Successfully downloaded $url"
break
else
echo "Failed to download $url, retrying..."
retry_count=$((retry_count + 1))
sleep 2
fi
done
if [ $retry_count -eq 3 ]; then
echo "Failed to download $url after 3 attempts"
fi
done
`},
VolumeMounts: []corev1.VolumeMount{
Expand Down

0 comments on commit eb0b4f5

Please sign in to comment.