Transient error bugfix

go.mod

@@ -9,17 +9,18 @@ require (
 	github.com/hashicorp/terraform-plugin-go v0.14.3
 	github.com/hashicorp/terraform-plugin-sdk/v2 v2.25.0
 	github.com/kofalt/go-memoize v0.0.0-20220914132407-0b5d6a304579
-	github.com/microsoft/go-mssqldb v0.20.0
+	github.com/microsoft/go-mssqldb v1.6.0
 	github.com/patrickmn/go-cache v2.1.0+incompatible
 	github.com/pkg/errors v0.9.1
-	github.com/stretchr/testify v1.8.1
+	github.com/sethvargo/go-retry v0.2.4
+	github.com/stretchr/testify v1.8.4
 )
 
 require (
-	github.com/Azure/azure-sdk-for-go/sdk/azcore v1.1.3 // indirect
-	github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.1.0 // indirect
-	github.com/Azure/azure-sdk-for-go/sdk/internal v1.0.0 // indirect
-	github.com/AzureAD/microsoft-authentication-library-for-go v0.5.1 // indirect
+	github.com/Azure/azure-sdk-for-go/sdk/azcore v1.7.1 // indirect
+	github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.3.0 // indirect
+	github.com/Azure/azure-sdk-for-go/sdk/internal v1.3.0 // indirect
+	github.com/AzureAD/microsoft-authentication-library-for-go v1.1.0 // indirect
 	github.com/Masterminds/goutils v1.1.1 // indirect
 	github.com/Masterminds/semver/v3 v3.1.1 // indirect
 	github.com/Masterminds/sprig/v3 v3.2.2 // indirect
@@ -29,8 +30,8 @@ require (
 	github.com/bgentry/speakeasy v0.1.0 // indirect
 	github.com/davecgh/go-spew v1.1.1 // indirect
 	github.com/fatih/color v1.13.0 // indirect
-	github.com/golang-jwt/jwt v3.2.2+incompatible // indirect
-	github.com/golang-sql/civil v0.0.0-20190719163853-cb61b32ac6fe // indirect
+	github.com/golang-jwt/jwt/v5 v5.0.0 // indirect
+	github.com/golang-sql/civil v0.0.0-20220223132316-b832511892a9 // indirect
 	github.com/golang-sql/sqlexp v0.1.0 // indirect
 	github.com/golang/protobuf v1.5.2 // indirect
 	github.com/google/go-cmp v0.5.9 // indirect
@@ -76,12 +77,12 @@ require (
 	github.com/vmihailenco/msgpack/v4 v4.3.12 // indirect
 	github.com/vmihailenco/tagparser v0.1.1 // indirect
 	github.com/zclconf/go-cty v1.12.1 // indirect
-	golang.org/x/crypto v0.6.0 // indirect
-	golang.org/x/mod v0.7.0 // indirect
-	golang.org/x/net v0.7.0 // indirect
+	golang.org/x/crypto v0.12.0 // indirect
+	golang.org/x/mod v0.14.0 // indirect
+	golang.org/x/net v0.14.0 // indirect
 	golang.org/x/sync v0.0.0-20220907140024-f12130a52804 // indirect
-	golang.org/x/sys v0.5.0 // indirect
-	golang.org/x/text v0.7.0 // indirect
+	golang.org/x/sys v0.11.0 // indirect
+	golang.org/x/text v0.12.0 // indirect
 	google.golang.org/appengine v1.6.6 // indirect
 	google.golang.org/genproto v0.0.0-20200711021454-869866162049 // indirect
 	google.golang.org/grpc v1.51.0 // indirect

internal/sql/connection.go

@@ -4,12 +4,13 @@ import (
 	"context"
 	"database/sql"
 	"fmt"
-	"github.com/PGSSoft/terraform-provider-mssql/internal/utils"
-	"github.com/kofalt/go-memoize"
 	"net/url"
 	"regexp"
 	"time"
 
+	"github.com/PGSSoft/terraform-provider-mssql/internal/utils"
+	"github.com/kofalt/go-memoize"
+
 	"github.com/hashicorp/terraform-plugin-framework/diag"
 	_ "github.com/microsoft/go-mssqldb"
 	_ "github.com/microsoft/go-mssqldb/azuread"
@@ -73,14 +74,14 @@ func (cd ConnectionDetails) Open(ctx context.Context) (Connection, diag.Diagnost
 
 func (c *connection) IsAzure(ctx context.Context) bool {
 	var edition string
-	if err := c.conn.QueryRowContext(ctx, "SELECT SERVERPROPERTY('edition')").Scan(&edition); err != nil {
+	if err := QueryRowContextWithRetry(ctx, c.conn, "SELECT SERVERPROPERTY('edition')").Scan(&edition); err != nil {
 		utils.AddError(ctx, "Failed to determine server edition", err)
 	}
 	return azureSQLEditionPattern.MatchString(edition)
 }
 
 func (c *connection) GetPermissions(ctx context.Context, principalId GenericServerPrincipalId) ServerPermissions {
-	res, err := c.conn.QueryContext(ctx, "SELECT [permission_name], [state] FROM sys.server_permissions WHERE [class]=100 AND [grantee_principal_id]=@p1", principalId)
+	res, err := QueryContextWithRetry(ctx, c.conn, "SELECT [permission_name], [state] FROM sys.server_permissions WHERE [class]=100 AND [grantee_principal_id]=@p1", principalId)
 	perms := ServerPermissions{}
 
 	switch err {
@@ -144,7 +145,7 @@ func (cd ConnectionDetails) getConnectionString(ctx context.Context) (string, di
 }
 
 func (c *connection) exec(ctx context.Context, query string, args ...any) sql.Result {
-	res, err := c.conn.ExecContext(ctx, query, args...)
+	res, err := ExecContextWithRetry(ctx, c.conn, query, args...)
 
 	if err != nil {
 		utils.AddError(ctx, "Could not execute SQL", err)
@@ -199,14 +200,14 @@ func (c *connection) getDBSqlConnection(ctx context.Context, dbName string) *sql
 
 func (c *connection) lookupServerPrincipalName(ctx context.Context, id GenericServerPrincipalId) string {
 	var name string
-	err := c.conn.QueryRowContext(ctx, "SELECT [name] FROM sys.server_principals WHERE [principal_id]=@p1", id).Scan(&name)
+	err := QueryRowContextWithRetry(ctx, c.conn, "SELECT [name] FROM sys.server_principals WHERE [principal_id]=@p1", id).Scan(&name)
 	utils.AddError(ctx, "Failed to lookup server principal name", err)
 	return name
 }
 
 func (c *connection) lookupServerPrincipalId(ctx context.Context, name string) GenericServerPrincipalId {
 	var id GenericServerPrincipalId
-	err := c.conn.QueryRowContext(ctx, "SELECT [principal_id] FROM sys.server_principals WHERE [name]=@p1", name).Scan(&id)
+	err := QueryRowContextWithRetry(ctx, c.conn, "SELECT [principal_id] FROM sys.server_principals WHERE [name]=@p1", name).Scan(&id)
 	utils.AddError(ctx, "Failed to lookup server principal ID", err)
 	return id
 }

internal/sql/connectionMock_test.go

@@ -3,6 +3,7 @@ package sql
 import (
 	"context"
 	"database/sql"
+
 	"github.com/PGSSoft/terraform-provider-mssql/internal/utils"
 	"github.com/stretchr/testify/mock"
 )
@@ -39,7 +40,7 @@ func (c *connectionMock) RevokePermission(ctx context.Context, principalId Gener
 }
 
 func (c *connectionMock) exec(ctx context.Context, query string, args ...any) sql.Result {
-	res, err := c.db.ExecContext(ctx, query, args...)
+	res, err := ExecContextWithRetry(ctx, c.db, query, args...)
 	if err != nil {
 		utils.AddError(ctx, "mock error", err)
 	}

internal/sql/database.go

@@ -4,8 +4,9 @@ import (
 	"context"
 	"database/sql"
 	"fmt"
-	"github.com/PGSSoft/terraform-provider-mssql/internal/utils"
 	"strings"
+
+	"github.com/PGSSoft/terraform-provider-mssql/internal/utils"
 )
 
 const NullDatabaseId = DatabaseId(-1)
@@ -69,7 +70,7 @@ func GetDatabase(_ context.Context, conn Connection, id DatabaseId) Database {
 func GetDatabaseByName(ctx context.Context, conn Connection, name string) Database {
 	id := DatabaseId(0)
 
-	if err := conn.getSqlConnection(ctx).QueryRowContext(ctx, "SELECT database_id FROM sys.databases WHERE [name] = @p1", name).Scan(&id); err != nil {
+	if err := QueryRowContextWithRetry(ctx, conn.getSqlConnection(ctx), "SELECT database_id FROM sys.databases WHERE [name] = @p1", name).Scan(&id); err != nil {
 		utils.AddError(ctx, fmt.Sprintf("Failed to retrieve DB ID for name '%s'", name), err)
 		return nil
 	}
@@ -81,7 +82,7 @@ func GetDatabases(ctx context.Context, conn Connection) map[DatabaseId]Database
 	const errorSummary = "Failed to retrieve list of DBs"
 	result := map[DatabaseId]Database{}
 
-	switch rows, err := conn.getSqlConnection(ctx).QueryContext(ctx, "SELECT [database_id] FROM sys.databases"); err {
+	switch rows, err := QueryContextWithRetry(ctx, conn.getSqlConnection(ctx), "SELECT [database_id] FROM sys.databases"); err {
 	case sql.ErrNoRows: // ignore
 	case nil:
 		for rows.Next() {
@@ -151,7 +152,7 @@ func (db *database) Query(ctx context.Context, script string) []map[string]strin
 		return nil
 	}
 
-	rows, err := conn.QueryContext(ctx, script)
+	rows, err := QueryContextWithRetry(ctx, conn, script)
 
 	if err != nil {
 		utils.AddError(ctx, "Failed to execute get state script", err)
@@ -191,7 +192,7 @@ func (db *database) Query(ctx context.Context, script string) []map[string]strin
 }
 
 func (db *database) Exec(ctx context.Context, script string) {
-	if _, err := db.connect(ctx).ExecContext(ctx, script); err != nil {
+	if _, err := ExecContextWithRetry(ctx, db.connect(ctx), script); err != nil {
 		utils.AddError(ctx, "Failed to execute SQL script", err)
 	}
 }
@@ -203,8 +204,8 @@ func (db *database) GetPermissions(ctx context.Context, id GenericDatabasePrinci
 		return nil
 	}
 
-	res, err := conn.
-		QueryContext(ctx, "SELECT [permission_name], [state] FROM sys.database_permissions WHERE [class] = 0 AND [state] IN ('G', 'W') AND [grantee_principal_id] = @p1", id)
+	res, err :=
+		QueryContextWithRetry(ctx, conn, "SELECT [permission_name], [state] FROM sys.database_permissions WHERE [class] = 0 AND [state] IN ('G', 'W') AND [grantee_principal_id] = @p1", id)
 
 	perms := DatabasePermissions{}
 
@@ -238,7 +239,7 @@ func (db *database) GrantPermission(ctx context.Context, id GenericDatabasePrinc
 				stat += " WITH GRANT OPTION"
 			}
 
-			_, err := conn.ExecContext(ctx, stat)
+			_, err := ExecContextWithRetry(ctx, conn, stat)
 			utils.AddError(ctx, "Failed to grant permission", err)
 		})
 }
@@ -254,7 +255,7 @@ func (db *database) UpdatePermission(ctx context.Context, id GenericDatabasePrin
 				stat = fmt.Sprintf("REVOKE GRANT OPTION FOR %s TO [%s]", permission.Name, userName)
 			}
 
-			_, err := conn.ExecContext(ctx, stat)
+			_, err := ExecContextWithRetry(ctx, conn, stat)
 			utils.AddError(ctx, "Failed to modify permission grant", err)
 		})
 }
@@ -266,16 +267,16 @@ func (db *database) RevokePermission(ctx context.Context, id GenericDatabasePrin
 	utils.StopOnError(ctx).
 		Then(func() {
 			stat := fmt.Sprintf("REVOKE %s TO [%s] CASCADE", permissionName, userName)
-			_, err := conn.ExecContext(ctx, stat)
+			_, err := ExecContextWithRetry(ctx, conn, stat)
 			utils.AddError(ctx, "Failed to revoke permission", err)
 		})
 }
 
 func (db *database) getSettingsRaw(ctx context.Context) (DatabaseSettings, error) {
 	var settings DatabaseSettings
-	err := db.conn.getSqlConnection(ctx).
-		QueryRowContext(ctx, "SELECT [name], collation_name FROM sys.databases WHERE [database_id] = @p1", db.id).
-		Scan(&settings.Name, &settings.Collation)
+	err :=
+		QueryRowContextWithRetry(ctx, db.conn.getSqlConnection(ctx), "SELECT [name], collation_name FROM sys.databases WHERE [database_id] = @p1", db.id).
+			Scan(&settings.Name, &settings.Collation)
 	return settings, err
 }
 
@@ -299,9 +300,9 @@ func (db *database) getUserName(ctx context.Context, id GenericDatabasePrincipal
 		Then(func() {
 			var err error
 			if id == EmptyDatabasePrincipalId {
-				err = conn.QueryRowContext(ctx, "SELECT USER_NAME()").Scan(&name)
+				err = QueryRowContextWithRetry(ctx, conn, "SELECT USER_NAME()").Scan(&name)
 			} else {
-				err = conn.QueryRowContext(ctx, "SELECT USER_NAME(@p1)", id).Scan(&name)
+				err = QueryRowContextWithRetry(ctx, conn, "SELECT USER_NAME(@p1)", id).Scan(&name)
 			}
 
 			if err != nil {

internal/sql/databaseRetry.go

@@ -0,0 +1,86 @@
+package sql
+
+import (
+	"context"
+	"database/sql"
+	"slices"
+	"time"
+
+	mssql "github.com/microsoft/go-mssqldb"
+	"github.com/sethvargo/go-retry"
+)
+
+// These are all error codes by the (mssql|Azure SQL) Server that can be retried
+var retryableErrors = []int32{539, 617, 952, 956, 988, 1205, 1807, 3055, 3762, 5034, 5059, 5061, 5065, 5295, 8628, 8645, 10922, 10930, 12111, 14258, 16528, 19510, 20689, 22380, 22498, 22754, 22758, 22759, 22760, 25003, 25738, 25740, 27118, 27230, 30024, 30026, 30085, 33115, 33116, 33136, 40602, 40613, 40642, 40648, 40671, 40675, 40806, 40807, 40825, 40938, 41828, 41838, 42104, 42106, 45156, 45157, 45161, 45168, 45169, 45182, 45509, 45541, 45727, 47132, 49510, 49518, 49918}
+
+// This backoff func is configured to wait a maximum of 100 seconds.
+// A common retryable errors is a paused db, that needs some time to
+// wake up to execute the query.
+// The auto-resume of paused db is "in the order of one minute".
+// See:
+// (https://learn.microsoft.com/en-us/azure/azure-sql/database/serverless-tier-overview?view=azuresql&tabs=general-purpose#latency)
+func ExpBackoff() retry.Backoff {
+	backoff := retry.NewFibonacci(1 * time.Second)
+	backoff = retry.WithMaxDuration(100*time.Second, backoff)
+	return backoff
+}
+
+// The err that is returned by executing a query is checked against the list of
+// all retryable errors, and if so the err is marked as retryable.
+func CheckIfRetryable(err error) (checkedErr error) {
+	if mssqldb, ok := err.(mssql.Error); ok {
+		if slices.Contains(retryableErrors, mssqldb.Number) {
+			return retry.RetryableError(err)
+		}
+	}
+	return err
+}
+
+// This is "conn.ExecContext" wrapped with a retry mechanism for retrying transient
+// errors. Should behave in all other regards the same as the original.
+func ExecContextWithRetry(ctx context.Context, conn *sql.DB, query string, args ...any) (res sql.Result, err error) {
+	backoff := ExpBackoff()
+
+	if err := retry.Do(ctx, backoff,
+		func(ctx context.Context) error {
+			if res, err = conn.ExecContext(ctx, query, args...); err != nil {
+				return CheckIfRetryable(err)
+			}
+			return nil
+		}); err != nil {
+		return res, err
+	}
+	return res, nil
+}
+
+// This is "conn.QueryContext" wrapped with a retry mechanism for retrying transient
+// errors. Should behave in all other regards the same as the original.
+func QueryContextWithRetry(ctx context.Context, conn *sql.DB, query string, args ...any) (rows *sql.Rows, err error) {
+	backoff := ExpBackoff()
+
+	if err := retry.Do(ctx, backoff,
+		func(ctx context.Context) error {
+			if rows, err = conn.QueryContext(ctx, query, args...); err != nil {
+				return CheckIfRetryable(err)
+			}
+			return nil
+		}); err != nil {
+		return rows, err
+	}
+	return rows, nil
+}
+
+// This is "conn.QueryRowContext" wrapped with a retry mechanism for retrying transient
+// errors. Should behave in all other regards the same as the original.
+func QueryRowContextWithRetry(ctx context.Context, conn *sql.DB, query string, args ...any) *sql.Row {
+	backoff := ExpBackoff()
+
+	var row *sql.Row
+	retry.Do(ctx, backoff,
+		func(ctx context.Context) (err error) {
+			row = conn.QueryRowContext(ctx, query, args...)
+			err = row.Err()
+			return CheckIfRetryable(err)
+		})
+	return row
+}