feat: Validate hub download checksums (#158)

Co-authored-by: Kemal Hadimli <[email protected]>

go.mod

@@ -5,7 +5,7 @@ go 1.21.1
 require (
 	github.com/apache/arrow/go/v14 v14.0.0-20231031200323-c49e24273160
 	github.com/avast/retry-go/v4 v4.5.0
-	github.com/cloudquery/cloudquery-api-go v1.4.3
+	github.com/cloudquery/cloudquery-api-go v1.4.4
 	github.com/docker/docker v24.0.7+incompatible
 	github.com/docker/go-connections v0.4.0
 	github.com/ghodss/yaml v1.0.0
@@ -74,7 +74,7 @@ require (
 	github.com/mattn/go-colorable v0.1.13 // indirect
 	github.com/mattn/go-isatty v0.0.19 // indirect
 	github.com/mattn/go-runewidth v0.0.15 // indirect
-	github.com/microcosm-cc/bluemonday v1.0.25 // indirect
+	github.com/microcosm-cc/bluemonday v1.0.26 // indirect
 	github.com/mitchellh/colorstring v0.0.0-20190213212951-d06e56a500db // indirect
 	github.com/moby/term v0.5.0 // indirect
 	github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect

go.sum

@@ -38,8 +38,8 @@ github.com/chenzhuoyu/iasm v0.9.0 h1:9fhXjVzq5hUy2gkhhgHl95zG2cEAhw9OSGs8toWWAwo
 github.com/chenzhuoyu/iasm v0.9.0/go.mod h1:Xjy2NpN3h7aUqeqM+woSuuvxmIe6+DDsiNLIrkAmYog=
 github.com/cloudquery/arrow/go/v14 v14.0.0-20231029080147-50d3871d0804 h1:y4EwAGtbzVFz1w9sXggukJmYMVO12925D12Bak7s0vI=
 github.com/cloudquery/arrow/go/v14 v14.0.0-20231029080147-50d3871d0804/go.mod h1:TqWp9yvMb9yZSxFNiij6cmZefm+1jw3oZU0L0w9lT7E=
-github.com/cloudquery/cloudquery-api-go v1.4.3 h1:G8JZiwwHDnoNrRhBVRfXve/DGsx1kJLC4+1ggQLz29I=
-github.com/cloudquery/cloudquery-api-go v1.4.3/go.mod h1:03fojQg0UpdgqXZ9tzZ5gF5CPad/F0sok66bsX6u4RA=
+github.com/cloudquery/cloudquery-api-go v1.4.4 h1:9VQoRxjWi9/rj1esfL3n6o6OwotoGtCNUAs3onnVwho=
+github.com/cloudquery/cloudquery-api-go v1.4.4/go.mod h1:03fojQg0UpdgqXZ9tzZ5gF5CPad/F0sok66bsX6u4RA=
 github.com/coreos/go-systemd/v22 v22.5.0/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc=
 github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
@@ -164,8 +164,8 @@ github.com/mattn/go-isatty v0.0.19/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D
 github.com/mattn/go-runewidth v0.0.14/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh153qUoGf23w=
 github.com/mattn/go-runewidth v0.0.15 h1:UNAjwbU9l54TA3KzvqLGxwWjHmMgBUVhBiTjelZgg3U=
 github.com/mattn/go-runewidth v0.0.15/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh153qUoGf23w=
-github.com/microcosm-cc/bluemonday v1.0.25 h1:4NEwSfiJ+Wva0VxN5B8OwMicaJvD8r9tlJWm9rtloEg=
-github.com/microcosm-cc/bluemonday v1.0.25/go.mod h1:ZIOjCQp1OrzBBPIJmfX4qDYFuhU02nx4bn030ixfHLE=
+github.com/microcosm-cc/bluemonday v1.0.26 h1:xbqSvqzQMeEHCqMi64VAs4d8uy6Mequs3rQ0k/Khz58=
+github.com/microcosm-cc/bluemonday v1.0.26/go.mod h1:JyzOCs9gkyQyjs+6h10UEVSe02CGwkhd72Xdqh78TWs=
 github.com/mitchellh/colorstring v0.0.0-20190213212951-d06e56a500db h1:62I3jR2EmQ4l5rM/4FEfDWcRD+abF5XlKShorW5LRoQ=
 github.com/mitchellh/colorstring v0.0.0-20190213212951-d06e56a500db/go.mod h1:l0dey0ia/Uv7NcFFVbCLtqEBQbrT4OCwCSKTEv6enCw=
 github.com/mitchellh/go-wordwrap v1.0.1 h1:TLuKupo69TCn6TQSyGxwI1EblZZEsQ0vMlAFQflz0v0=

managedplugin/download.go

@@ -3,6 +3,7 @@ package managedplugin
 import (
 	"archive/zip"
 	"context"
+	"crypto/sha256"
 	"errors"
 	"fmt"
 	"io"
@@ -102,53 +103,53 @@ func DownloadPluginFromHub(ctx context.Context, authToken, localPath, team, name
 	}
 
 	target := fmt.Sprintf("%s_%s", runtime.GOOS, runtime.GOARCH)
-	// We don't want to follow redirects because we want to get the download URL and show progress bar while downloading
-	client := &http.Client{
-		CheckRedirect: func(req *http.Request, via []*http.Request) error {
-			return http.ErrUseLastResponse
-		},
-	}
-	c, err := cloudquery_api.NewClient(APIBaseURL(), cloudquery_api.WithRequestEditorFn(func(ctx context.Context, req *http.Request) error {
-		if authToken != "" {
-			req.Header.Set("Authorization", fmt.Sprintf("Bearer %s", authToken))
-		}
-		return nil
-	}))
-	c.Client = client
+	c, err := cloudquery_api.NewClientWithResponses(APIBaseURL(),
+		cloudquery_api.WithRequestEditorFn(func(ctx context.Context, req *http.Request) error {
+			if authToken != "" {
+				req.Header.Set("Authorization", fmt.Sprintf("Bearer %s", authToken))
+			}
+			return nil
+		}))
 	if err != nil {
 		return fmt.Errorf("failed to create Hub API client: %w", err)
 	}
 
-	downloadURL, err := c.DownloadPluginAsset(ctx, team, cloudquery_api.PluginKind(typ.String()), name, version, target)
+	aj := "application/json"
+	resp, err := c.DownloadPluginAssetWithResponse(ctx, team, cloudquery_api.PluginKind(typ.String()), name, version, target, &cloudquery_api.DownloadPluginAssetParams{Accept: &aj})
 	if err != nil {
 		return fmt.Errorf("failed to get plugin url: %w", err)
 	}
-	defer downloadURL.Body.Close()
-	switch downloadURL.StatusCode {
-	case http.StatusOK, http.StatusNoContent, http.StatusFound:
-		// we allow these status codes, but typically expect a redirect (302)
+	switch resp.StatusCode() {
+	case http.StatusOK:
+		// we allow this status code
 	case http.StatusUnauthorized:
 		return fmt.Errorf("unauthorized. Try logging in via `cloudquery login`")
 	case http.StatusNotFound:
 		return fmt.Errorf("failed to download plugin %v %v/%v@%v: plugin version not found. If you're trying to use a private plugin you'll need to run `cloudquery login` first", typ, team, name, version)
 	case http.StatusTooManyRequests:
 		return fmt.Errorf("too many download requests. Try logging in via `cloudquery login` to increase rate limits")
 	default:
-		return fmt.Errorf("failed to download plugin %v %v/%v@%v: unexpected status code %v", typ, team, name, version, downloadURL.StatusCode)
+		return fmt.Errorf("failed to download plugin %v %v/%v@%v: unexpected status code %v", typ, team, name, version, resp.StatusCode())
 	}
-	location, ok := downloadURL.Header["Location"]
-	if !ok {
-		return fmt.Errorf("failed to get plugin url for %v %v/%v@%v: missing location header from response", typ, team, name, version)
+	if resp.JSON200 == nil {
+		return fmt.Errorf("failed to get plugin url for %v %v/%v@%v: missing json response", typ, team, name, version)
 	}
+	location := resp.JSON200.Location
 	if len(location) == 0 {
-		return fmt.Errorf("failed to get plugin url: empty location header from response")
+		return fmt.Errorf("failed to get plugin url: empty location from response")
 	}
 	pluginZipPath := localPath + ".zip"
-	err = downloadFile(ctx, pluginZipPath, location[0])
+	writtenChecksum, err := downloadFile(ctx, pluginZipPath, location)
 	if err != nil {
 		return fmt.Errorf("failed to download plugin: %w", err)
 	}
 
+	if resp.JSON200.Checksum == "" {
+		fmt.Printf("Warning - checksum not verified: %s\n", writtenChecksum)
+	} else if writtenChecksum != resp.JSON200.Checksum {
+		return fmt.Errorf("checksum mismatch: expected %s, got %s", resp.JSON200.Checksum, writtenChecksum)
+	}
+
 	archive, err := zip.OpenReader(pluginZipPath)
 	if err != nil {
 		return fmt.Errorf("failed to open plugin archive: %w", err)
@@ -191,7 +192,7 @@ func DownloadPluginFromGithub(ctx context.Context, localPath string, org string,
 	if err != nil {
 		return fmt.Errorf("failed to get plugin url: %w", err)
 	}
-	if err := downloadFile(ctx, pluginZipPath, downloadURL); err != nil {
+	if _, err := downloadFile(ctx, pluginZipPath, downloadURL); err != nil {
 		return fmt.Errorf("failed to download plugin: %w", err)
 	}
 
@@ -239,23 +240,21 @@ func DownloadPluginFromGithub(ctx context.Context, localPath string, org string,
 	return nil
 }
 
-func downloadFile(ctx context.Context, localPath string, downloadURL string) error {
+func downloadFile(ctx context.Context, localPath string, downloadURL string) (string, error) {
 	// Create the file
 	out, err := os.Create(localPath)
 	if err != nil {
-		return fmt.Errorf("failed to create file %s: %w", localPath, err)
+		return "", fmt.Errorf("failed to create file %s: %w", localPath, err)
 	}
 	defer out.Close()
 
-	err = downloadFileFromURL(ctx, out, downloadURL)
-	if err != nil {
-		return err
-	}
-	return nil
+	return downloadFileFromURL(ctx, out, downloadURL)
 }
 
-func downloadFileFromURL(ctx context.Context, out *os.File, downloadURL string) error {
+func downloadFileFromURL(ctx context.Context, out *os.File, downloadURL string) (string, error) {
+	checksum := ""
 	err := retry.Do(func() error {
+		checksum = ""
 		// Get the data
 		req, err := http.NewRequestWithContext(ctx, http.MethodGet, downloadURL, nil)
 		if err != nil {
@@ -286,11 +285,13 @@ func downloadFileFromURL(ctx context.Context, out *os.File, downloadURL string)
 		fmt.Printf("Downloading %s\n", urlForLog)
 		bar := downloadProgressBar(resp.ContentLength, "Downloading")
 
+		s := sha256.New()
 		// Writer the body to file
-		_, err = io.Copy(io.MultiWriter(out, bar), resp.Body)
+		_, err = io.Copy(io.MultiWriter(out, bar, s), resp.Body)
 		if err != nil {
 			return fmt.Errorf("failed to copy body to file %s: %w", out.Name(), err)
 		}
+		checksum = fmt.Sprintf("%x", s.Sum(nil))
 		return nil
 	}, retry.RetryIf(func(err error) bool {
 		return err.Error() == "statusCode != 200"
@@ -301,12 +302,12 @@ func downloadFileFromURL(ctx context.Context, out *os.File, downloadURL string)
 	if err != nil {
 		for _, e := range err.(retry.Error) {
 			if e.Error() == "not found" {
-				return e
+				return "", e
 			}
 		}
-		return fmt.Errorf("failed downloading URL %q. Error %w", downloadURL, err)
+		return "", fmt.Errorf("failed downloading URL %q. Error %w", downloadURL, err)
 	}
-	return nil
+	return checksum, nil
 }
 
 func downloadProgressBar(maxBytes int64, description ...string) *progressbar.ProgressBar {