diff --git a/go.mod b/go.mod index cd53278..f55a00a 100644 --- a/go.mod +++ b/go.mod @@ -5,7 +5,7 @@ go 1.21.1 require ( github.com/apache/arrow/go/v14 v14.0.0-20231031200323-c49e24273160 github.com/avast/retry-go/v4 v4.5.0 - github.com/cloudquery/cloudquery-api-go v1.4.3 + github.com/cloudquery/cloudquery-api-go v1.4.4 github.com/docker/docker v24.0.7+incompatible github.com/docker/go-connections v0.4.0 github.com/ghodss/yaml v1.0.0 @@ -74,7 +74,7 @@ require ( github.com/mattn/go-colorable v0.1.13 // indirect github.com/mattn/go-isatty v0.0.19 // indirect github.com/mattn/go-runewidth v0.0.15 // indirect - github.com/microcosm-cc/bluemonday v1.0.25 // indirect + github.com/microcosm-cc/bluemonday v1.0.26 // indirect github.com/mitchellh/colorstring v0.0.0-20190213212951-d06e56a500db // indirect github.com/moby/term v0.5.0 // indirect github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect diff --git a/go.sum b/go.sum index 2479ee0..2484290 100644 --- a/go.sum +++ b/go.sum @@ -38,8 +38,8 @@ github.com/chenzhuoyu/iasm v0.9.0 h1:9fhXjVzq5hUy2gkhhgHl95zG2cEAhw9OSGs8toWWAwo github.com/chenzhuoyu/iasm v0.9.0/go.mod h1:Xjy2NpN3h7aUqeqM+woSuuvxmIe6+DDsiNLIrkAmYog= github.com/cloudquery/arrow/go/v14 v14.0.0-20231029080147-50d3871d0804 h1:y4EwAGtbzVFz1w9sXggukJmYMVO12925D12Bak7s0vI= github.com/cloudquery/arrow/go/v14 v14.0.0-20231029080147-50d3871d0804/go.mod h1:TqWp9yvMb9yZSxFNiij6cmZefm+1jw3oZU0L0w9lT7E= -github.com/cloudquery/cloudquery-api-go v1.4.3 h1:G8JZiwwHDnoNrRhBVRfXve/DGsx1kJLC4+1ggQLz29I= -github.com/cloudquery/cloudquery-api-go v1.4.3/go.mod h1:03fojQg0UpdgqXZ9tzZ5gF5CPad/F0sok66bsX6u4RA= +github.com/cloudquery/cloudquery-api-go v1.4.4 h1:9VQoRxjWi9/rj1esfL3n6o6OwotoGtCNUAs3onnVwho= +github.com/cloudquery/cloudquery-api-go v1.4.4/go.mod h1:03fojQg0UpdgqXZ9tzZ5gF5CPad/F0sok66bsX6u4RA= github.com/coreos/go-systemd/v22 v22.5.0/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= @@ -164,8 +164,8 @@ github.com/mattn/go-isatty v0.0.19/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D github.com/mattn/go-runewidth v0.0.14/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh153qUoGf23w= github.com/mattn/go-runewidth v0.0.15 h1:UNAjwbU9l54TA3KzvqLGxwWjHmMgBUVhBiTjelZgg3U= github.com/mattn/go-runewidth v0.0.15/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh153qUoGf23w= -github.com/microcosm-cc/bluemonday v1.0.25 h1:4NEwSfiJ+Wva0VxN5B8OwMicaJvD8r9tlJWm9rtloEg= -github.com/microcosm-cc/bluemonday v1.0.25/go.mod h1:ZIOjCQp1OrzBBPIJmfX4qDYFuhU02nx4bn030ixfHLE= +github.com/microcosm-cc/bluemonday v1.0.26 h1:xbqSvqzQMeEHCqMi64VAs4d8uy6Mequs3rQ0k/Khz58= +github.com/microcosm-cc/bluemonday v1.0.26/go.mod h1:JyzOCs9gkyQyjs+6h10UEVSe02CGwkhd72Xdqh78TWs= github.com/mitchellh/colorstring v0.0.0-20190213212951-d06e56a500db h1:62I3jR2EmQ4l5rM/4FEfDWcRD+abF5XlKShorW5LRoQ= github.com/mitchellh/colorstring v0.0.0-20190213212951-d06e56a500db/go.mod h1:l0dey0ia/Uv7NcFFVbCLtqEBQbrT4OCwCSKTEv6enCw= github.com/mitchellh/go-wordwrap v1.0.1 h1:TLuKupo69TCn6TQSyGxwI1EblZZEsQ0vMlAFQflz0v0= diff --git a/managedplugin/download.go b/managedplugin/download.go index 8ac1626..56b84cd 100644 --- a/managedplugin/download.go +++ b/managedplugin/download.go @@ -3,6 +3,7 @@ package managedplugin import ( "archive/zip" "context" + "crypto/sha256" "errors" "fmt" "io" @@ -102,31 +103,25 @@ func DownloadPluginFromHub(ctx context.Context, authToken, localPath, team, name } target := fmt.Sprintf("%s_%s", runtime.GOOS, runtime.GOARCH) - // We don't want to follow redirects because we want to get the download URL and show progress bar while downloading - client := &http.Client{ - CheckRedirect: func(req *http.Request, via []*http.Request) error { - return http.ErrUseLastResponse - }, - } - c, err := cloudquery_api.NewClient(APIBaseURL(), cloudquery_api.WithRequestEditorFn(func(ctx context.Context, req *http.Request) error { - if authToken != "" { - req.Header.Set("Authorization", fmt.Sprintf("Bearer %s", authToken)) - } - return nil - })) - c.Client = client + c, err := cloudquery_api.NewClientWithResponses(APIBaseURL(), + cloudquery_api.WithRequestEditorFn(func(ctx context.Context, req *http.Request) error { + if authToken != "" { + req.Header.Set("Authorization", fmt.Sprintf("Bearer %s", authToken)) + } + return nil + })) if err != nil { return fmt.Errorf("failed to create Hub API client: %w", err) } - downloadURL, err := c.DownloadPluginAsset(ctx, team, cloudquery_api.PluginKind(typ.String()), name, version, target) + aj := "application/json" + resp, err := c.DownloadPluginAssetWithResponse(ctx, team, cloudquery_api.PluginKind(typ.String()), name, version, target, &cloudquery_api.DownloadPluginAssetParams{Accept: &aj}) if err != nil { return fmt.Errorf("failed to get plugin url: %w", err) } - defer downloadURL.Body.Close() - switch downloadURL.StatusCode { - case http.StatusOK, http.StatusNoContent, http.StatusFound: - // we allow these status codes, but typically expect a redirect (302) + switch resp.StatusCode() { + case http.StatusOK: + // we allow this status code case http.StatusUnauthorized: return fmt.Errorf("unauthorized. Try logging in via `cloudquery login`") case http.StatusNotFound: @@ -134,21 +129,27 @@ func DownloadPluginFromHub(ctx context.Context, authToken, localPath, team, name case http.StatusTooManyRequests: return fmt.Errorf("too many download requests. Try logging in via `cloudquery login` to increase rate limits") default: - return fmt.Errorf("failed to download plugin %v %v/%v@%v: unexpected status code %v", typ, team, name, version, downloadURL.StatusCode) + return fmt.Errorf("failed to download plugin %v %v/%v@%v: unexpected status code %v", typ, team, name, version, resp.StatusCode()) } - location, ok := downloadURL.Header["Location"] - if !ok { - return fmt.Errorf("failed to get plugin url for %v %v/%v@%v: missing location header from response", typ, team, name, version) + if resp.JSON200 == nil { + return fmt.Errorf("failed to get plugin url for %v %v/%v@%v: missing json response", typ, team, name, version) } + location := resp.JSON200.Location if len(location) == 0 { - return fmt.Errorf("failed to get plugin url: empty location header from response") + return fmt.Errorf("failed to get plugin url: empty location from response") } pluginZipPath := localPath + ".zip" - err = downloadFile(ctx, pluginZipPath, location[0]) + writtenChecksum, err := downloadFile(ctx, pluginZipPath, location) if err != nil { return fmt.Errorf("failed to download plugin: %w", err) } + if resp.JSON200.Checksum == "" { + fmt.Printf("Warning - checksum not verified: %s\n", writtenChecksum) + } else if writtenChecksum != resp.JSON200.Checksum { + return fmt.Errorf("checksum mismatch: expected %s, got %s", resp.JSON200.Checksum, writtenChecksum) + } + archive, err := zip.OpenReader(pluginZipPath) if err != nil { return fmt.Errorf("failed to open plugin archive: %w", err) @@ -191,7 +192,7 @@ func DownloadPluginFromGithub(ctx context.Context, localPath string, org string, if err != nil { return fmt.Errorf("failed to get plugin url: %w", err) } - if err := downloadFile(ctx, pluginZipPath, downloadURL); err != nil { + if _, err := downloadFile(ctx, pluginZipPath, downloadURL); err != nil { return fmt.Errorf("failed to download plugin: %w", err) } @@ -239,23 +240,21 @@ func DownloadPluginFromGithub(ctx context.Context, localPath string, org string, return nil } -func downloadFile(ctx context.Context, localPath string, downloadURL string) error { +func downloadFile(ctx context.Context, localPath string, downloadURL string) (string, error) { // Create the file out, err := os.Create(localPath) if err != nil { - return fmt.Errorf("failed to create file %s: %w", localPath, err) + return "", fmt.Errorf("failed to create file %s: %w", localPath, err) } defer out.Close() - err = downloadFileFromURL(ctx, out, downloadURL) - if err != nil { - return err - } - return nil + return downloadFileFromURL(ctx, out, downloadURL) } -func downloadFileFromURL(ctx context.Context, out *os.File, downloadURL string) error { +func downloadFileFromURL(ctx context.Context, out *os.File, downloadURL string) (string, error) { + checksum := "" err := retry.Do(func() error { + checksum = "" // Get the data req, err := http.NewRequestWithContext(ctx, http.MethodGet, downloadURL, nil) if err != nil { @@ -286,11 +285,13 @@ func downloadFileFromURL(ctx context.Context, out *os.File, downloadURL string) fmt.Printf("Downloading %s\n", urlForLog) bar := downloadProgressBar(resp.ContentLength, "Downloading") + s := sha256.New() // Writer the body to file - _, err = io.Copy(io.MultiWriter(out, bar), resp.Body) + _, err = io.Copy(io.MultiWriter(out, bar, s), resp.Body) if err != nil { return fmt.Errorf("failed to copy body to file %s: %w", out.Name(), err) } + checksum = fmt.Sprintf("%x", s.Sum(nil)) return nil }, retry.RetryIf(func(err error) bool { return err.Error() == "statusCode != 200" @@ -301,12 +302,12 @@ func downloadFileFromURL(ctx context.Context, out *os.File, downloadURL string) if err != nil { for _, e := range err.(retry.Error) { if e.Error() == "not found" { - return e + return "", e } } - return fmt.Errorf("failed downloading URL %q. Error %w", downloadURL, err) + return "", fmt.Errorf("failed downloading URL %q. Error %w", downloadURL, err) } - return nil + return checksum, nil } func downloadProgressBar(maxBytes int64, description ...string) *progressbar.ProgressBar {