Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

feat: Allow plugins to be downloaded using a user team_name context #162

Merged
merged 4 commits into from
Nov 14, 2023
Merged

feat: Allow plugins to be downloaded using a user team_name context #162

Show file tree
Hide file tree
Changes from 1 commit
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
d7ef714
feat: Allow plugins to be downloaded using a user `team_name` context
mnorbury Nov 13, 2023
2eeea5e
Addressing comments
mnorbury Nov 13, 2023
9fad7ba
Further comment changes
mnorbury Nov 13, 2023
2bdb221
Refactor to `HubDownloadOptions`
mnorbury Nov 13, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion go.mod
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,7 @@ go 1.21.1
require (
github.com/apache/arrow/go/v14 v14.0.0-20231031200323-c49e24273160
github.com/avast/retry-go/v4 v4.5.0
github.com/cloudquery/cloudquery-api-go v1.4.4
github.com/cloudquery/cloudquery-api-go v1.4.5
github.com/docker/docker v24.0.7+incompatible
github.com/docker/go-connections v0.4.0
github.com/ghodss/yaml v1.0.0
Expand Down
4 changes: 2 additions & 2 deletions go.sum
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -38,8 +38,8 @@ github.com/chenzhuoyu/iasm v0.9.0 h1:9fhXjVzq5hUy2gkhhgHl95zG2cEAhw9OSGs8toWWAwo
github.com/chenzhuoyu/iasm v0.9.0/go.mod h1:Xjy2NpN3h7aUqeqM+woSuuvxmIe6+DDsiNLIrkAmYog=
github.com/cloudquery/arrow/go/v14 v14.0.0-20231029080147-50d3871d0804 h1:y4EwAGtbzVFz1w9sXggukJmYMVO12925D12Bak7s0vI=
github.com/cloudquery/arrow/go/v14 v14.0.0-20231029080147-50d3871d0804/go.mod h1:TqWp9yvMb9yZSxFNiij6cmZefm+1jw3oZU0L0w9lT7E=
github.com/cloudquery/cloudquery-api-go v1.4.4 h1:9VQoRxjWi9/rj1esfL3n6o6OwotoGtCNUAs3onnVwho=
github.com/cloudquery/cloudquery-api-go v1.4.4/go.mod h1:03fojQg0UpdgqXZ9tzZ5gF5CPad/F0sok66bsX6u4RA=
github.com/cloudquery/cloudquery-api-go v1.4.5 h1:nZyHuzodDSZJLjNi12raFpX1ErGACLBehBFN6LZ+d0k=
github.com/cloudquery/cloudquery-api-go v1.4.5/go.mod h1:03fojQg0UpdgqXZ9tzZ5gF5CPad/F0sok66bsX6u4RA=
github.com/coreos/go-systemd/v22 v22.5.0/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc=
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
Expand Down
71 changes: 47 additions & 24 deletions managedplugin/download.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -92,7 +92,7 @@ func getURLLocation(ctx context.Context, org string, name string, version string
return "", fmt.Errorf("failed to find plugin %s/%s version %s", org, name, version)
}

func DownloadPluginFromHub(ctx context.Context, authToken, localPath, team, name, version string, typ PluginType) error {
func DownloadPluginFromHub(ctx context.Context, authToken, teamName, localPath, pluginTeam, name, version string, typ PluginType) error {
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I wonder if it's time to use a struct here so it's not so easy to get the order of all the string parameters wrong

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I would love that but how far to take it? This pattern is everywhere and technically the methods are public.

Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Technically public yes, but we're already breaking the signature with this change by adding a parameter, right? May as well make it a struct

Maybe something like func DownloadPluginFromHub(ctx context.Context, opts HubDownloadOptions) with

type HubDownloadOptions struct {
    AuthToken string
    TeamName string
    LocalPath string
    PluginTeam string
    PluginKind string
    PluginName string
    PluginVersion string
}

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Good suggestion - added in 2bdb221

downloadDir := filepath.Dir(localPath)
if _, err := os.Stat(localPath); err == nil {
return nil
Expand All @@ -102,39 +102,26 @@ func DownloadPluginFromHub(ctx context.Context, authToken, localPath, team, name
return fmt.Errorf("failed to create plugin directory %s: %w", downloadDir, err)
}

target := fmt.Sprintf("%s_%s", runtime.GOOS, runtime.GOARCH)
c, err := cloudquery_api.NewClientWithResponses(APIBaseURL(),
cloudquery_api.WithRequestEditorFn(func(ctx context.Context, req *http.Request) error {
if authToken != "" {
req.Header.Set("Authorization", fmt.Sprintf("Bearer %s", authToken))
}
return nil
}))
if err != nil {
return fmt.Errorf("failed to create Hub API client: %w", err)
}

aj := "application/json"
resp, err := c.DownloadPluginAssetWithResponse(ctx, team, cloudquery_api.PluginKind(typ.String()), name, version, target, &cloudquery_api.DownloadPluginAssetParams{Accept: &aj})
pluginAsset, statusCode, err := downloadPluginAssetFromHub(ctx, authToken, teamName, pluginTeam, name, version, typ)
if err != nil {
return fmt.Errorf("failed to get plugin url: %w", err)
}
switch resp.StatusCode() {
switch statusCode {
case http.StatusOK:
// we allow this status code
case http.StatusUnauthorized:
return fmt.Errorf("unauthorized. Try logging in via `cloudquery login`")
case http.StatusNotFound:
return fmt.Errorf("failed to download plugin %v %v/%v@%v: plugin version not found. If you're trying to use a private plugin you'll need to run `cloudquery login` first", typ, team, name, version)
return fmt.Errorf("failed to download plugin %v %v/%v@%v: plugin version not found. If you're trying to use a private plugin you'll need to run `cloudquery login` first", typ, pluginTeam, name, version)
case http.StatusTooManyRequests:
return fmt.Errorf("too many download requests. Try logging in via `cloudquery login` to increase rate limits")
default:
return fmt.Errorf("failed to download plugin %v %v/%v@%v: unexpected status code %v", typ, team, name, version, resp.StatusCode())
return fmt.Errorf("failed to download plugin %v %v/%v@%v: unexpected status code %v", typ, pluginTeam, name, version, statusCode)
}
if resp.JSON200 == nil {
return fmt.Errorf("failed to get plugin url for %v %v/%v@%v: missing json response", typ, team, name, version)
if pluginAsset == nil {
return fmt.Errorf("failed to get plugin url for %v %v/%v@%v: missing json response", typ, pluginTeam, name, version)
}
location := resp.JSON200.Location
location := pluginAsset.Location
if len(location) == 0 {
return fmt.Errorf("failed to get plugin url: empty location from response")
}
Expand All @@ -144,10 +131,10 @@ func DownloadPluginFromHub(ctx context.Context, authToken, localPath, team, name
return fmt.Errorf("failed to download plugin: %w", err)
}

if resp.JSON200.Checksum == "" {
if pluginAsset.Checksum == "" {
fmt.Printf("Warning - checksum not verified: %s\n", writtenChecksum)
} else if writtenChecksum != resp.JSON200.Checksum {
return fmt.Errorf("checksum mismatch: expected %s, got %s", resp.JSON200.Checksum, writtenChecksum)
} else if writtenChecksum != pluginAsset.Checksum {
return fmt.Errorf("checksum mismatch: expected %s, got %s", pluginAsset.Checksum, writtenChecksum)
}

archive, err := zip.OpenReader(pluginZipPath)
Expand Down Expand Up @@ -176,6 +163,42 @@ func DownloadPluginFromHub(ctx context.Context, authToken, localPath, team, name
return nil
}

func downloadPluginAssetFromHub(ctx context.Context, authToken, teamName, pluginTeam, name, version string, typ PluginType) (*cloudquery_api.PluginAsset, int, error) {
var pluginAsset *cloudquery_api.PluginAsset
var statusCode int

c, err := cloudquery_api.NewClientWithResponses(APIBaseURL(),
cloudquery_api.WithRequestEditorFn(func(ctx context.Context, req *http.Request) error {
if authToken != "" {
req.Header.Set("Authorization", fmt.Sprintf("Bearer %s", authToken))
}
return nil
}))
if err != nil {
return pluginAsset, statusCode, fmt.Errorf("failed to create Hub API client: %w", err)
}

target := fmt.Sprintf("%s_%s", runtime.GOOS, runtime.GOARCH)

aj := "application/json"

if teamName != "" {
resp, err := c.DownloadPluginAssetByTeamWithResponse(ctx, teamName, pluginTeam, cloudquery_api.PluginKind(typ.String()), name, version, target, &cloudquery_api.DownloadPluginAssetByTeamParams{Accept: &aj})
if err != nil {
return pluginAsset, statusCode, fmt.Errorf("failed to get plugin url: %w", err)
disq marked this conversation as resolved.
Show resolved Hide resolved
}
pluginAsset, statusCode = resp.JSON200, resp.StatusCode()
} else {
resp, err := c.DownloadPluginAssetWithResponse(ctx, pluginTeam, cloudquery_api.PluginKind(typ.String()), name, version, target, &cloudquery_api.DownloadPluginAssetParams{Accept: &aj})
if err != nil {
return pluginAsset, statusCode, fmt.Errorf("failed to get plugin url: %w", err)
}
pluginAsset, statusCode = resp.JSON200, resp.StatusCode()
}

return pluginAsset, statusCode, nil
disq marked this conversation as resolved.
Show resolved Hide resolved
}

func DownloadPluginFromGithub(ctx context.Context, localPath string, org string, name string, version string, typ PluginType) error {
downloadDir := filepath.Dir(localPath)
pluginZipPath := localPath + ".zip"
Expand Down
2 changes: 1 addition & 1 deletion managedplugin/download_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -48,7 +48,7 @@ func TestDownloadPluginFromCloudQueryHub(t *testing.T) {
}
for _, tc := range cases {
t.Run(tc.testName, func(t *testing.T) {
err := DownloadPluginFromHub(context.Background(), "", path.Join(tmp, tc.testName), tc.team, tc.plugin, tc.version, tc.typ)
err := DownloadPluginFromHub(context.Background(), "", "", path.Join(tmp, tc.testName), tc.team, tc.plugin, tc.version, tc.typ)
if (err != nil) != tc.wantErr {
t.Errorf("TestDownloadPluginFromCloudQueryIntegration() error = %v, wantErr %v", err, tc.wantErr)
return
Expand Down
6 changes: 6 additions & 0 deletions managedplugin/options.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -45,3 +45,9 @@ func WithAuthToken(authToken string) Option {
c.authToken = authToken
}
}

func WithTeamName(teamName string) Option {
return func(c *Client) {
c.teamName = teamName
}
}
3 changes: 2 additions & 1 deletion managedplugin/plugin.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -91,6 +91,7 @@ type Client struct {
metrics *Metrics
registry Registry
authToken string
teamName string
}

// typ will be deprecated soon but now required for a transition period
Expand Down Expand Up @@ -182,7 +183,7 @@ func (c *Client) downloadPlugin(ctx context.Context, typ PluginType) error {
org, name := pathSplit[0], pathSplit[1]
c.LocalPath = filepath.Join(c.directory, "plugins", typ.String(), org, name, c.config.Version, "plugin")
c.LocalPath = WithBinarySuffix(c.LocalPath)
return DownloadPluginFromHub(ctx, c.authToken, c.LocalPath, org, name, c.config.Version, typ)
return DownloadPluginFromHub(ctx, c.authToken, c.teamName, c.LocalPath, org, name, c.config.Version, typ)
default:
return fmt.Errorf("unknown registry %s", c.config.Registry.String())
}
Expand Down
Loading