Skip to content

Commit

Permalink
fix: Attempt to launch plugin up to 3 times (#395)
Browse files Browse the repository at this point in the history
#### Summary

Fixes cloudquery/cloudquery-issues#2280 (internal issue).
Sometimes plugins fail to launch the first time (we're not sure why, seems to be happening only on MacOS and for large plugins).
Retrying the same binary works, so this PR implements that.

Example logs:

```
2024-08-26T08:48:08Z INF Loading spec(s) args=["examples/aws-postgres.yml"] invocation_id=4356ddcc-f66d-4231-bdcd-90d4e803adb5 module=cli
2024-08-26T08:48:08Z DBG starting plugin attempt=1 invocation_id=4356ddcc-f66d-4231-bdcd-90d4e803adb5 module=cli path=******
2024-08-26T08:48:13Z INF Plugin server listening address=/var/folders/9s/pgf_fjl160j81k95zb5gm7vw0000gn/T/cq-VIAmPPIOcYreNayw.sock invocation_id=4356ddcc-f66d-4231-bdcd-90d4e803adb5 module=cli
2024-08-26T08:48:13Z DBG failed to start plugin, retrying error="failed to run plugin ******: signal: killed" attempt=0 invocation_id=4356ddcc-f66d-4231-bdcd-90d4e803adb5 module=cli
2024-08-26T08:48:14Z DBG starting plugin attempt=2 invocation_id=4356ddcc-f66d-4231-bdcd-90d4e803adb5 module=cli path=******
2024-08-26T08:48:14Z INF Plugin server listening address=/var/folders/9s/pgf_fjl160j81k95zb5gm7vw0000gn/T/cq-AscwEzGdxCtujUjO.sock invocation_id=4356ddcc-f66d-4231-bdcd-90d4e803adb5 module=cli
2024-08-26T08:48:15Z DBG plugin started successfully attempt=2 invocation_id=4356ddcc-f66d-4231-bdcd-90d4e803adb5 module=cli path=******
```

---
  • Loading branch information
erezrokah authored Aug 26, 2024
1 parent e8f6594 commit 3007cab
Showing 1 changed file with 30 additions and 9 deletions.
39 changes: 30 additions & 9 deletions managedplugin/plugin.go
Original file line number Diff line number Diff line change
Expand Up @@ -423,15 +423,36 @@ func getFreeTCPAddr() (string, error) {
}

func (c *Client) startLocal(ctx context.Context, path string) error {
if c.useTCP {
tcpAddr, err := getFreeTCPAddr()
if err != nil {
return fmt.Errorf("failed to get free port: %w", err)
}
c.tcpAddr = tcpAddr
return c.startLocalTCP(ctx, path)
}
return c.startLocalUnixSocket(ctx, path)
attempt := 0
return retry.Do(
func() error {
attempt++
c.logger.Debug().Str("path", path).Int("attempt", attempt).Msg("starting plugin")
var err error
if c.useTCP {
var tcpAddr string
tcpAddr, err = getFreeTCPAddr()
if err != nil {
err = fmt.Errorf("failed to get free port: %w", err)
} else {
c.tcpAddr = tcpAddr
err = c.startLocalTCP(ctx, path)
}
} else {
err = c.startLocalUnixSocket(ctx, path)
}
if err == nil {
c.logger.Debug().Str("path", path).Int("attempt", attempt).Msg("plugin started successfully")
}
return err
},
retry.Attempts(3),
retry.Delay(1*time.Second),
retry.LastErrorOnly(true),
retry.OnRetry(func(n uint, err error) {
c.logger.Debug().Err(err).Int("attempt", int(n)).Msg("failed to start plugin, retrying")
}),
)
}

func (c *Client) startLocalTCP(ctx context.Context, path string) error {
Expand Down

0 comments on commit 3007cab

Please sign in to comment.