feat(cross-account): support filter eni which cross account

pkg/daemon/daemon_test.go

@@ -379,16 +379,15 @@ func (v *APIMockDB) GetENI(mac string) (res *types.ENI, err error) {
 	return
 }
 
-func (v *APIMockDB) GetAttachedENIs(withTrunk bool) (total int, eniList []*types.ENI, err error) {
+func (v *APIMockDB) GetAttachedENIs(withTrunk bool) (eniList []*types.ENI, err error) {
 	if !v.readOnlyRateLimiter.TryAccept() {
-		return 0, nil, errors.New(apiErr.AccountFlowLimitExceeded)
+		return nil, errors.New(apiErr.AccountFlowLimitExceeded)
 	}
 	v.eniCache.Range(func(key, value any) bool {
 		eni := value.(*ENIKeeper)
 		if eni.Trunk && !withTrunk {
 			return true
 		}
-		total++
 		if eni.celloCreated {
 			eniList = append(eniList, &eni.ENI)
 		}
@@ -397,6 +396,18 @@ func (v *APIMockDB) GetAttachedENIs(withTrunk bool) (total int, eniList []*types
 	return
 }
 
+func (v *APIMockDB) GetTotalAttachedEniCnt() (int, error) {
+	if !v.readOnlyRateLimiter.TryAccept() {
+		return 0, errors.New(apiErr.AccountFlowLimitExceeded)
+	}
+	total := 0
+	v.eniCache.Range(func(key, value any) bool {
+		total++
+		return true
+	})
+	return total, nil
+}
+
 func (v *APIMockDB) GetSecondaryENIMACs() ([]string, error) {
 	var result []string
 	v.eniCache.Range(func(key, value any) bool {

pkg/daemon/eni.go

@@ -124,7 +124,7 @@ func newEniResourceManager(cfg *config.Config, subnet helper.SubnetManager, secM
 		return nil, err
 	}
 	poolConfig := generateENIPoolCfg(cfg, limit.GetLimit())
-	_, created, err := volcApi.GetAttachedENIs(false)
+	created, err := volcApi.GetAttachedENIs(false)
 	if err != nil {
 		return nil, fmt.Errorf("get attached enis failed while init, %v", err)
 	}
@@ -265,7 +265,7 @@ func (f *eniFactory) Valid(resource types.NetResource) error {
 
 // List lists all NetResources.
 func (f *eniFactory) List() (map[types.ResStatus]map[string]types.NetResource, error) {
-	_, enis, err := f.volcApi.GetAttachedENIs(false)
+	enis, err := f.volcApi.GetAttachedENIs(false)
 	if err != nil {
 		return nil, err
 	}

pkg/daemon/eni_multi_ip.go

@@ -136,7 +136,7 @@ func newEniIPResourceManager(cfg *config.Config, subnet helper.SubnetManager, se
 	m := &eniIPResourceManager{}
 	poolConfig := generateIPPoolCfg(cfg, limit.GetLimit())
 
-	_, created, err := volcApi.GetAttachedENIs(false)
+	created, err := volcApi.GetAttachedENIs(false)
 	if err != nil {
 		return nil, fmt.Errorf("get attached enis failed while init, %v", err)
 	}
@@ -1050,7 +1050,7 @@ func (f *eniIPFactory) GC() error {
 		eniId string
 		ip    types.IPSet
 	}{}
-	_, enis, err := f.volcApi.GetAttachedENIs(false)
+	enis, err := f.volcApi.GetAttachedENIs(false)
 	if err != nil {
 		return fmt.Errorf("get attachedENI failed, %v", err)
 	}

pkg/provider/volcengine/cellohelper/eni_tag.go

@@ -56,3 +56,25 @@ func BuildFilterForDescribeNetworkInterfacesInput(tags map[string]string) []*vpc
 	}
 	return tagsInput
 }
+
+// ConvertTagForDescribeNetworkInterfacesOutput convert list of vpc.TagForDescribeNetworkInterfacesOutput to map[string]string
+func ConvertTagForDescribeNetworkInterfacesOutput(output []*vpc.TagForDescribeNetworkInterfacesOutput) map[string]string {
+	tags := map[string]string{}
+	for _, item := range output {
+		if item == nil {
+			continue
+		}
+		tags[volcengine.StringValue(item.Key)] = volcengine.StringValue(item.Value)
+	}
+	return tags
+}
+
+// AssertTag assert actual tags match the expected tags
+func AssertTag(expected, actual map[string]string) bool {
+	for k, v := range expected {
+		if value, exist := actual[k]; !exist || value != v {
+			return false
+		}
+	}
+	return true
+}

pkg/provider/volcengine/cellohelper/instance.go

@@ -134,7 +134,11 @@ func (m *defaultInstanceLimit) update() error {
 			fmt.Sprintf("ECS instance quota updated from %v to %v", oldLimit, *limit))
 	}
 
-	total, created, err := m.api.GetAttachedENIs(true)
+	created, err := m.api.GetAttachedENIs(true)
+	if err != nil {
+		return err
+	}
+	total, err := m.api.GetTotalAttachedEniCnt()
 	if err != nil {
 		return err
 	}

pkg/provider/volcengine/cellohelper/volc_api.go

@@ -25,8 +25,6 @@ import (
 	"time"
 
 	"github.com/containernetworking/plugins/pkg/ip"
-	"github.com/vishvananda/netlink"
-	"golang.org/x/sys/unix"
 	v1 "k8s.io/api/core/v1"
 	k8sErr "k8s.io/apimachinery/pkg/util/errors"
 	"k8s.io/apimachinery/pkg/util/wait"
@@ -71,8 +69,8 @@ type VolcAPI interface {
 	// GetENI get eni by mac from metadata
 	GetENI(mac string) (*types.ENI, error)
 
-	// GetAttachedENIs return counts of attached enis except rdma eni and list of eni created by cello
-	GetAttachedENIs(withTrunk bool) (int, []*types.ENI, error)
+	// GetAttachedENIs return all attached eni created by cello
+	GetAttachedENIs(withTrunk bool) ([]*types.ENI, error)
 
 	GetSecondaryENIMACs() ([]string, error)
 
@@ -87,6 +85,9 @@ type VolcAPI interface {
 
 	// GetInstanceLimit return instance InstanceLimits
 	GetInstanceLimit() (*InstanceLimits, error)
+
+	// GetTotalAttachedEniCnt return count of all eni attached to instance, even across accounts
+	GetTotalAttachedEniCnt() (int, error)
 }
 
 type VolcApiImpl struct {
@@ -359,127 +360,75 @@ func (e *VolcApiImpl) FreeENI(eniID string) error {
 	return e.freeENI(eniID, 2*time.Second)
 }
 
-// filterRdmaWithMac filter out rdma eni.
-func (e *VolcApiImpl) filterRdmaWithMac(macs []string) ([]string, error) {
-	macsMap := make(map[string]struct{})
-	for _, m := range macs {
-		macsMap[m] = struct{}{}
-	}
-	var resp *ecs.DescribeInstancesOutput
-	var err error
-	werr := wait.ExponentialBackoff(backoff.BackOff(backoff.APIFastRetry), func() (bool, error) {
-		resp, err = e.ec2Client.DescribeInstances(&ecs.DescribeInstancesInput{
-			InstanceIds: volcengine.StringSlice([]string{e.GetInstanceId()}),
-		})
-		if err != nil {
-			log.Warnf("DescribeInstances %s failed, %s", e.GetInstanceId(), err.Error())
-			return false, nil
-		}
-		if len(resp.Instances) != 1 ||
-			volcengine.StringValue(resp.Instances[0].InstanceId) != e.GetInstanceId() {
-			return false, fmt.Errorf("DescribeInstances %s failed, %s",
-				e.GetInstanceId(), err.Error())
-		}
-		return true, nil
-	})
-	if err = apiErr.BackoffErrWrapper(werr, err); err != nil {
-		return nil, err
-	}
-
-	rdmaIPAddrs := resp.Instances[0].RdmaIpAddresses
-	var rdmaIPAddrsMap = make(map[string]struct{})
-	for _, ri := range rdmaIPAddrs {
-		rdmaIPAddrsMap[volcengine.StringValue(ri)] = struct{}{}
-	}
-
-	links, err := netlink.LinkList()
+// GetAttachedENIs return all attached eni created by cello
+func (e *VolcApiImpl) GetAttachedENIs(withTrunk bool) (result []*types.ENI, err error) {
+	enis, err := e.getNetworkInterfacesByDescribe(ENIStatusInuse, "", nil, BuildFilterForDescribeNetworkInterfacesInput(e.tags))
 	if err != nil {
-		return nil, fmt.Errorf("list links failed: %s", err)
-	}
-	// filter
-	for _, link := range links {
-		addrs, err := netlink.AddrList(link, unix.AF_INET)
-		if err != nil {
-			return nil, fmt.Errorf("list addrs for %s failed: %s", link.Attrs().Name, err.Error())
-		}
-		for _, addr := range addrs {
-			if _, ok := rdmaIPAddrsMap[addr.IP.String()]; ok {
-				log.Debugf("Found RDMA Card[name:%s ip:%s mac:%s], skipped it.", link.Attrs().Name, addr.IP.String(), link.Attrs().HardwareAddr.String())
-				delete(macsMap, link.Attrs().HardwareAddr.String())
-				break
-			}
-		}
+		return nil, fmt.Errorf("filter eni by tags failed, %v", err)
 	}
 
-	var macRes []string
-	for k := range macsMap {
-		macRes = append(macRes, k)
+	var macs []string
+	celloCreatedEni := map[string]*ec2.NetworkInterfaceSetForDescribeNetworkInterfacesOutput{}
+	for _, eni := range enis {
+		celloCreatedEni[volcengine.StringValue(eni.NetworkInterfaceId)] = eni
+		macs = append(macs, volcengine.StringValue(eni.MacAddress))
 	}
-	return macRes, nil
-}
+	log.Infof("Attached eni created by cello(withTrunk: %v): %v", withTrunk, macs)
 
-func (e *VolcApiImpl) GetAttachedENIs(withTrunk bool) (int, []*types.ENI, error) {
-	ctx := context.Background()
-	macs, err := e.metadataSvc.GetENIsMacs(ctx)
-	if err != nil {
-		return 0, nil, fmt.Errorf("%s", err.Error())
-	}
-	macs, err = e.filterRdmaWithMac(macs)
-	if err != nil {
-		return 0, nil, fmt.Errorf("filterRdmaWithMac failed: %s", err.Error())
-	}
-	total := len(macs)
-	var eniList []*types.ENI
-	var result []*types.ENI
-	var eniIds []string
 	for _, mac := range macs {
-		eni, err := e.GetENI(mac)
-		if err != nil {
-			return 0, nil, fmt.Errorf("%s", err.Error())
+		eni, inErr := e.GetENI(mac)
+		if inErr != nil {
+			return nil, inErr
 		}
-		eniList = append(eniList, eni)
-		eniIds = append(eniIds, eni.ID)
-	}
-
-	if len(eniIds) == 0 {
-		return 0, result, nil
-	}
-
-	enis, err := e.getNetworkInterfacesByDescribe(ENIStatusInuse, "", eniIds, BuildFilterForDescribeNetworkInterfacesInput(e.tags))
-	if err != nil {
-		return 0, nil, err
-	}
-	enisMap := map[string]*ec2.NetworkInterfaceSetForDescribeNetworkInterfacesOutput{}
-	for _, eni := range enis {
-		enisMap[volcengine.StringValue(eni.NetworkInterfaceId)] = eni
-	}
-	for _, eni := range eniList {
-		if i, exist := enisMap[eni.ID]; exist {
-			eni.Trunk = volcengine.StringValue(i.Type) == ENITypeTrunk
+		if item, exist := celloCreatedEni[eni.ID]; exist {
+			eni.Trunk = volcengine.StringValue(item.Type) == ENITypeTrunk
 			if eni.Trunk && !withTrunk {
 				continue
 			}
-			if e.ipFamily.EnableIPv6() && len(i.IPv6Sets) > 0 {
-				eni.PrimaryIP.IPv6 = net.ParseIP(volcengine.StringValue(i.IPv6Sets[0]))
+			if e.ipFamily.EnableIPv6() && len(item.IPv6Sets) > 0 {
+				eni.PrimaryIP.IPv6 = net.ParseIP(volcengine.StringValue(item.IPv6Sets[0]))
 			}
 			result = append(result, eni)
 		}
 	}
-	return total, result, nil
+	return
+}
+
+// GetTotalAttachedEniCnt return count of all eni attached to instance, even across accounts
+// contains primary、secondary、trunk
+func (e *VolcApiImpl) GetTotalAttachedEniCnt() (int, error) {
+	var inErr error
+	var output *ecs.DescribeInstancesOutput
+	err := wait.ExponentialBackoff(backoff.BackOff(backoff.APIFastRetry), func() (bool, error) {
+		output, inErr = e.ec2Client.DescribeInstances(&ecs.DescribeInstancesInput{
+			VpcId:       volcengine.String(e.GetVpcId()),
+			InstanceIds: []*string{volcengine.String(e.GetInstanceId())},
+		})
+		if inErr != nil {
+			return false, nil
+		}
+		return true, nil
+	})
+	if err = apiErr.BackoffErrWrapper(err, inErr); err != nil {
+		return 0, fmt.Errorf("desribe instance failed, %v, %v", err, inErr)
+	}
+	if output == nil || len(output.Instances) != 1 {
+		return 0, fmt.Errorf("desribe instance failed, no result")
+	}
+	return len(output.Instances[0].NetworkInterfaces), nil
 }
 
 func (e *VolcApiImpl) GetSecondaryENIMACs() ([]string, error) {
+	// In some subsequent scenarios, such as rdma and cross-accounts,
+	// the full mac obtained from the metadata service also includes non-secondary network interfaces,
+	// so openapi can only be used instead.
 	var result []string
-	primaryMac := e.GetPrimaryENIMac()
-	macs, err := e.metadataSvc.GetENIsMacs(context.Background())
+	enis, err := e.getNetworkInterfacesByDescribe(ENIStatusInuse, ENITypeSecondary, nil, BuildFilterForDescribeNetworkInterfacesInput(e.tags))
 	if err != nil {
-		return nil, err
+		return nil, fmt.Errorf("filter eni by tags failed, %v", err)
 	}
-	for _, mac := range macs {
-		if mac == primaryMac {
-			continue
-		}
-		result = append(result, mac)
+	for _, eni := range enis {
+		result = append(result, volcengine.StringValue(eni.MacAddress))
 	}
 	return result, nil
 }
@@ -1040,6 +989,6 @@ func New(apiClient ec2.EC2, ipStack types.IPFamily, subnetMgr SubnetManager, ins
 
 	go wait.Forever(impl.cleanUpLeakedENIs, time.Hour)
 
-	log.Infof("ECS and VPC client created")
+	log.Infof("VolcApiImpl created")
 	return impl, nil
 }

pkg/provider/volcengine/metadata/ec2metadawrapper.go

@@ -77,6 +77,8 @@ func (meta EC2MetadataWrapper) GetAvailabilityZone(ctx context.Context) (string,
 }
 
 // GetENIsMacs get all macs of ENIs which attached the instance from metadata.
+// NOTICE: this will get all interfaces macs include rdma which unable to get any information,
+// and other interfaces not created by cello, even cross-account interfaces
 func (meta EC2MetadataWrapper) GetENIsMacs(ctx context.Context) ([]string, error) {
 	start := time.Now()
 	data, err := meta.GetMetadata(ctx, enisMacsPath)