diff --git a/app.go b/app.go index 2d92cde..a198df9 100644 --- a/app.go +++ b/app.go @@ -1,6 +1,7 @@ package main import ( + "github.com/pkg/errors" "os" "os/signal" "syscall" @@ -9,20 +10,20 @@ import ( "k8s.io/utils/clock" ) -type Azure interface { - GetUsers() ([]AzureUser, error) - GetGroupsWithMembers() ([]AzureGroupWithMembers, error) +type Source interface { + GetUsers() ([]SourceUser, error) + GetGroupsWithMembers() ([]SourceGroupWithMembers, error) } type App struct { - syncInterval time.Duration + syncInterval *time.Duration usernameReplaces []ReplacementPair groupnameReplaces []ReplacementPair - removeLimit int - banDuration time.Duration + removeLimit *int + banDuration *time.Duration ytsaurus *Ytsaurus - azure Azure + source Source stopCh chan struct{} sigCh chan os.Signal @@ -30,17 +31,32 @@ type App struct { } func NewApp(cfg *Config, logger appLoggerType) (*App, error) { - azure, err := NewAzureReal(cfg.Azure, logger) - if err != nil { - return nil, err + if cfg.Azure == nil && cfg.Ldap == nil { + return nil, errors.New("no source (source or ldap) is specified") + } + + var err error + var source Source + if cfg.Azure != nil { + source, err = NewAzureReal(cfg.Azure, logger) + if err != nil { + return nil, err + } + } + + if cfg.Ldap != nil { + source, err = NewLdap(cfg.Ldap, logger) + if err != nil { + return nil, err + } } - return NewAppCustomized(cfg, logger, azure, clock.RealClock{}) + return NewAppCustomized(cfg, logger, source, clock.RealClock{}) } // NewAppCustomized used in tests. -func NewAppCustomized(cfg *Config, logger appLoggerType, azure Azure, clock clock.PassiveClock) (*App, error) { - yt, err := NewYtsaurus(cfg.Ytsaurus, logger, clock) +func NewAppCustomized(cfg *Config, logger appLoggerType, source Source, clock clock.PassiveClock) (*App, error) { + yt, err := NewYtsaurus(&cfg.Ytsaurus, logger, clock) if err != nil { return nil, err } @@ -56,7 +72,7 @@ func NewAppCustomized(cfg *Config, logger appLoggerType, azure Azure, clock cloc banDuration: cfg.App.BanBeforeRemoveDuration, ytsaurus: yt, - azure: azure, + source: source, stopCh: make(chan struct{}), sigCh: sigCh, @@ -66,8 +82,8 @@ func NewAppCustomized(cfg *Config, logger appLoggerType, azure Azure, clock cloc func (a *App) Start() { a.logger.Info("Starting the application") - if a.syncInterval > 0 { - ticker := time.NewTicker(a.syncInterval) + if a.syncInterval != nil && *a.syncInterval > 0 { + ticker := time.NewTicker(*a.syncInterval) for { select { case <-a.stopCh: @@ -83,7 +99,7 @@ func (a *App) Start() { } } else { a.logger.Info( - "app.sync_interval config variable is not greater than zero, " + + "app.sync_interval config variable is not specified or is not greater than zero, " + "auto sync is disabled. Send SIGUSR1 for manual sync.", ) for { diff --git a/app_integration_test.go b/app_integration_test.go index 63a3835..fc42d2c 100644 --- a/app_integration_test.go +++ b/app_integration_test.go @@ -16,7 +16,7 @@ const ( runLocalYtsaurus = false ) -// TestAppIntegration checks sync with real Azure API and local yt +// TestAppIntegration checks sync with real Source API and local yt // It requires AZURE_CLIENT_SECRET to be set. func TestAppIntegration(t *testing.T) { require.NoError(t, os.Setenv(defaultYtsaurusSecretEnvVar, ytDevToken)) diff --git a/app_test.go b/app_test.go index a97c82f..6937815 100644 --- a/app_test.go +++ b/app_test.go @@ -2,7 +2,11 @@ package main import ( "context" + "fmt" + "github.com/go-ldap/ldap/v3" + "go.ytsaurus.tech/library/go/ptr" "os" + "strings" "testing" "time" @@ -16,6 +20,9 @@ import ( const ( ytDevToken = "password" + aliceName = "alice" + bobName = "bob" + carolName = "carol" ) type testCase struct { @@ -23,556 +30,580 @@ type testCase struct { appConfig *AppConfig testTime time.Time - azureUsersSetUp []AzureUser - ytUsersSetUp []YtsaurusUser - ytUsersExpected []YtsaurusUser + sourceType SourceType - azureGroupsSetUp []AzureGroupWithMembers - ytGroupsSetUp []YtsaurusGroupWithMembers - ytGroupsExpected []YtsaurusGroupWithMembers -} + sourceUsersSetUp []SourceUser + ytUsersSetUp []YtsaurusUser + ytUsersExpected []YtsaurusUser -var ( - testTimeStr = "2023-10-20T12:00:00Z" - initialTestTime = parseAppTime(testTimeStr) + sourceGroupsSetUp []SourceGroupWithMembers + ytGroupsSetUp []YtsaurusGroupWithMembers + ytGroupsExpected []YtsaurusGroupWithMembers +} - aliceAzure = AzureUser{ - PrincipalName: "alice@acme.com", - AzureID: "fake-az-id-alice", - Email: "alice@acme.com", - FirstName: "Alice", - LastName: "Henderson", - DisplayName: "Henderson, Alice (ACME)", - } - bobAzure = AzureUser{ - PrincipalName: "Bob@acme.com", - AzureID: "fake-az-id-bob", - Email: "Bob@acme.com", - FirstName: "Bob", - LastName: "Sanders", - DisplayName: "Sanders, Bob (ACME)", - } - carolAzure = AzureUser{ - PrincipalName: "carol@acme.com", - AzureID: "fake-az-id-carol", - Email: "carol@acme.com", - FirstName: "Carol", - LastName: "Sanders", - DisplayName: "Sanders, Carol (ACME)", - } - aliceAzureChangedLastName = AzureUser{ - PrincipalName: aliceAzure.PrincipalName, - AzureID: aliceAzure.AzureID, - Email: aliceAzure.Email, - FirstName: aliceAzure.FirstName, - LastName: "Smith", - DisplayName: aliceAzure.DisplayName, - } - bobAzureChangedEmail = AzureUser{ - PrincipalName: "bobby@example.com", - AzureID: bobAzure.AzureID, - Email: "bobby@example.com", - FirstName: bobAzure.FirstName, - LastName: bobAzure.LastName, - DisplayName: bobAzure.DisplayName, - } - devsAzureGroup = AzureGroup{ - Identity: "acme.devs|all", - AzureID: "fake-az-acme.devs", - DisplayName: "acme.devs|all", - } - hqAzureGroup = AzureGroup{ - Identity: "acme.hq", - AzureID: "fake-az-acme.hq", - DisplayName: "acme.hq", - } - devsAzureGroupChangedDisplayName = AzureGroup{ - Identity: "acme.developers|all", - AzureID: devsAzureGroup.AzureID, - DisplayName: "acme.developers|all", - } - hqAzureGroupChangedBackwardCompatible = AzureGroup{ - Identity: "acme.hq|all", - AzureID: hqAzureGroup.AzureID, - DisplayName: "acme.hq|all", +func getUserId(name string) string { + switch name { + case aliceName: + return "1" + case bobName: + return "2" + case carolName: + return "3" } + return "4" +} - aliceYtsaurus = YtsaurusUser{ - Username: "alice", - AzureID: aliceAzure.AzureID, - PrincipalName: aliceAzure.PrincipalName, - Email: aliceAzure.Email, - FirstName: aliceAzure.FirstName, - LastName: aliceAzure.LastName, - DisplayName: aliceAzure.DisplayName, - } - bobYtsaurus = YtsaurusUser{ - Username: "bob", - AzureID: bobAzure.AzureID, - PrincipalName: bobAzure.PrincipalName, - Email: bobAzure.Email, - FirstName: bobAzure.FirstName, - LastName: bobAzure.LastName, - DisplayName: bobAzure.DisplayName, - } - carolYtsaurus = YtsaurusUser{ - Username: "carol", - AzureID: carolAzure.AzureID, - PrincipalName: carolAzure.PrincipalName, - Email: carolAzure.Email, - FirstName: carolAzure.FirstName, - LastName: carolAzure.LastName, - DisplayName: carolAzure.DisplayName, - } - aliceYtsaurusChangedLastName = YtsaurusUser{ - Username: aliceYtsaurus.Username, - AzureID: aliceYtsaurus.AzureID, - PrincipalName: aliceYtsaurus.PrincipalName, - Email: aliceYtsaurus.Email, - FirstName: aliceYtsaurus.FirstName, - LastName: aliceAzureChangedLastName.LastName, - DisplayName: aliceYtsaurus.DisplayName, - } - bobYtsaurusChangedEmail = YtsaurusUser{ - Username: "bobby:example.com", - AzureID: bobYtsaurus.AzureID, - PrincipalName: bobAzureChangedEmail.PrincipalName, - Email: bobAzureChangedEmail.Email, - FirstName: bobYtsaurus.FirstName, - LastName: bobYtsaurus.LastName, - DisplayName: bobYtsaurus.DisplayName, - } - bobYtsaurusBanned = YtsaurusUser{ - Username: bobYtsaurus.Username, - AzureID: bobYtsaurus.AzureID, - PrincipalName: bobYtsaurus.PrincipalName, - Email: bobYtsaurus.Email, - FirstName: bobYtsaurus.FirstName, - LastName: bobYtsaurus.LastName, - DisplayName: bobYtsaurus.DisplayName, - BannedSince: initialTestTime, - } - carolYtsaurusBanned = YtsaurusUser{ - Username: carolYtsaurus.Username, - AzureID: carolYtsaurus.AzureID, - PrincipalName: carolYtsaurus.PrincipalName, - Email: carolYtsaurus.Email, - FirstName: carolYtsaurus.FirstName, - LastName: carolYtsaurus.LastName, - DisplayName: carolYtsaurus.DisplayName, - BannedSince: initialTestTime.Add(40 * time.Hour), - } - devsYtsaurusGroup = YtsaurusGroup{ - Name: "acme.devs", - AzureID: devsAzureGroup.AzureID, - DisplayName: "acme.devs|all", +func getSourceUser(name string, sourceType SourceType) SourceUser { + switch sourceType { + case LdapSourceType: + return LdapUser{ + BasicSourceUser: BasicSourceUser{SourceType: LdapSourceType}, + Username: fmt.Sprintf("%v@acme.com", name), + Uid: getUserId(name), + FirstName: fmt.Sprintf("%v@acme.com-firstname", name), + } + case AzureSourceType: + return AzureUser{ + BasicSourceUser: BasicSourceUser{SourceType: AzureSourceType}, + PrincipalName: fmt.Sprintf("%v@acme.com", name), + AzureID: fmt.Sprintf("fake-az-id-%v", name), + Email: fmt.Sprintf("%v@acme.com", name), + FirstName: fmt.Sprintf("%v@acme.com-firstname", name), + LastName: fmt.Sprintf("%v-lastname", name), + DisplayName: fmt.Sprintf("Henderson, %v (ACME)", name), + } } - qaYtsaurusGroup = YtsaurusGroup{ - Name: "acme.qa", - AzureID: "fake-az-acme.qa", - DisplayName: "acme.qa|all", + return nil +} + +func getUpdatedSourceUser(name string, sourceType SourceType) SourceUser { + sourceUser := getSourceUser(name, sourceType) + switch sourceType { + case LdapSourceType: + ldapSourceUser := sourceUser.(LdapUser) + return LdapUser{ + BasicSourceUser: ldapSourceUser.BasicSourceUser, + Username: ldapSourceUser.Username, + Uid: ldapSourceUser.Uid, + FirstName: ldapSourceUser.FirstName + "-updated", + } + case AzureSourceType: + azureSourceUser := sourceUser.(AzureUser) + return AzureUser{ + BasicSourceUser: azureSourceUser.BasicSourceUser, + PrincipalName: azureSourceUser.PrincipalName, + AzureID: azureSourceUser.AzureID, + Email: azureSourceUser.Email + "-updated", + FirstName: azureSourceUser.FirstName, + LastName: azureSourceUser.LastName, + DisplayName: azureSourceUser.DisplayName, + } } - hqYtsaurusGroup = YtsaurusGroup{ - Name: "acme.hq", - AzureID: hqAzureGroup.AzureID, - DisplayName: "acme.hq", + return nil +} + +func getYtsaurusUser(sourceUser SourceUser) YtsaurusUser { + name := sourceUser.GetName() + for _, replacement := range defaultUsernameReplacements { + name = strings.Replace(name, replacement.From, replacement.To, -1) } - devsYtsaurusGroupChangedDisplayName = YtsaurusGroup{ - Name: "acme.developers", - AzureID: devsAzureGroup.AzureID, - DisplayName: "acme.developers|all", + return YtsaurusUser{Username: name, SourceUser: sourceUser} +} + +func bannedYtsaurusUser(ytUser YtsaurusUser, bannedSince time.Time) YtsaurusUser { + return YtsaurusUser{Username: ytUser.Username, SourceUser: ytUser.SourceUser, BannedSince: bannedSince} +} + +func getSourceGroup(name string, sourceType SourceType) SourceGroup { + switch sourceType { + case AzureSourceType: + return AzureGroup{ + BasicSourceGroup: BasicSourceGroup{SourceType: AzureSourceType}, + Identity: fmt.Sprintf("acme.%v|all", name), + AzureID: fmt.Sprintf("fake-az-acme.%v", name), + DisplayName: fmt.Sprintf("acme.%v|all", name), + } + case LdapSourceType: + return LdapGroup{ + BasicSourceGroup: BasicSourceGroup{SourceType: LdapSourceType}, + Groupname: fmt.Sprintf("acme.%v|all", name), + } } - hqYtsaurusGroupChangedBackwardCompatible = YtsaurusGroup{ - Name: "acme.hq", - AzureID: hqAzureGroup.AzureID, - DisplayName: "acme.hq|all", + return nil +} + +func getUpdatedSourceGroup(name string, sourceType SourceType) SourceGroup { + sourceGroup := getSourceGroup(name, sourceType) + switch sourceType { + case LdapSourceType: + // TODO(nadya73): add more fields. + ldapSourceGroup := sourceGroup.(LdapGroup) + return LdapGroup{ + BasicSourceGroup: ldapSourceGroup.BasicSourceGroup, + Groupname: ldapSourceGroup.Groupname, + } + case AzureSourceType: + azureSourceGroup := sourceGroup.(AzureGroup) + return AzureGroup{ + BasicSourceGroup: azureSourceGroup.BasicSourceGroup, + Identity: azureSourceGroup.Identity, + AzureID: azureSourceGroup.AzureID, + DisplayName: azureSourceGroup.DisplayName + "-updated", + } } + return nil +} - defaultUsernameReplacements = []ReplacementPair{ - {"@acme.com", ""}, - {"@", ":"}, +func getChangedBackwardCompatibleSourceGroup(name string, sourceType SourceType) SourceGroup { + if sourceType != AzureSourceType { + return nil } - defaultGroupnameReplacements = []ReplacementPair{ - {"|all", ""}, + sourceGroup := getSourceGroup(name, sourceType) + azureSourceGroup := sourceGroup.(AzureGroup) + return AzureGroup{ + BasicSourceGroup: azureSourceGroup.BasicSourceGroup, + Identity: azureSourceGroup.Identity + "-changed", + AzureID: azureSourceGroup.AzureID, + DisplayName: azureSourceGroup.DisplayName + "-updated", } - defaultAppConfig = &AppConfig{ - UsernameReplacements: defaultUsernameReplacements, - GroupnameReplacements: defaultGroupnameReplacements, +} + +func getYtsaurusGroup(sourceGroup SourceGroup) YtsaurusGroup { + name := sourceGroup.GetName() + for _, replacement := range defaultGroupnameReplacements { + name = strings.Replace(name, replacement.From, replacement.To, -1) } + return YtsaurusGroup{Name: name, SourceGroup: sourceGroup} +} - // we test several things in each test case, because of long wait for local ytsaurus - // container start. - testCases = []testCase{ +// We test several things in each test case, because of long wait for local ytsaurus +// container start. +func getTestCases(sourceType SourceType) []testCase { + testCases := []testCase{ { - name: "a-skip-b-create-c-remove", - azureUsersSetUp: []AzureUser{ - aliceAzure, - bobAzure, + name: "a-skip-b-create-c-remove", + sourceType: sourceType, + sourceUsersSetUp: []SourceUser{ + getSourceUser(aliceName, sourceType), + getSourceUser(bobName, sourceType), }, ytUsersSetUp: []YtsaurusUser{ - aliceYtsaurus, - carolYtsaurus, + getYtsaurusUser(getSourceUser(aliceName, sourceType)), + getYtsaurusUser(getSourceUser(carolName, sourceType)), }, ytUsersExpected: []YtsaurusUser{ - aliceYtsaurus, - bobYtsaurus, + getYtsaurusUser(getSourceUser(aliceName, sourceType)), + getYtsaurusUser(getSourceUser(bobName, sourceType)), }, }, { - name: "bob-is-banned", + name: "bob-is-banned", + sourceType: sourceType, appConfig: &AppConfig{ UsernameReplacements: defaultUsernameReplacements, GroupnameReplacements: defaultGroupnameReplacements, - BanBeforeRemoveDuration: 24 * time.Hour, + BanBeforeRemoveDuration: ptr.Duration(24 * time.Hour), }, - ytUsersSetUp: []YtsaurusUser{ - aliceYtsaurus, - bobYtsaurus, + sourceUsersSetUp: []SourceUser{ + getSourceUser(aliceName, sourceType), }, - azureUsersSetUp: []AzureUser{ - aliceAzure, + ytUsersSetUp: []YtsaurusUser{ + getYtsaurusUser(getSourceUser(aliceName, sourceType)), + getYtsaurusUser(getSourceUser(bobName, sourceType)), }, ytUsersExpected: []YtsaurusUser{ - aliceYtsaurus, - bobYtsaurusBanned, + getYtsaurusUser(getSourceUser(aliceName, sourceType)), + bannedYtsaurusUser(getYtsaurusUser(getSourceUser(bobName, sourceType)), initialTestTime), }, }, { - name: "bob-was-banned-now-deleted-carol-was-banned-now-back", + name: "bob-was-banned-now-deleted-carol-was-banned-now-back", + sourceType: sourceType, // Bob was banned at initialTestTime, // 2 days have passed (more than setting allows) —> he should be removed. - // Carol was banned 8 hours ago and has been found in Azure -> she should be restored. + // Carol was banned 8 hours ago and has been found in Source -> she should be restored. testTime: initialTestTime.Add(48 * time.Hour), appConfig: &AppConfig{ UsernameReplacements: defaultUsernameReplacements, GroupnameReplacements: defaultGroupnameReplacements, - BanBeforeRemoveDuration: 24 * time.Hour, + BanBeforeRemoveDuration: ptr.Duration(24 * time.Hour), }, - ytUsersSetUp: []YtsaurusUser{ - aliceYtsaurus, - bobYtsaurusBanned, - carolYtsaurusBanned, + sourceUsersSetUp: []SourceUser{ + getSourceUser(aliceName, sourceType), + getSourceUser(carolName, sourceType), }, - azureUsersSetUp: []AzureUser{ - aliceAzure, - carolAzure, + ytUsersSetUp: []YtsaurusUser{ + getYtsaurusUser(getSourceUser(aliceName, sourceType)), + bannedYtsaurusUser(getYtsaurusUser(getSourceUser(bobName, sourceType)), initialTestTime), + bannedYtsaurusUser(getYtsaurusUser(getSourceUser(carolName, sourceType)), initialTestTime.Add(40*time.Hour)), }, ytUsersExpected: []YtsaurusUser{ - aliceYtsaurus, - carolYtsaurus, + getYtsaurusUser(getSourceUser(aliceName, sourceType)), + getYtsaurusUser(getSourceUser(carolName, sourceType)), }, }, { - name: "remove-limit-users-3", + name: "remove-limit-users-3", + sourceType: sourceType, appConfig: &AppConfig{ UsernameReplacements: defaultUsernameReplacements, GroupnameReplacements: defaultGroupnameReplacements, - RemoveLimit: 3, + RemoveLimit: ptr.Int(3), }, - azureUsersSetUp: []AzureUser{}, + sourceUsersSetUp: []SourceUser{}, ytUsersSetUp: []YtsaurusUser{ - aliceYtsaurus, - bobYtsaurus, - carolYtsaurus, + getYtsaurusUser(getSourceUser(aliceName, sourceType)), + getYtsaurusUser(getSourceUser(bobName, sourceType)), + getYtsaurusUser(getSourceUser(carolName, sourceType)), }, // no one is deleted: limitation works ytUsersExpected: []YtsaurusUser{ - aliceYtsaurus, - bobYtsaurus, - carolYtsaurus, + getYtsaurusUser(getSourceUser(aliceName, sourceType)), + getYtsaurusUser(getSourceUser(bobName, sourceType)), + getYtsaurusUser(getSourceUser(carolName, sourceType)), }, }, { - name: "remove-limit-groups-3", + name: "remove-limit-groups-3", + sourceType: sourceType, appConfig: &AppConfig{ UsernameReplacements: defaultUsernameReplacements, GroupnameReplacements: defaultGroupnameReplacements, - RemoveLimit: 3, + RemoveLimit: ptr.Int(3), }, - azureGroupsSetUp: []AzureGroupWithMembers{}, + sourceGroupsSetUp: []SourceGroupWithMembers{}, ytGroupsSetUp: []YtsaurusGroupWithMembers{ - NewEmptyYtsaurusGroupWithMembers(devsYtsaurusGroup), - NewEmptyYtsaurusGroupWithMembers(qaYtsaurusGroup), - NewEmptyYtsaurusGroupWithMembers(hqYtsaurusGroup), + NewEmptyYtsaurusGroupWithMembers(getYtsaurusGroup(getSourceGroup("dev", sourceType))), + NewEmptyYtsaurusGroupWithMembers(getYtsaurusGroup(getSourceGroup("qa", sourceType))), + NewEmptyYtsaurusGroupWithMembers(getYtsaurusGroup(getSourceGroup("hq", sourceType))), }, // no group is deleted: limitation works ytGroupsExpected: []YtsaurusGroupWithMembers{ - NewEmptyYtsaurusGroupWithMembers(devsYtsaurusGroup), - NewEmptyYtsaurusGroupWithMembers(qaYtsaurusGroup), - NewEmptyYtsaurusGroupWithMembers(hqYtsaurusGroup), + NewEmptyYtsaurusGroupWithMembers(getYtsaurusGroup(getSourceGroup("dev", sourceType))), + NewEmptyYtsaurusGroupWithMembers(getYtsaurusGroup(getSourceGroup("qa", sourceType))), + NewEmptyYtsaurusGroupWithMembers(getYtsaurusGroup(getSourceGroup("hq", sourceType))), }, }, { - name: "a-changed-name-b-changed-email", - azureUsersSetUp: []AzureUser{ - aliceAzureChangedLastName, - bobAzureChangedEmail, + name: "a-changed-name-b-changed-email", + sourceType: sourceType, + sourceUsersSetUp: []SourceUser{ + getUpdatedSourceUser(aliceName, sourceType), + getUpdatedSourceUser(bobName, sourceType), }, ytUsersSetUp: []YtsaurusUser{ - aliceYtsaurus, - bobYtsaurus, + getYtsaurusUser(getSourceUser(aliceName, sourceType)), + getYtsaurusUser(getSourceUser(bobName, sourceType)), }, ytUsersExpected: []YtsaurusUser{ - aliceYtsaurusChangedLastName, - bobYtsaurusChangedEmail, + getYtsaurusUser(getUpdatedSourceUser(aliceName, sourceType)), + getYtsaurusUser(getUpdatedSourceUser(bobName, sourceType)), }, }, { - name: "skip-create-remove-group-no-members-change-correct-name-replace", - azureUsersSetUp: []AzureUser{ - aliceAzure, - bobAzure, - carolAzure, + name: "skip-create-remove-group-no-members-change-correct-name-replace", + sourceType: sourceType, + sourceUsersSetUp: []SourceUser{ + getSourceUser(aliceName, sourceType), + getSourceUser(bobName, sourceType), + getSourceUser(carolName, sourceType), }, ytUsersSetUp: []YtsaurusUser{ - aliceYtsaurus, - bobYtsaurus, - carolYtsaurus, + getYtsaurusUser(getSourceUser(aliceName, sourceType)), + getYtsaurusUser(getSourceUser(bobName, sourceType)), + getYtsaurusUser(getSourceUser(carolName, sourceType)), }, ytUsersExpected: []YtsaurusUser{ - aliceYtsaurus, - bobYtsaurus, - carolYtsaurus, + getYtsaurusUser(getSourceUser(aliceName, sourceType)), + getYtsaurusUser(getSourceUser(bobName, sourceType)), + getYtsaurusUser(getSourceUser(carolName, sourceType)), }, ytGroupsSetUp: []YtsaurusGroupWithMembers{ { - YtsaurusGroup: devsYtsaurusGroup, - Members: NewStringSetFromItems(aliceYtsaurus.Username), + YtsaurusGroup: getYtsaurusGroup(getSourceGroup("devs", sourceType)), + Members: NewStringSetFromItems(aliceName), }, { - YtsaurusGroup: qaYtsaurusGroup, - Members: NewStringSetFromItems(bobYtsaurus.Username), + YtsaurusGroup: getYtsaurusGroup(getSourceGroup("qa", sourceType)), + Members: NewStringSetFromItems(bobName), }, }, - azureGroupsSetUp: []AzureGroupWithMembers{ + sourceGroupsSetUp: []SourceGroupWithMembers{ { - AzureGroup: devsAzureGroup, - Members: NewStringSetFromItems(aliceAzure.AzureID), + SourceGroup: getSourceGroup("devs", sourceType), + Members: NewStringSetFromItems(getSourceUser(aliceName, sourceType).GetId()), }, { - AzureGroup: hqAzureGroup, - Members: NewStringSetFromItems(carolAzure.AzureID), + SourceGroup: getSourceGroup("hq", sourceType), + Members: NewStringSetFromItems(getSourceUser(carolName, sourceType).GetId()), }, }, ytGroupsExpected: []YtsaurusGroupWithMembers{ { - YtsaurusGroup: devsYtsaurusGroup, - Members: NewStringSetFromItems(aliceYtsaurus.Username), + YtsaurusGroup: getYtsaurusGroup(getSourceGroup("devs", sourceType)), + Members: NewStringSetFromItems(aliceName), }, { - YtsaurusGroup: hqYtsaurusGroup, - Members: NewStringSetFromItems(carolYtsaurus.Username), + YtsaurusGroup: getYtsaurusGroup(getSourceGroup("hq", sourceType)), + Members: NewStringSetFromItems(carolName), }, }, }, { - name: "memberships-add-remove", - azureUsersSetUp: []AzureUser{ - aliceAzure, - bobAzure, - carolAzure, + name: "memberships-add-remove", + sourceType: sourceType, + sourceUsersSetUp: []SourceUser{ + getSourceUser(aliceName, sourceType), + getSourceUser(bobName, sourceType), + getSourceUser(carolName, sourceType), }, ytUsersSetUp: []YtsaurusUser{ - aliceYtsaurus, - bobYtsaurus, - carolYtsaurus, + getYtsaurusUser(getSourceUser(aliceName, sourceType)), + getYtsaurusUser(getSourceUser(bobName, sourceType)), + getYtsaurusUser(getSourceUser(carolName, sourceType)), }, ytUsersExpected: []YtsaurusUser{ - aliceYtsaurus, - bobYtsaurus, - carolYtsaurus, + getYtsaurusUser(getSourceUser(aliceName, sourceType)), + getYtsaurusUser(getSourceUser(bobName, sourceType)), + getYtsaurusUser(getSourceUser(carolName, sourceType)), }, ytGroupsSetUp: []YtsaurusGroupWithMembers{ { - YtsaurusGroup: devsYtsaurusGroup, + YtsaurusGroup: getYtsaurusGroup(getSourceGroup("devs", sourceType)), Members: NewStringSetFromItems( - aliceYtsaurus.Username, - bobYtsaurus.Username, + aliceName, + bobName, ), }, }, - azureGroupsSetUp: []AzureGroupWithMembers{ + sourceGroupsSetUp: []SourceGroupWithMembers{ { - AzureGroup: devsAzureGroup, + SourceGroup: getSourceGroup("devs", sourceType), Members: NewStringSetFromItems( - aliceAzure.AzureID, - carolAzure.AzureID, + getSourceUser(aliceName, sourceType).GetId(), + getSourceUser(carolName, sourceType).GetId(), ), }, }, ytGroupsExpected: []YtsaurusGroupWithMembers{ { - YtsaurusGroup: devsYtsaurusGroup, + YtsaurusGroup: getYtsaurusGroup(getSourceGroup("devs", sourceType)), Members: NewStringSetFromItems( - aliceYtsaurus.Username, - carolYtsaurus.Username, + aliceName, + carolName, ), }, }, }, - { - name: "display-name-changes", - azureUsersSetUp: []AzureUser{ - aliceAzure, - bobAzure, - carolAzure, + } + + if sourceType == AzureSourceType { + testCases = append(testCases, testCase{ + name: "display-name-changes", + sourceType: sourceType, + sourceUsersSetUp: []SourceUser{ + getSourceUser(aliceName, sourceType), + getSourceUser(bobName, sourceType), + getSourceUser(carolName, sourceType), }, ytUsersSetUp: []YtsaurusUser{ - aliceYtsaurus, - bobYtsaurus, - carolYtsaurus, + getYtsaurusUser(getSourceUser(aliceName, sourceType)), + getYtsaurusUser(getSourceUser(bobName, sourceType)), + getYtsaurusUser(getSourceUser(carolName, sourceType)), }, ytUsersExpected: []YtsaurusUser{ - aliceYtsaurus, - bobYtsaurus, - carolYtsaurus, + getYtsaurusUser(getSourceUser(aliceName, sourceType)), + getYtsaurusUser(getSourceUser(bobName, sourceType)), + getYtsaurusUser(getSourceUser(carolName, sourceType)), }, ytGroupsSetUp: []YtsaurusGroupWithMembers{ { - YtsaurusGroup: devsYtsaurusGroup, + YtsaurusGroup: getYtsaurusGroup(getSourceGroup("devs", sourceType)), Members: NewStringSetFromItems( - aliceYtsaurus.Username, - bobYtsaurus.Username, + aliceName, + bobName, ), }, { - YtsaurusGroup: hqYtsaurusGroup, + YtsaurusGroup: getYtsaurusGroup(getSourceGroup("hq", sourceType)), Members: NewStringSetFromItems( - aliceYtsaurus.Username, - bobYtsaurus.Username, + aliceName, + bobName, ), }, }, - azureGroupsSetUp: []AzureGroupWithMembers{ + sourceGroupsSetUp: []SourceGroupWithMembers{ { // This group should be updated. - AzureGroup: devsAzureGroupChangedDisplayName, + SourceGroup: getUpdatedSourceGroup("devs", sourceType), // Members list are also updated. Members: NewStringSetFromItems( - aliceAzure.AzureID, - carolAzure.AzureID, + getSourceUser(aliceName, sourceType).GetId(), + getSourceUser(carolName, sourceType).GetId(), ), }, { // for this group only displayName should be updated - AzureGroup: hqAzureGroupChangedBackwardCompatible, + SourceGroup: getChangedBackwardCompatibleSourceGroup("hq", sourceType), // members also changed Members: NewStringSetFromItems( - aliceAzure.AzureID, - carolAzure.AzureID, + getSourceUser(aliceName, sourceType).GetId(), + getSourceUser(carolName, sourceType).GetId(), ), }, }, ytGroupsExpected: []YtsaurusGroupWithMembers{ { - YtsaurusGroup: devsYtsaurusGroupChangedDisplayName, + YtsaurusGroup: getYtsaurusGroup(getUpdatedSourceGroup("devs", sourceType)), Members: NewStringSetFromItems( - aliceYtsaurus.Username, - carolYtsaurus.Username, + aliceName, + carolName, ), }, { - YtsaurusGroup: hqYtsaurusGroupChangedBackwardCompatible, + YtsaurusGroup: getYtsaurusGroup(getChangedBackwardCompatibleSourceGroup("hq", sourceType)), Members: NewStringSetFromItems( - aliceYtsaurus.Username, - carolYtsaurus.Username, + aliceName, + carolName, ), }, }, - }, + }) + } + return testCases +} + +var ( + testTimeStr = "2023-10-20T12:00:00Z" + initialTestTime = parseAppTime(testTimeStr) + + defaultUsernameReplacements = []ReplacementPair{ + {"@acme.com", ""}, + {"@", ":"}, + } + defaultGroupnameReplacements = []ReplacementPair{ + {"|all", ""}, + } + defaultAppConfig = &AppConfig{ + UsernameReplacements: defaultUsernameReplacements, + GroupnameReplacements: defaultGroupnameReplacements, } ) -// TestAppSync uses local YTsaurus container and fake Azure to test all the cases: -// [x] If Azure user not in YTsaurus -> created; -// [x] If Azure user already in YTsaurus no changes -> skipped; -// [x] If Azure user already in YTsaurus with changes -> updated; -// [x] If user in YTsaurus but not in Azure (and ban_before_remove_duration=0) -> removed; -// [x] If user in YTsaurus but not in Azure (and ban_before_remove_duration != 0) -> banned -> removed; +// TestAppSync uses local YTsaurus container and fake Source to test all the cases: +// [x] If Source user not in YTsaurus -> created; +// [x] If Source user already in YTsaurus no changes -> skipped; +// [x] If Source user already in YTsaurus with changes -> updated; +// [x] If user in YTsaurus but not in Source (and ban_before_remove_duration=0) -> removed; +// [x] If user in YTsaurus but not in Source (and ban_before_remove_duration != 0) -> banned -> removed; // [x] If Azure user without @azure attribute in YTsaurus —> ignored; -// [x] Azure user field updates is reflected in YTsaurus user; +// [x] Source user field updates is reflected in YTsaurus user; // [x] YTsaurus username is built according to config; // [x] YTsaurus username is in lowercase; -// [x] If Azure group is not exist in YTsaurus -> creating YTsaurus with members; -// [x] If YTsaurus group is not exist in Azure -> delete YTsaurus group; -// [x] If Azure group membership changed -> update in YTsaurus group membership; -// [x] If Azure group fields (though there are none extra fields) changed -> update YTsaurus group; -// [x] If Azure group displayName changed -> recreate YTsaurus group; -// [x] If Azure group displayName changed AND Azure members changed -> recreate YTsaurus group with actual members set; +// [x] If Source group is not exist in YTsaurus -> creating YTsaurus with members; +// [x] If YTsaurus group is not exist in Source -> delete YTsaurus group; +// [x] If Source group membership changed -> update in YTsaurus group membership; +// [x] If Source group fields (though there are none extra fields) changed -> update YTsaurus group; +// [x] If Source group displayName changed -> recreate YTsaurus group; +// [x] If Source group displayName changed AND Source members changed -> recreate YTsaurus group with actual members set; // [x] YTsaurus group name is built according to config; // [x] Remove limits config option works. func TestAppSync(t *testing.T) { require.NoError(t, os.Setenv(defaultYtsaurusSecretEnvVar, ytDevToken)) - for _, tc := range testCases { - t.Run( - tc.name, - func(tc testCase) func(t *testing.T) { - return func(t *testing.T) { - if tc.testTime.IsZero() { - tc.testTime = initialTestTime - } - clock := testclock.NewFakePassiveClock(initialTestTime) - - ytLocal := NewYtsaurusLocal() - defer func() { require.NoError(t, ytLocal.Stop()) }() - require.NoError(t, ytLocal.Start()) - - azure := NewAzureFake() - azure.setUsers(tc.azureUsersSetUp) - azure.setGroups(tc.azureGroupsSetUp) - - ytClient, err := ytLocal.GetClient() - require.NoError(t, err) - - initialYtUsers, initialYtGroups := getAllYtsaurusObjects(t, ytClient) - setupYtsaurusObjects(t, ytClient, tc.ytUsersSetUp, tc.ytGroupsSetUp) - - if tc.appConfig == nil { - tc.appConfig = defaultAppConfig - } - app, err := NewAppCustomized( - &Config{ - App: tc.appConfig, - Azure: &AzureConfig{}, - Ytsaurus: &YtsaurusConfig{ - Proxy: ytLocal.GetProxy(), - ApplyUserChanges: true, - ApplyGroupChanges: true, - ApplyMemberChanges: true, - LogLevel: "DEBUG", + for _, sourceType := range []SourceType{LdapSourceType, AzureSourceType} { + for _, tc := range getTestCases(sourceType) { + t.Run( + fmt.Sprintf("%v-%v", sourceType, tc.name), + func(tc testCase) func(t *testing.T) { + return func(t *testing.T) { + if tc.testTime.IsZero() { + tc.testTime = initialTestTime + } + clock := testclock.NewFakePassiveClock(initialTestTime) + + ytLocal := NewYtsaurusLocal() + defer func() { require.NoError(t, ytLocal.Stop()) }() + require.NoError(t, ytLocal.Start()) + + var source Source + + switch tc.sourceType { + case AzureSourceType: + azure := NewAzureFake() + azure.setUsers(tc.sourceUsersSetUp) + azure.setGroups(tc.sourceGroupsSetUp) + + source = azure + case LdapSourceType: + ldapLocal := NewOpenLdapLocal() + + defer func() { require.NoError(t, ldapLocal.Stop()) }() + require.NoError(t, ldapLocal.Start()) + + ldapConfig, err := ldapLocal.GetConfig() + require.NoError(t, err) + ldapSource, err := NewLdap(ldapConfig, getDevelopmentLogger()) + require.NoError(t, err) + + setupLdapObjects(t, ldapSource.Connection, tc.sourceUsersSetUp, tc.sourceGroupsSetUp) + source = ldapSource + } + + ytClient, err := ytLocal.GetClient() + require.NoError(t, err) + + initialYtUsers, initialYtGroups := getAllYtsaurusObjects(t, ytClient) + setupYtsaurusObjects(t, ytClient, tc.ytUsersSetUp, tc.ytGroupsSetUp) + + if tc.appConfig == nil { + tc.appConfig = defaultAppConfig + } + app, err := NewAppCustomized( + &Config{ + App: *tc.appConfig, + Azure: &AzureConfig{}, + Ldap: &LdapConfig{}, + Ytsaurus: YtsaurusConfig{ + Proxy: ytLocal.GetProxy(), + ApplyUserChanges: true, + ApplyGroupChanges: true, + ApplyMemberChanges: true, + LogLevel: "DEBUG", + }, }, - }, getDevelopmentLogger(), - azure, - clock, - ) - require.NoError(t, err) - - app.syncOnce() - - // we have eventually here, because user removal takes some time. - require.Eventually( - t, - func() bool { - udiff, gdiff := diffYtsaurusObjects(t, ytClient, tc.ytUsersExpected, initialYtUsers, tc.ytGroupsExpected, initialYtGroups) - actualUsers, actualGroups := getAllYtsaurusObjects(t, ytClient) - if udiff != "" { - t.Log("Users diff is not empty yet:", udiff) - t.Log("expected users", tc.ytUsersExpected) - t.Log("actual users", actualUsers) - } - if gdiff != "" { - t.Log("Groups diff is not empty yet:", gdiff) - t.Log("expected groups", tc.ytGroupsExpected) - t.Log("actual groups", actualGroups) - } - return udiff == "" && gdiff == "" - }, - 3*time.Second, - 300*time.Millisecond, - ) - } - }(tc), - ) + getDevelopmentLogger(), + source, + clock, + ) + require.NoError(t, err) + + app.syncOnce() + + // We have eventually here, because user removal takes some time. + require.Eventually( + t, + func() bool { + udiff, gdiff := diffYtsaurusObjects(t, ytClient, tc.ytUsersExpected, initialYtUsers, tc.ytGroupsExpected, initialYtGroups) + actualUsers, actualGroups := getAllYtsaurusObjects(t, ytClient) + if udiff != "" { + t.Log("Users diff is not empty yet:", udiff) + t.Log("expected users", tc.ytUsersExpected) + t.Log("actual users", actualUsers) + } + if gdiff != "" { + t.Log("Groups diff is not empty yet:", gdiff) + t.Log("expected groups", tc.ytGroupsExpected) + t.Log("actual groups", actualGroups) + } + return udiff == "" && gdiff == "" + }, + 3*time.Second, + 300*time.Millisecond, + ) + } + }(tc), + ) + } } } @@ -615,7 +646,7 @@ func TestManageUnmanagedUsersIsForbidden(t *testing.T) { "Prevented attempt to change manual managed user", ) require.ErrorContains(t, - ytsaurus.UpdateUser(username, YtsaurusUser{Username: username, Email: "dummy@acme.com"}), + ytsaurus.UpdateUser(username, YtsaurusUser{Username: username, SourceUser: AzureUser{Email: "dummy@acme.com"}}), "Prevented attempt to change manual managed user", ) } @@ -629,6 +660,114 @@ func getAllYtsaurusObjects(t *testing.T, client yt.Client) (users []YtsaurusUser return allUsers, allGroups } +func setupLdapObjects(t *testing.T, conn *ldap.Conn, users []SourceUser, groups []SourceGroupWithMembers) { + require.NoError(t, conn.Add(&ldap.AddRequest{ + DN: "ou=People,dc=example,dc=org", + Attributes: []ldap.Attribute{ + { + Type: "objectClass", + Vals: []string{"organizationalUnit"}, + }, + { + Type: "ou", + Vals: []string{"People"}, + }, + }, + })) + + require.NoError(t, conn.Add(&ldap.AddRequest{ + DN: "ou=Group,dc=example,dc=org", + Attributes: []ldap.Attribute{ + { + Type: "objectClass", + Vals: []string{"organizationalUnit"}, + }, + { + Type: "ou", + Vals: []string{"Group"}, + }, + }, + })) + + for _, user := range users { + ldapUser := user.(LdapUser) + addRequest := ldap.AddRequest{ + DN: fmt.Sprintf("uid=%s,ou=People,dc=example,dc=org", user.GetId()), + Attributes: []ldap.Attribute{ + { + Type: "objectClass", + Vals: []string{"top", "posixAccount", "inetOrgPerson"}, + }, + { + Type: "ou", + Vals: []string{"People"}, + }, + { + Type: "cn", + Vals: []string{user.GetName()}, + }, + { + Type: "uid", + Vals: []string{user.GetId()}, + }, + { + Type: "uidNumber", + Vals: []string{user.GetId()}, + }, + { + Type: "gidNumber", + Vals: []string{user.GetId()}, + }, + { + Type: "givenName", + Vals: []string{ldapUser.FirstName}, + }, + { + Type: "homeDirectory", + Vals: []string{ldapUser.GetId()}, + }, + { + Type: "sn", + Vals: []string{ldapUser.GetName() + "-surname"}, + }, + }, + } + require.NoError(t, conn.Add(&addRequest)) + } + + for groupId, group := range groups { + ldapGroup := group.SourceGroup.(LdapGroup) + + members := make([]string, 0) + for member := range group.Members.Iter() { + members = append(members, member) + } + + addRequest := ldap.AddRequest{ + DN: fmt.Sprintf("cn=%s,ou=Group,dc=example,dc=org", ldapGroup.GetId()), + Attributes: []ldap.Attribute{ + { + Type: "objectClass", + Vals: []string{"top", "posixGroup"}, + }, + { + Type: "cn", + Vals: []string{ldapGroup.GetName()}, + }, + { + Type: "gidNumber", + Vals: []string{fmt.Sprint(groupId)}, + }, + { + Type: "memberUid", + Vals: members, + }, + }, + } + require.NoError(t, conn.Add(&addRequest)) + } +} + func setupYtsaurusObjects(t *testing.T, client yt.Client, users []YtsaurusUser, groups []YtsaurusGroupWithMembers) { t.Log("Setting up yt for test") for _, user := range users { diff --git a/azure_fake.go b/azure_fake.go index 2ac1e06..937261c 100644 --- a/azure_fake.go +++ b/azure_fake.go @@ -1,26 +1,26 @@ package main type AzureFake struct { - users []AzureUser - groups []AzureGroupWithMembers + users []SourceUser + groups []SourceGroupWithMembers } func NewAzureFake() *AzureFake { return &AzureFake{} } -func (a *AzureFake) setUsers(users []AzureUser) { +func (a *AzureFake) setUsers(users []SourceUser) { a.users = users } -func (a *AzureFake) setGroups(groups []AzureGroupWithMembers) { +func (a *AzureFake) setGroups(groups []SourceGroupWithMembers) { a.groups = groups } -func (a *AzureFake) GetUsers() ([]AzureUser, error) { +func (a *AzureFake) GetUsers() ([]SourceUser, error) { return a.users, nil } -func (a *AzureFake) GetGroupsWithMembers() ([]AzureGroupWithMembers, error) { +func (a *AzureFake) GetGroupsWithMembers() ([]SourceGroupWithMembers, error) { return a.groups, nil } diff --git a/azure_real.go b/azure_real.go index 4b9c19c..5ecf5a4 100644 --- a/azure_real.go +++ b/azure_real.go @@ -17,14 +17,10 @@ import ( ) const ( - scope = "https://graph.microsoft.com/.default" - defaultAzureTimeout = 3 * time.Second - defaultAzureSecretEnvVar = "AZURE_CLIENT_SECRET" - msgraphExpandLimit = 20 + scope = "https://graph.microsoft.com/.default" + msgraphExpandLimit = 20 ) -type AzureID = string - var ( defaultUserFieldsToSelect = []string{ "userPrincipalName", @@ -41,33 +37,6 @@ var ( } ) -type AzureUser struct { - // PrincipalName is unique human-readable Azure user field, used (possibly with changes) - // for the corresponding YTsaurus user's `name` attribute. - PrincipalName string - - AzureID AzureID - Email string - FirstName string - LastName string - DisplayName string -} - -type AzureGroup struct { - // Identity is unique human-readable Azure user field, used (possibly with changes) - // for the corresponding YTsaurus user's `name` attribute. - Identity string - - AzureID AzureID - DisplayName string -} - -type AzureGroupWithMembers struct { - AzureGroup - // Members is a set of strings, representing users' AzureID. - Members StringSet -} - type AzureReal struct { graphClient *msgraphsdk.GraphServiceClient @@ -98,7 +67,7 @@ func NewAzureReal(cfg *AzureConfig, logger appLoggerType) (*AzureReal, error) { nil, ) if err != nil { - return nil, errors.Wrap(err, "failed to create Azure secret credentials") + return nil, errors.Wrap(err, "failed to create Source secret credentials") } graphClient, err := msgraphsdk.NewGraphServiceClientWithCredentials(cred, []string{scope}) @@ -129,7 +98,7 @@ func handleNil[T any](s *T) T { return result } -func (a *AzureReal) GetUsers() ([]AzureUser, error) { +func (a *AzureReal) GetUsers() ([]SourceUser, error) { ctx, cancel := context.WithTimeout(context.Background(), a.timeout) defer cancel() @@ -139,7 +108,7 @@ func (a *AzureReal) GetUsers() ([]AzureUser, error) { } usersSkipped := 0 - var users []AzureUser + var users []SourceUser for _, user := range usersRaw { principalName := handleNil(user.GetUserPrincipalName()) id := handleNil(user.GetId()) @@ -173,11 +142,11 @@ func (a *AzureReal) GetUsers() ([]AzureUser, error) { } } - a.logger.Infow("Fetched users from Azure AD", "got", len(usersRaw), "skipped", usersSkipped) + a.logger.Infow("Fetched users from Source AD", "got", len(usersRaw), "skipped", usersSkipped) return users, nil } -func (a *AzureReal) GetGroupsWithMembers() ([]AzureGroupWithMembers, error) { +func (a *AzureReal) GetGroupsWithMembers() ([]SourceGroupWithMembers, error) { ctx, cancel := context.WithTimeout(context.Background(), a.timeout) defer cancel() @@ -187,7 +156,7 @@ func (a *AzureReal) GetGroupsWithMembers() ([]AzureGroupWithMembers, error) { } groupsSkipped := 0 - var groups []AzureGroupWithMembers + var groups []SourceGroupWithMembers for _, group := range groupsRaw { displayName := handleNil(group.GetDisplayName()) id := handleNil(group.GetId()) @@ -225,8 +194,8 @@ func (a *AzureReal) GetGroupsWithMembers() ([]AzureGroupWithMembers, error) { a.maybePrintDebugLogs(id, "azure_members_count", len(memberIDs.ToSlice())) groups = append(groups, - AzureGroupWithMembers{ - AzureGroup: AzureGroup{ + SourceGroupWithMembers{ + SourceGroup: AzureGroup{ Identity: displayName, AzureID: id, DisplayName: displayName, @@ -235,11 +204,11 @@ func (a *AzureReal) GetGroupsWithMembers() ([]AzureGroupWithMembers, error) { }) } - a.logger.Infow("Fetched groups from Azure AD", "got", len(groupsRaw), "skipped", groupsSkipped) + a.logger.Infow("Fetched groups from Source AD", "got", len(groupsRaw), "skipped", groupsSkipped) return groups, nil } -func (a *AzureReal) maybePrintDebugLogs(id AzureID, args ...any) { +func (a *AzureReal) maybePrintDebugLogs(id ObjectID, args ...any) { args = append([]any{"id", id}, args...) for _, debugID := range a.debugAzureIDs { if id == debugID { @@ -285,7 +254,7 @@ func (a *AzureReal) getUsersRaw(ctx context.Context, fieldsToSelect []string, fi return true }) if err != nil { - return nil, errors.Wrap(err, "failed to iterate over Azure users") + return nil, errors.Wrap(err, "failed to iterate over Source users") } return rawUsers, nil } @@ -326,7 +295,7 @@ func (a *AzureReal) getGroupsWithMembersRaw(ctx context.Context, fieldsToSelect return true }) if err != nil { - return nil, errors.Wrap(err, "failed to iterate over Azure groups") + return nil, errors.Wrap(err, "failed to iterate over Source groups") } return rawGroups, nil } @@ -364,7 +333,7 @@ func (a *AzureReal) getGroupMembers(ctx context.Context, groupID string) ([]mode return true }) if err != nil { - return nil, errors.Wrap(err, "failed to iterate over Azure group members") + return nil, errors.Wrap(err, "failed to iterate over Source group members") } return rawMembers, nil diff --git a/azure_real_integration_test.go b/azure_real_integration_test.go index 18037b2..5864cc6 100644 --- a/azure_real_integration_test.go +++ b/azure_real_integration_test.go @@ -14,7 +14,7 @@ import ( //go:embed config.local.yaml var _localConfig embed.FS -// TestPrintAzureUsersIntegration tests nothing, but can be used to debug Azure users retrieved from ms graph api. +// TestPrintAzureUsersIntegration tests nothing, but can be used to debug Source users retrieved from ms graph api. // In particular, it can be used to tune userFilter for production use. // It requires AZURE_CLIENT_SECRET env var and `config.local.yaml` file (which is .gitignored). func TestPrintAzureUsersIntegration(t *testing.T) { @@ -79,7 +79,7 @@ func TestPrintAzureUsersIntegration(t *testing.T) { require.NotEmpty(t, usersRaw) } -// TestPrintAzureGroupsIntegration tests nothing, but can be used to debug Azure groups retrieved from ms graph api. +// TestPrintAzureGroupsIntegration tests nothing, but can be used to debug Source groups retrieved from ms graph api. // In particular, it can be used to tune groupsFilter for production use. // It requires AZURE_CLIENT_SECRET env var and `config.local.yaml` file (which is .gitignored). func TestPrintAzureGroupsIntegrationRaw(t *testing.T) { diff --git a/config.go b/config.go index 00a2835..d3d7fad 100644 --- a/config.go +++ b/config.go @@ -5,19 +5,21 @@ import ( ) type Config struct { - App *AppConfig `yaml:"app"` - Azure *AzureConfig `yaml:"azure"` - Ytsaurus *YtsaurusConfig `yaml:"ytsaurus"` - Logging *LoggingConfig `yaml:"logging"` + App AppConfig `yaml:"app"` + Ytsaurus YtsaurusConfig `yaml:"ytsaurus"` + Logging LoggingConfig `yaml:"logging"` + + // One of them should be specified. + Azure *AzureConfig `yaml:"azure,omitempty"` + Ldap *LdapConfig `yaml:"ldap,omitempty"` } type AppConfig struct { // SyncInterval is the interval between full synchronizations. - // Zero value means that auto-sync disabled (sync can be invoked only manually). - SyncInterval time.Duration `yaml:"sync_interval"` + // If it is not speciied or value is zero than auto-sync disabled (sync can be invoked only manually). + SyncInterval *time.Duration `yaml:"sync_interval"` - // UsernameReplacements is a list of replaces which will be applied to the userPrincipalName Azure field before - // using as username in Ytsaurus. + // UsernameReplacements is a list of replaces which will be applied to a username for source (Source or Ldap). // For example, you may use it to strip off characters like @ which are not recommended for use // in usernames as they are required to be escaped in YPath. UsernameReplacements []ReplacementPair `yaml:"username_replacements"` @@ -25,11 +27,12 @@ type AppConfig struct { // If count users or groups for planned delete in on sync cycle reaches RemoveLimit // app will fail that sync cycle. - RemoveLimit int `yaml:"remove_limit"` + // No limit if it is not specified. + RemoveLimit *int `yaml:"remove_limit,omitempty"` // BanBeforeRemoveDuration is a duration of a graceful ban before finally removing the user from YTsaurus. - // Default value is 0s, which means remove straight after user was found to be missing from Azure,. - BanBeforeRemoveDuration time.Duration `yaml:"ban_before_remove_duration"` + // If it is not specified, user will be removed straight after user was found to be missing from source (Source or Ldap). + BanBeforeRemoveDuration *time.Duration `yaml:"ban_before_remove_duration"` } type ReplacementPair struct { @@ -41,6 +44,7 @@ type AzureConfig struct { Tenant string `yaml:"tenant"` ClientID string `yaml:"client_id"` ClientSecretEnvVar string `yaml:"client_secret_env_var"` // default: "AZURE_CLIENT_SECRET" + // UsersFilter is MS Graph $filter value used for user fetching requests. // See https://learn.microsoft.com/en-us/graph/api/user-list?#optional-query-parameters UsersFilter string `yaml:"users_filter"` @@ -48,13 +52,54 @@ type AzureConfig struct { // See https://learn.microsoft.com/en-us/graph/api/group-list GroupsFilter string `yaml:"groups_filter"` // GroupsDisplayNameSuffixPostFilter applied to the fetched groups display names. - GroupsDisplayNameSuffixPostFilter string `yaml:"groups_display_name_suffix_post_filter"` - Timeout time.Duration `yaml:"timeout"` + Timeout time.Duration `yaml:"timeout"` + + // TODO(nadya73): support for ldap also, but with other name. + GroupsDisplayNameSuffixPostFilter string `yaml:"groups_display_name_suffix_post_filter"` + // TODO(nadya73): support for ldap also, but with other name. // DebugAzureIDs is a list of ids for which app will print more debug info in logs. DebugAzureIDs []string `yaml:"debug_azure_ids"` } +type LdapUsersConfig struct { + // A filter for getting users. + // For example, `(objectClass=account)`. + Filter string `yaml:"filter"` + // An attribute type which will be used as @name attribute. + // For example, `cn`. + UsernameAttributeType string `yaml:"username_attribute_type"` + // For example, `uid`. + UidAttributeType string `yaml:"uid_attribute_type"` + FirstNameAttributeType *string `yaml:"first_name_attribute_type"` + // A list of usernames for which app will print more debug info in logs. + DebugUsernames []string `yaml:"debug_usernames"` +} + +type LdapGroupsConfig struct { + // A filter for getting groups. + // For example, `(objectClass=posixGroup)`. + Filter string `yaml:"filter"` + // An attribute type which will be used as @name attribute. + // For example, `cn`. + GroupnameAttributeType string `yaml:"groupname_attribute_type"` + // An attribute type which will be used for getting group members. + // For example, `memberUid`. + MemberUidAttributeType string `yaml:"member_uid_attribute_type"` + + // A list of groupnames for which app will print more debug info in logs. + DebugGroupnames []string `yaml:"debug_groupnames"` +} + +type LdapConfig struct { + Address string `yaml:"address"` + BindDN string `yaml:"bind_dn"` + BindPasswordEnvVar string `yaml:"bind_password_env_var"` + Users LdapUsersConfig `yaml:"users"` + Groups LdapGroupsConfig `yaml:"groups"` + BaseDN string `yaml:"base_dn"` +} + type YtsaurusConfig struct { Proxy string `yaml:"proxy"` // SecretEnvVar is a name of env variable with YTsaurus token. Default: "YT_TOKEN". diff --git a/config_test.go b/config_test.go index 53c502e..6d075a2 100644 --- a/config_test.go +++ b/config_test.go @@ -2,6 +2,7 @@ package main import ( "embed" + "go.ytsaurus.tech/library/go/ptr" "testing" "time" @@ -17,7 +18,7 @@ func TestConfig(t *testing.T) { cfg, err := loadConfig(configPath) require.NoError(t, err) - require.Equal(t, 5*time.Minute, cfg.App.SyncInterval) + require.Equal(t, ptr.Duration(5*time.Minute), cfg.App.SyncInterval) require.Equal(t, []ReplacementPair{ {From: "@acme.com", To: ""}, {From: "@", To: ":"}, @@ -25,8 +26,8 @@ func TestConfig(t *testing.T) { require.Equal(t, []ReplacementPair{ {From: "|all", To: ""}, }, cfg.App.GroupnameReplacements) - require.Equal(t, 10, cfg.App.RemoveLimit) - require.Equal(t, 7*24*time.Hour, cfg.App.BanBeforeRemoveDuration) + require.Equal(t, ptr.Int(10), cfg.App.RemoveLimit) + require.Equal(t, ptr.Duration(7*24*time.Hour), cfg.App.BanBeforeRemoveDuration) require.Equal(t, "acme.onmicrosoft.com", cfg.Azure.Tenant) require.Equal(t, "abcdefgh-a000-b111-c222-abcdef123456", cfg.Azure.ClientID) @@ -45,7 +46,7 @@ func TestConfig(t *testing.T) { require.Equal(t, "WARN", cfg.Logging.Level) require.Equal(t, true, cfg.Logging.IsProduction) - logger, err := configureLogger(cfg.Logging) + logger, err := configureLogger(&cfg.Logging) require.NoError(t, err) logger.Debugw("test logging message", "key", "val") } diff --git a/diff.go b/diff.go index d5802c0..771f662 100644 --- a/diff.go +++ b/diff.go @@ -25,36 +25,42 @@ func (a *App) syncOnce() { } func (a *App) isRemoveLimitReached(objectsCount int) bool { - if a.removeLimit <= 0 { + if a.removeLimit == nil || *a.removeLimit <= 0 { return false } - return objectsCount >= a.removeLimit + return objectsCount >= *a.removeLimit } -// syncUsers syncs AD users with YTsaurus cluster and returns /actual/ map[AzureID]YtsaurusUser +// syncUsers syncs AD users with YTsaurus cluster and returns /actual/ map[ObjectID]YtsaurusUser // after applying changes. -func (a *App) syncUsers() (map[AzureID]YtsaurusUser, error) { +func (a *App) syncUsers() (map[ObjectID]YtsaurusUser, error) { a.logger.Info("Start syncing users") - azureUsers, err := a.azure.GetUsers() - if err != nil { - return nil, errors.Wrap(err, "failed to get Azure users") + var err error + var sourceUsers []SourceUser + + if a.source != nil { + sourceUsers, err = a.source.GetUsers() + if err != nil { + return nil, errors.Wrap(err, "failed to get Source users") + } } + ytUsers, err := a.ytsaurus.GetUsers() if err != nil { return nil, errors.Wrap(err, "failed to get YTsaurus users") } - azureUsersMap := make(map[AzureID]AzureUser) - ytUsersMap := make(map[AzureID]YtsaurusUser) + sourceUsersMap := make(map[ObjectID]SourceUser) + ytUsersMap := make(map[ObjectID]YtsaurusUser) - for _, user := range azureUsers { - azureUsersMap[user.AzureID] = user + for _, user := range sourceUsers { + sourceUsersMap[user.GetId()] = user } for _, user := range ytUsers { - ytUsersMap[user.AzureID] = user + ytUsersMap[user.SourceUser.GetId()] = user } - diff := a.diffUsers(azureUsersMap, ytUsersMap) + diff := a.diffUsers(sourceUsersMap, ytUsersMap) if a.isRemoveLimitReached(len(diff.remove)) { return nil, fmt.Errorf("delete limit in one cycle reached: %d %v", len(diff.remove), diff) } @@ -74,7 +80,7 @@ func (a *App) syncUsers() (map[AzureID]YtsaurusUser, error) { removedCount++ } // Actualizing user map for group sync later. - delete(ytUsersMap, user.AzureID) + delete(ytUsersMap, user.SourceUser.GetId()) } for _, user := range diff.create { err = a.ytsaurus.CreateUser(user) @@ -83,7 +89,7 @@ func (a *App) syncUsers() (map[AzureID]YtsaurusUser, error) { a.logger.Errorw("failed to create user", zap.Error(err), "user", user) } // Actualizing user map for group sync later. - ytUsersMap[user.AzureID] = user + ytUsersMap[user.SourceUser.GetId()] = user } for _, updatedUser := range diff.update { err = a.ytsaurus.UpdateUser(updatedUser.OldUsername, updatedUser.YtsaurusUser) @@ -92,7 +98,7 @@ func (a *App) syncUsers() (map[AzureID]YtsaurusUser, error) { a.logger.Errorw("failed to update user", zap.Error(err), "user", updatedUser) } // Actualizing user map for group sync later. - ytUsersMap[updatedUser.AzureID] = updatedUser.YtsaurusUser + ytUsersMap[updatedUser.SourceUser.GetId()] = updatedUser.YtsaurusUser } a.logger.Infow("Finish syncing users", "created", len(diff.create)-createErrCount, @@ -106,11 +112,11 @@ func (a *App) syncUsers() (map[AzureID]YtsaurusUser, error) { return ytUsersMap, nil } -func (a *App) syncGroups(usersMap map[AzureID]YtsaurusUser) error { +func (a *App) syncGroups(usersMap map[ObjectID]YtsaurusUser) error { a.logger.Info("Start syncing groups") - azureGroups, err := a.azure.GetGroupsWithMembers() + azureGroups, err := a.source.GetGroupsWithMembers() if err != nil { - return errors.Wrap(err, "failed to get Azure groups") + return errors.Wrap(err, "failed to get Source groups") } ytGroups, err := a.ytsaurus.GetGroupsWithMembers() if err != nil { @@ -189,30 +195,30 @@ type groupDiff struct { } func (a *App) diffGroups( - azureGroups []AzureGroupWithMembers, + sourceGroups []SourceGroupWithMembers, ytGroups []YtsaurusGroupWithMembers, - usersMap map[AzureID]YtsaurusUser, + usersMap map[ObjectID]YtsaurusUser, ) groupDiff { var groupsToCreate, groupsToRemove []YtsaurusGroup var groupsToUpdate []UpdatedYtsaurusGroup var membersToAdd, membersToRemove []YtsaurusMembership - azureGroupsWithMembersMap := make(map[AzureID]AzureGroupWithMembers) - ytGroupsWithMembersMap := make(map[AzureID]YtsaurusGroupWithMembers) + sourceGroupsWithMembersMap := make(map[ObjectID]SourceGroupWithMembers) + ytGroupsWithMembersMap := make(map[ObjectID]YtsaurusGroupWithMembers) - for _, group := range azureGroups { - azureGroupsWithMembersMap[group.AzureID] = group + for _, group := range sourceGroups { + sourceGroupsWithMembersMap[group.SourceGroup.GetId()] = group } for _, group := range ytGroups { - ytGroupsWithMembersMap[group.AzureID] = group + ytGroupsWithMembersMap[group.SourceGroup.GetId()] = group } - // Collecting groups to create (the ones that exist in Azure but not in YTsaurus). - for azureID, azureGroupWithMembers := range azureGroupsWithMembersMap { - if _, ok := ytGroupsWithMembersMap[azureID]; !ok { - newYtsaurusGroup := a.buildYtsaurusGroup(azureGroupWithMembers.AzureGroup) + // Collecting groups to create (the ones that exist in Source but not in YTsaurus). + for objectID, sourceGroupWithMembers := range sourceGroupsWithMembersMap { + if _, ok := ytGroupsWithMembersMap[objectID]; !ok { + newYtsaurusGroup := a.buildYtsaurusGroup(sourceGroupWithMembers.SourceGroup) groupsToCreate = append(groupsToCreate, newYtsaurusGroup) - for username := range a.buildYtsaurusGroupMembers(azureGroupWithMembers, usersMap).Iter() { + for username := range a.buildYtsaurusGroupMembers(sourceGroupWithMembers, usersMap).Iter() { membersToAdd = append(membersToAdd, YtsaurusMembership{ GroupName: newYtsaurusGroup.Name, Username: username, @@ -221,17 +227,17 @@ func (a *App) diffGroups( } } - for azureID, ytGroupWithMembers := range ytGroupsWithMembersMap { - // Collecting groups to remove (the ones that exist in YTsaurus and not in Azure). - azureGroupWithMembers, ok := azureGroupsWithMembersMap[azureID] + for objectID, ytGroupWithMembers := range ytGroupsWithMembersMap { + // Collecting groups to remove (the ones that exist in YTsaurus and not in Source). + sourceGroupWithMembers, ok := sourceGroupsWithMembersMap[objectID] if !ok { groupsToRemove = append(groupsToRemove, ytGroupWithMembers.YtsaurusGroup) continue } - // Collecting groups with changed Azure fields (actually we have only displayName for now which + // Collecting groups with changed Source fields (actually we have only displayName for now which // should change, though we still handle that just in case). - groupChanged, updatedYtGroup := a.isGroupChanged(azureGroupWithMembers.AzureGroup, ytGroupWithMembers.YtsaurusGroup) + groupChanged, updatedYtGroup := a.isGroupChanged(sourceGroupWithMembers.SourceGroup, ytGroupWithMembers.YtsaurusGroup) // Group name can change after update, so we ensure that correct one is used for membership updates. actualGroupname := ytGroupWithMembers.YtsaurusGroup.Name if groupChanged { @@ -244,7 +250,7 @@ func (a *App) diffGroups( actualGroupname = updatedYtGroup.YtsaurusGroup.Name } - membersCreate, membersRemove := a.isGroupMembersChanged(azureGroupWithMembers, ytGroupWithMembers, usersMap) + membersCreate, membersRemove := a.isGroupMembersChanged(sourceGroupWithMembers, ytGroupWithMembers, usersMap) for _, username := range membersCreate { membersToAdd = append(membersToAdd, YtsaurusMembership{ GroupName: actualGroupname, @@ -275,24 +281,24 @@ type usersDiff struct { } func (a *App) diffUsers( - azureUsersMap map[AzureID]AzureUser, - ytUsersMap map[AzureID]YtsaurusUser, + sourceUsersMap map[ObjectID]SourceUser, + ytUsersMap map[ObjectID]YtsaurusUser, ) usersDiff { var create, remove []YtsaurusUser var update []UpdatedYtsaurusUser - for azureID, azureUser := range azureUsersMap { - if _, ok := ytUsersMap[azureID]; !ok { - create = append(create, a.buildYtsaurusUser(azureUser)) + for objectID, sourceUser := range sourceUsersMap { + if _, ok := ytUsersMap[objectID]; !ok { + create = append(create, a.buildYtsaurusUser(sourceUser)) } } - for azureID, ytUser := range ytUsersMap { - azureUser, ok := azureUsersMap[azureID] + for objectID, ytUser := range ytUsersMap { + sourceUser, ok := sourceUsersMap[objectID] if !ok { remove = append(remove, ytUser) continue } - userChanged, updatedYtUser := a.isUserChanged(azureUser, ytUser) + userChanged, updatedYtUser := a.isUserChanged(sourceUser, ytUser) if !userChanged { continue } @@ -305,49 +311,47 @@ func (a *App) diffUsers( } } -func (a *App) buildUsername(azureUser AzureUser) string { - username := azureUser.PrincipalName - for _, replace := range a.usernameReplaces { - username = strings.Replace(username, replace.From, replace.To, -1) +func (a *App) buildUsername(sourceUser SourceUser) string { + username := sourceUser.GetName() + if a.usernameReplaces != nil { + for _, replace := range a.usernameReplaces { + username = strings.Replace(username, replace.From, replace.To, -1) + } } username = strings.ToLower(username) return username } -func (a *App) buildGroupName(azureGroup AzureGroup) string { - name := azureGroup.Identity - for _, replace := range a.groupnameReplaces { - name = strings.Replace(name, replace.From, replace.To, -1) +func (a *App) buildGroupName(sourceGroup SourceGroup) string { + name := sourceGroup.GetName() + if a.groupnameReplaces != nil { + for _, replace := range a.groupnameReplaces { + name = strings.Replace(name, replace.From, replace.To, -1) + } } name = strings.ToLower(name) return name } -func (a *App) buildYtsaurusUser(azureUser AzureUser) YtsaurusUser { +func (a *App) buildYtsaurusUser(sourceUser SourceUser) YtsaurusUser { return YtsaurusUser{ - Username: a.buildUsername(azureUser), - AzureID: azureUser.AzureID, - PrincipalName: azureUser.PrincipalName, - Email: azureUser.Email, - FirstName: azureUser.FirstName, - LastName: azureUser.LastName, - DisplayName: azureUser.DisplayName, - // If we have Azure user —> he is not banned. + Username: a.buildUsername(sourceUser), + SourceUser: sourceUser, + // If we have Source user —> he is not banned. BannedSince: time.Time{}, } } -func (a *App) buildYtsaurusGroup(azureGroup AzureGroup) YtsaurusGroup { +func (a *App) buildYtsaurusGroup(sourceGroup SourceGroup) YtsaurusGroup { return YtsaurusGroup{ - Name: a.buildGroupName(azureGroup), - AzureID: azureGroup.AzureID, - DisplayName: azureGroup.DisplayName, + Name: a.buildGroupName(sourceGroup), + SourceGroup: sourceGroup, } } -func (a *App) buildYtsaurusGroupMembers(azureGroupWithMembers AzureGroupWithMembers, usersMap map[AzureID]YtsaurusUser) StringSet { +func (a *App) buildYtsaurusGroupMembers(sourceGroupWithMembers SourceGroupWithMembers, usersMap map[ObjectID]YtsaurusUser) StringSet { members := NewStringSet() - for azureID := range azureGroupWithMembers.Members.Iter() { + for azureID := range sourceGroupWithMembers.Members.Iter() { ytUser, ok := usersMap[azureID] if !ok { // User is unknown to the YTsaurus (can be accountEnabled=false). @@ -366,8 +370,8 @@ type UpdatedYtsaurusUser struct { } // If isUserChanged detects that user is changed, it returns UpdatedYtsaurusUser. -func (a *App) isUserChanged(azureUser AzureUser, ytUser YtsaurusUser) (bool, UpdatedYtsaurusUser) { - newUser := a.buildYtsaurusUser(azureUser) +func (a *App) isUserChanged(sourceUser SourceUser, ytUser YtsaurusUser) (bool, UpdatedYtsaurusUser) { + newUser := a.buildYtsaurusUser(sourceUser) if newUser == ytUser { return false, UpdatedYtsaurusUser{} } @@ -382,17 +386,17 @@ type UpdatedYtsaurusGroup struct { } // If isGroupChanged detects that group itself (not members) is changed, it returns UpdatedYtsaurusGroup. -func (a *App) isGroupChanged(azureGroup AzureGroup, ytGroup YtsaurusGroup) (bool, UpdatedYtsaurusGroup) { - newGroup := a.buildYtsaurusGroup(azureGroup) - if newGroup.Name == ytGroup.Name && newGroup.DisplayName == ytGroup.DisplayName { +func (a *App) isGroupChanged(sourceGroup SourceGroup, ytGroup YtsaurusGroup) (bool, UpdatedYtsaurusGroup) { + newGroup := a.buildYtsaurusGroup(sourceGroup) + if newGroup == ytGroup { return false, UpdatedYtsaurusGroup{} } return true, UpdatedYtsaurusGroup{YtsaurusGroup: newGroup, OldName: ytGroup.Name} } // If isGroupMembersChanged detects that group members are changed, it returns lists of usernames to create and remove. -func (a *App) isGroupMembersChanged(azureGroup AzureGroupWithMembers, ytGroup YtsaurusGroupWithMembers, usersMap map[AzureID]YtsaurusUser) (create, remove []string) { - newMembers := a.buildYtsaurusGroupMembers(azureGroup, usersMap) +func (a *App) isGroupMembersChanged(sourceGroup SourceGroupWithMembers, ytGroup YtsaurusGroupWithMembers, usersMap map[ObjectID]YtsaurusUser) (create, remove []string) { + newMembers := a.buildYtsaurusGroupMembers(sourceGroup, usersMap) oldMembers := ytGroup.Members create = newMembers.Difference(oldMembers).ToSlice() @@ -402,15 +406,15 @@ func (a *App) isGroupMembersChanged(azureGroup AzureGroupWithMembers, ytGroup Yt func (a *App) banOrRemoveUser(user YtsaurusUser) (wasBanned, wasRemoved bool, err error) { // Ban settings is disabled. - if a.banDuration == 0 { + if a.banDuration == nil { return false, true, a.ytsaurus.RemoveUser(user.Username) } // If user is not already banned we should do it. - if !user.IsBanned() && a.banDuration != 0 { + if !user.IsBanned() && *a.banDuration != 0 { return true, false, a.ytsaurus.BanUser(user.Username) } // If user was banned longer than setting permits, we remove it. - if user.IsBanned() && time.Since(user.BannedSince) > a.banDuration { + if user.IsBanned() && time.Since(user.BannedSince) > *a.banDuration { return false, true, a.ytsaurus.RemoveUser(user.Username) } a.logger.Debugw("user is banned, but not yet removed", "user", user.Username, "since", user.BannedSince) diff --git a/go.mod b/go.mod index f340080..c29f748 100644 --- a/go.mod +++ b/go.mod @@ -5,14 +5,14 @@ go 1.20 require ( github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.4.0 github.com/deckarep/golang-set/v2 v2.3.1 - github.com/google/go-cmp v0.5.9 + github.com/google/go-cmp v0.6.0 github.com/jessevdk/go-flags v1.5.0 github.com/microsoft/kiota-abstractions-go v1.3.0 github.com/microsoftgraph/msgraph-sdk-go v1.24.0 github.com/microsoftgraph/msgraph-sdk-go-core v1.0.0 github.com/pkg/errors v0.9.1 github.com/stretchr/testify v1.8.4 - github.com/testcontainers/testcontainers-go v0.26.0 + github.com/testcontainers/testcontainers-go v0.28.0 go.uber.org/zap v1.26.0 go.ytsaurus.tech/yt/go v0.0.13 gopkg.in/yaml.v3 v3.0.1 @@ -24,20 +24,25 @@ require ( github.com/Azure/azure-sdk-for-go/sdk/azcore v1.8.0 // indirect github.com/Azure/azure-sdk-for-go/sdk/internal v1.3.0 // indirect github.com/Azure/go-ansiterm v0.0.0-20210617225240-d185dfc1b5a1 // indirect + github.com/Azure/go-ntlmssp v0.0.0-20221128193559-754e69321358 // indirect github.com/AzureAD/microsoft-authentication-library-for-go v1.2.0 // indirect github.com/Microsoft/go-winio v0.6.1 // indirect - github.com/Microsoft/hcsshim v0.11.1 // indirect + github.com/Microsoft/hcsshim v0.11.4 // indirect github.com/andybalholm/brotli v1.0.4 // indirect github.com/cenkalti/backoff/v4 v4.2.1 // indirect github.com/cjlapao/common-go v0.0.39 // indirect - github.com/containerd/containerd v1.7.7 // indirect + github.com/containerd/containerd v1.7.12 // indirect github.com/containerd/log v0.1.0 // indirect github.com/cpuguy83/dockercfg v0.3.1 // indirect github.com/davecgh/go-spew v1.1.1 // indirect + github.com/distribution/reference v0.5.0 // indirect github.com/docker/distribution v2.8.2+incompatible // indirect - github.com/docker/docker v24.0.6+incompatible // indirect - github.com/docker/go-connections v0.4.0 // indirect + github.com/docker/docker v25.0.2+incompatible // indirect + github.com/docker/go-connections v0.5.0 // indirect github.com/docker/go-units v0.5.0 // indirect + github.com/felixge/httpsnoop v1.0.3 // indirect + github.com/go-asn1-ber/asn1-ber v1.5.5 // indirect + github.com/go-ldap/ldap/v3 v3.4.6 // indirect github.com/go-logr/logr v1.2.4 // indirect github.com/go-logr/stdr v1.2.2 // indirect github.com/go-ole/go-ole v1.2.6 // indirect @@ -47,7 +52,7 @@ require ( github.com/golang/protobuf v1.5.3 // indirect github.com/golang/snappy v0.0.4 // indirect github.com/google/tink/go v1.7.0 // indirect - github.com/google/uuid v1.4.0 // indirect + github.com/google/uuid v1.6.0 // indirect github.com/klauspost/compress v1.16.7 // indirect github.com/kylelemons/godebug v1.1.0 // indirect github.com/lufia/plan9stats v0.0.0-20211012122336-39d0f177ccd0 // indirect @@ -62,6 +67,7 @@ require ( github.com/mitchellh/reflectwalk v1.0.2 // indirect github.com/moby/patternmatcher v0.6.0 // indirect github.com/moby/sys/sequential v0.5.0 // indirect + github.com/moby/sys/user v0.1.0 // indirect github.com/moby/term v0.5.0 // indirect github.com/morikuni/aec v1.0.0 // indirect github.com/opencontainers/go-digest v1.0.0 // indirect @@ -72,13 +78,15 @@ require ( github.com/pkg/browser v0.0.0-20210911075715-681adbf594b8 // indirect github.com/pmezard/go-difflib v1.0.0 // indirect github.com/power-devops/perfstat v0.0.0-20210106213030-5aafc221ea8c // indirect - github.com/shirou/gopsutil/v3 v3.23.9 // indirect + github.com/shirou/gopsutil/v3 v3.23.12 // indirect github.com/shoenig/go-m1cpu v0.1.6 // indirect github.com/sirupsen/logrus v1.9.3 // indirect github.com/std-uritemplate/std-uritemplate/go v0.0.42 // indirect + github.com/testcontainers/testcontainers-go/modules/openldap v0.28.0 // indirect github.com/tklauser/go-sysconf v0.3.12 // indirect github.com/tklauser/numcpus v0.6.1 // indirect github.com/yusufpapurcu/wmi v1.2.3 // indirect + go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.45.0 // indirect go.opentelemetry.io/otel v1.19.0 // indirect go.opentelemetry.io/otel/metric v1.19.0 // indirect go.opentelemetry.io/otel/trace v1.19.0 // indirect @@ -90,15 +98,15 @@ require ( go.ytsaurus.tech/library/go/ptr v0.0.1 // indirect go.ytsaurus.tech/library/go/x/xreflect v0.0.2 // indirect go.ytsaurus.tech/library/go/x/xruntime v0.0.3 // indirect - golang.org/x/crypto v0.14.0 // indirect + golang.org/x/crypto v0.17.0 // indirect golang.org/x/exp v0.0.0-20230725093048-515e97ebf090 // indirect golang.org/x/mod v0.11.0 // indirect golang.org/x/net v0.17.0 // indirect - golang.org/x/sys v0.13.0 // indirect - golang.org/x/text v0.13.0 // indirect - golang.org/x/tools v0.7.0 // indirect + golang.org/x/sys v0.16.0 // indirect + golang.org/x/text v0.14.0 // indirect + golang.org/x/tools v0.10.0 // indirect golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2 // indirect - google.golang.org/genproto/googleapis/rpc v0.0.0-20230525234030-28d5490b6b19 // indirect - google.golang.org/grpc v1.57.1 // indirect + google.golang.org/genproto/googleapis/rpc v0.0.0-20230711160842-782d3b101e98 // indirect + google.golang.org/grpc v1.58.3 // indirect google.golang.org/protobuf v1.31.0 // indirect ) diff --git a/go.sum b/go.sum index 2eb4fa1..8df8c6e 100644 --- a/go.sum +++ b/go.sum @@ -9,6 +9,8 @@ github.com/Azure/azure-sdk-for-go/sdk/internal v1.3.0 h1:sXr+ck84g/ZlZUOZiNELInm github.com/Azure/azure-sdk-for-go/sdk/internal v1.3.0/go.mod h1:okt5dMMTOFjX/aovMlrjvvXoPMBVSPzk9185BT0+eZM= github.com/Azure/go-ansiterm v0.0.0-20210617225240-d185dfc1b5a1 h1:UQHMgLO+TxOElx5B5HZ4hJQsoJ/PvUvKRhJHDQXO8P8= github.com/Azure/go-ansiterm v0.0.0-20210617225240-d185dfc1b5a1/go.mod h1:xomTg63KZ2rFqZQzSB4Vz2SUXa1BpHTVz9L5PTmPC4E= +github.com/Azure/go-ntlmssp v0.0.0-20221128193559-754e69321358 h1:mFRzDkZVAjdal+s7s0MwaRv9igoPqLRdzOLzw/8Xvq8= +github.com/Azure/go-ntlmssp v0.0.0-20221128193559-754e69321358/go.mod h1:chxPXzSsl7ZWRAuOIE23GDNzjWuZquvFlgA8xmpunjU= github.com/AzureAD/microsoft-authentication-library-for-go v1.2.0 h1:hVeq+yCyUi+MsoO/CU95yqCIcdzra5ovzk8Q2BBpV2M= github.com/AzureAD/microsoft-authentication-library-for-go v1.2.0/go.mod h1:wP83P5OoQ5p6ip3ScPr0BAq0BvuPAvacpEuSzyouqAI= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= @@ -16,6 +18,9 @@ github.com/Microsoft/go-winio v0.6.1 h1:9/kr64B9VUZrLm5YYwbGtUJnMgqWVOdUAXu6Migc github.com/Microsoft/go-winio v0.6.1/go.mod h1:LRdKpFKfdobln8UmuiYcKPot9D2v6svN5+sAH+4kjUM= github.com/Microsoft/hcsshim v0.11.1 h1:hJ3s7GbWlGK4YVV92sO88BQSyF4ZLVy7/awqOlPxFbA= github.com/Microsoft/hcsshim v0.11.1/go.mod h1:nFJmaO4Zr5Y7eADdFOpYswDDlNVbvcIJJNJLECr5JQg= +github.com/Microsoft/hcsshim v0.11.4 h1:68vKo2VN8DE9AdN4tnkWnmdhqdbpUFM8OF3Airm7fz8= +github.com/Microsoft/hcsshim v0.11.4/go.mod h1:smjE4dvqPX9Zldna+t5FG3rnoHhaB7QYxPRqGcpAD9w= +github.com/alexbrainman/sspi v0.0.0-20210105120005-909beea2cc74/go.mod h1:cEWa1LVoE5KvSD9ONXsZrj0z6KqySlCCNKHlLzbqAt4= github.com/andybalholm/brotli v1.0.4 h1:V7DdXeJtZscaqfNuAdSRuRFzuiKlHSC/Zh3zl9qY3JY= github.com/andybalholm/brotli v1.0.4/go.mod h1:fO7iG3H7G2nSZ7m0zPUDn85XEX2GTukHGRSepvi9Eig= github.com/cenkalti/backoff/v4 v4.2.1 h1:y4OZtCnogmCPw98Zjyt5a6+QwPLGkiQsYW5oUqylYbM= @@ -27,6 +32,8 @@ github.com/cjlapao/common-go v0.0.39/go.mod h1:M3dzazLjTjEtZJbbxoA5ZDiGCiHmpwqW9 github.com/containerd/console v1.0.3/go.mod h1:7LqA/THxQ86k76b8c/EMSiaJ3h1eZkMkXar0TQ1gf3U= github.com/containerd/containerd v1.7.7 h1:QOC2K4A42RQpcrZyptP6z9EJZnlHfHJUfZrAAHe15q4= github.com/containerd/containerd v1.7.7/go.mod h1:3c4XZv6VeT9qgf9GMTxNTMFxGJrGpI2vz1yk4ye+YY8= +github.com/containerd/containerd v1.7.12 h1:+KQsnv4VnzyxWcfO9mlxxELaoztsDEjOuCMPAuPqgU0= +github.com/containerd/containerd v1.7.12/go.mod h1:/5OMpE1p0ylxtEUGY8kuCYkDRzJm9NO1TFMWjUpdevk= github.com/containerd/log v0.1.0 h1:TCJt7ioM2cr/tfR8GPbGf9/VRAX8D2B4PjzCpfX540I= github.com/containerd/log v0.1.0/go.mod h1:VRRf09a7mHDIRezVKTRCrOq78v577GXq3bSa3EhrzVo= github.com/coreos/go-systemd/v22 v22.3.2/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc= @@ -40,18 +47,30 @@ github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/deckarep/golang-set/v2 v2.3.1 h1:vjmkvJt/IV27WXPyYQpAh4bRyWJc5Y435D17XQ9QU5A= github.com/deckarep/golang-set/v2 v2.3.1/go.mod h1:VAky9rY/yGXJOLEDv3OMci+7wtDpOF4IN+y82NBOac4= +github.com/distribution/reference v0.5.0 h1:/FUIFXtfc/x2gpa5/VGfiGLuOIdYa1t65IKK2OFGvA0= +github.com/distribution/reference v0.5.0/go.mod h1:BbU0aIcezP1/5jX/8MP0YiH4SdvB5Y4f/wlDRiLyi3E= github.com/dnaeon/go-vcr v1.2.0 h1:zHCHvJYTMh1N7xnV7zf1m1GPBF9Ad0Jk/whtQ1663qI= github.com/docker/distribution v2.8.2+incompatible h1:T3de5rq0dB1j30rp0sA2rER+m322EBzniBPB6ZIzuh8= github.com/docker/distribution v2.8.2+incompatible/go.mod h1:J2gT2udsDAN96Uj4KfcMRqY0/ypR+oyYUYmja8H+y+w= github.com/docker/docker v24.0.6+incompatible h1:hceabKCtUgDqPu+qm0NgsaXf28Ljf4/pWFL7xjWWDgE= github.com/docker/docker v24.0.6+incompatible/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk= +github.com/docker/docker v25.0.2+incompatible h1:/OaKeauroa10K4Nqavw4zlhcDq/WBcPMc5DbjOGgozY= +github.com/docker/docker v25.0.2+incompatible/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk= github.com/docker/go-connections v0.4.0 h1:El9xVISelRB7BuFusrZozjnkIM5YnzCViNKohAFqRJQ= github.com/docker/go-connections v0.4.0/go.mod h1:Gbd7IOopHjR8Iph03tsViu4nIes5XhDvyHbTtUxmeec= +github.com/docker/go-connections v0.5.0 h1:USnMq7hx7gwdVZq1L49hLXaFtUdTADjXGp+uj1Br63c= +github.com/docker/go-connections v0.5.0/go.mod h1:ov60Kzw0kKElRwhNs9UlUHAE/F9Fe6GLaXnqyDdmEXc= github.com/docker/go-units v0.4.0/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk= github.com/docker/go-units v0.5.0 h1:69rxXcBk27SvSaaxTtLh/8llcHD8vYHT7WSdRZ/jvr4= github.com/docker/go-units v0.5.0/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk= +github.com/felixge/httpsnoop v1.0.3 h1:s/nj+GCswXYzN5v2DpNMuMQYe+0DDwt5WVCU6CWBdXk= +github.com/felixge/httpsnoop v1.0.3/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U= github.com/frankban/quicktest v1.11.3/go.mod h1:wRf/ReqHper53s+kmmSZizM8NamnL3IM0I9ntUbOk+k= github.com/frankban/quicktest v1.14.5 h1:dfYrrRyLtiqT9GyKXgdh+k4inNeTvmGbuSgZ3lx3GhA= +github.com/go-asn1-ber/asn1-ber v1.5.5 h1:MNHlNMBDgEKD4TcKr36vQN68BA00aDfjIt3/bD50WnA= +github.com/go-asn1-ber/asn1-ber v1.5.5/go.mod h1:hEBeB/ic+5LoWskz+yKT7vGhhPYkProFKoKdwZRWMe0= +github.com/go-ldap/ldap/v3 v3.4.6 h1:ert95MdbiG7aWo/oPYp9btL3KJlMPKnP58r09rI8T+A= +github.com/go-ldap/ldap/v3 v3.4.6/go.mod h1:IGMQANNtxpsOzj7uUAMjpGBaOVTC4DYyIy8VsTdxmtc= github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= github.com/go-logr/logr v1.2.4 h1:g01GSCwiDw2xSZfjJ2/T9M+S6pFdcNtFYsp+Y43HYDQ= github.com/go-logr/logr v1.2.4/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= @@ -77,10 +96,15 @@ github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/ github.com/google/go-cmp v0.5.6/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38= github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= +github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= +github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/google/tink/go v1.7.0 h1:6Eox8zONGebBFcCBqkVmt60LaWZa6xg1cl/DwAh/J1w= github.com/google/tink/go v1.7.0/go.mod h1:GAUOd+QE3pgj9q8VKIGTCP33c/B7eb4NhxLcgTJZStM= +github.com/google/uuid v1.3.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/google/uuid v1.4.0 h1:MtMxsa51/r9yyhkyLsVeVt0B+BGQZzpQiTQ4eHZ8bc4= github.com/google/uuid v1.4.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= +github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/jessevdk/go-flags v1.5.0 h1:1jKYvbxEjfUl0fmqTCOfonvskHHXMjBySTLW4y9LFvc= github.com/jessevdk/go-flags v1.5.0/go.mod h1:Fw0T6WPc1dYxT4mKEZRfG5kJhaTDP9pj1c2EWnYs/m4= github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= @@ -125,6 +149,8 @@ github.com/moby/patternmatcher v0.6.0/go.mod h1:hDPoyOpDY7OrrMDLaYoY3hf52gNCR/YO github.com/moby/sys/mountinfo v0.5.0/go.mod h1:3bMD3Rg+zkqx8MRYPi7Pyb0Ie97QEBmdxbhnCLlSvSU= github.com/moby/sys/sequential v0.5.0 h1:OPvI35Lzn9K04PBbCLW0g4LcFAJgHsvXsRyewg5lXtc= github.com/moby/sys/sequential v0.5.0/go.mod h1:tH2cOOs5V9MlPiXcQzRC+eEyab644PWKGRYaaV5ZZlo= +github.com/moby/sys/user v0.1.0 h1:WmZ93f5Ux6het5iituh9x2zAG7NFY9Aqi49jjE1PaQg= +github.com/moby/sys/user v0.1.0/go.mod h1:fKJhFOnsCN6xZ5gSfbM6zaHGgDJMrqt9/reuj4T7MmU= github.com/moby/term v0.5.0 h1:xt8Q1nalod/v7BqbG21f8mQPqH+xAaC9C3N3wfWbVP0= github.com/moby/term v0.5.0/go.mod h1:8FzsFHVUBGZdbDsJw/ot+X+d5HLUbvklYLJ9uGfcI3Y= github.com/morikuni/aec v1.0.0 h1:nP9CBfwrvYnBRgY6qfDQkygYDmYwOilePFkwzv4dU8A= @@ -155,6 +181,8 @@ github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQD github.com/seccomp/libseccomp-golang v0.9.2-0.20220502022130-f33da4d89646/go.mod h1:JA8cRccbGaA1s33RQf7Y1+q9gHmZX1yB/z9WDN1C6fg= github.com/shirou/gopsutil/v3 v3.23.9 h1:ZI5bWVeu2ep4/DIxB4U9okeYJ7zp/QLTO4auRb/ty/E= github.com/shirou/gopsutil/v3 v3.23.9/go.mod h1:x/NWSb71eMcjFIO0vhyGW5nZ7oSIgVjrCnADckb85GA= +github.com/shirou/gopsutil/v3 v3.23.12 h1:z90NtUkp3bMtmICZKpC4+WaknU1eXtp5vtbQ11DgpE4= +github.com/shirou/gopsutil/v3 v3.23.12/go.mod h1:1FrWgea594Jp7qmjHUUPlJDTPgcsb9mGnXDxavtikzM= github.com/shoenig/go-m1cpu v0.1.6 h1:nxdKQNcEB6vzgA2E2bvzKIYRuNj7XNJ4S/aRSwKzFtM= github.com/shoenig/go-m1cpu v0.1.6/go.mod h1:1JJMcUBvfNwpq05QDQVAnx3gUHr9IYF7GNg9SUEw2VQ= github.com/shoenig/test v0.6.4 h1:kVTaSd7WLz5WZ2IaoM0RSzRsUD+m8wRR+5qvntpn4LU= @@ -178,6 +206,10 @@ github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXl github.com/syndtr/gocapability v0.0.0-20200815063812-42c35b437635/go.mod h1:hkRG7XYTFWNJGYcbNJQlaLq0fg1yr4J4t/NcTQtrfww= github.com/testcontainers/testcontainers-go v0.26.0 h1:uqcYdoOHBy1ca7gKODfBd9uTHVK3a7UL848z09MVZ0c= github.com/testcontainers/testcontainers-go v0.26.0/go.mod h1:ICriE9bLX5CLxL9OFQ2N+2N+f+803LNJ1utJb1+Inx0= +github.com/testcontainers/testcontainers-go v0.28.0 h1:1HLm9qm+J5VikzFDYhOd+Zw12NtOl+8drH2E8nTY1r8= +github.com/testcontainers/testcontainers-go v0.28.0/go.mod h1:COlDpUXbwW3owtpMkEB1zo9gwb1CoKVKlyrVPejF4AU= +github.com/testcontainers/testcontainers-go/modules/openldap v0.28.0 h1:hR3BtDFSPUXvMZNuVbw7YcqAFsvjwt/VNA6ZTjtDZiM= +github.com/testcontainers/testcontainers-go/modules/openldap v0.28.0/go.mod h1:8/VlUrL0D5j0G3Vy3c+APgdBH3RZJcIETkualsRA8W4= github.com/tklauser/go-sysconf v0.3.12 h1:0QaGUFOdQaIVdPgfITYzaTegZvdCjmYO52cSFAEVmqU= github.com/tklauser/go-sysconf v0.3.12/go.mod h1:Ho14jnntGE1fpdOqQEEaiKRpvIavV0hSfmBq8nJbHYI= github.com/tklauser/numcpus v0.6.1 h1:ng9scYS7az0Bk4OZLvrNXNSAO2Pxr1XXRAPyjhIx+Fk= @@ -187,8 +219,11 @@ github.com/vishvananda/netlink v1.1.0/go.mod h1:cTgwzPIzzgDAYoQrMm0EdrjRUBkTqKYp github.com/vishvananda/netns v0.0.0-20191106174202-0a2b9b5464df/go.mod h1:JP3t17pCcGlemwknint6hfoeCVQrEMVwxRLRjXpq+BU= github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= github.com/yusufpapurcu/wmi v1.2.3 h1:E1ctvB7uKFMOJw3fdOW32DwGE9I7t++CRUEMKvFoFiw= github.com/yusufpapurcu/wmi v1.2.3/go.mod h1:SBZ9tNy3G9/m5Oi98Zks0QjeHVDvuK0qfxQmPyzfmi0= +go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.45.0 h1:x8Z78aZx8cOF0+Kkazoc7lwUNMGy0LrzEMxTm4BbTxg= +go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.45.0/go.mod h1:62CPTSry9QZtOaSsE3tOzhx6LzDhHnXJ6xHeMNNiM6Q= go.opentelemetry.io/otel v1.19.0 h1:MuS/TNf4/j4IXsZuJegVzI1cwut7Qc00344rgH7p8bs= go.opentelemetry.io/otel v1.19.0/go.mod h1:i0QyjOq3UPoTzff0PJB2N66fb4S0+rSbSB15/oyH9fY= go.opentelemetry.io/otel/metric v1.19.0 h1:aTzpGtV0ar9wlV4Sna9sdJyII5jTVJEvKETPiOKwvpE= @@ -220,12 +255,18 @@ go.ytsaurus.tech/yt/go v0.0.13/go.mod h1:r3puwhqLgS/TD9VZ8/3rhuxtdD02bQ+yuZWncNu golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= +golang.org/x/crypto v0.13.0/go.mod h1:y6Z2r+Rw4iayiXXAIxJIDAJ1zMW4yaTpebo8fPOliYc= golang.org/x/crypto v0.14.0 h1:wBqGXzWJW6m1XrIKlAH0Hs1JJ7+9KBwnIO8v66Q9cHc= golang.org/x/crypto v0.14.0/go.mod h1:MVFd36DqK4CsrnJYDkBA3VC4m2GkXAM0PvzMCn4JQf4= +golang.org/x/crypto v0.17.0 h1:r8bRNjWL3GshPW3gkd+RpvzWrZAwPS49OmTGZ/uhM4k= +golang.org/x/crypto v0.17.0/go.mod h1:gCAAfMLgwOJRpTjQ2zCCt2OcSfYMTeZVSRtQlPC7Nq4= golang.org/x/exp v0.0.0-20230725093048-515e97ebf090 h1:Di6/M8l0O2lCLc6VVRWhgCiApHV8MnQurBnFSHsQtNY= golang.org/x/exp v0.0.0-20230725093048-515e97ebf090/go.mod h1:FXUEEKJgO7OQYeo8N01OfiKP8RXMtf6e8aTskBGqWdc= golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= +golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= golang.org/x/mod v0.11.0 h1:bUO06HqtnRcc/7l71XBe4WcqTZ+3AH1J59zWDDwLKgU= golang.org/x/mod v0.11.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= @@ -233,11 +274,17 @@ golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLL golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/net v0.0.0-20201224014010-6772e930b67b/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= +golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= +golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= +golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= +golang.org/x/net v0.10.0/go.mod h1:0qNGK6F8kojg2nk9dLZ2mShWaEBan6FAoqfSigmmuDg= golang.org/x/net v0.17.0 h1:pVaXccu2ozPjCXewfr1S7xza/zcXTity9cCdXQYSjIM= golang.org/x/net v0.17.0/go.mod h1:NxSsAGuq816PNPmqtQdLE42eU2Fs7NoRIZrHJAlaCOE= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.3.0 h1:ftCYgMx6zT/asHUrPw8BLLscYtGznsLAnjq5RH9P66E= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -250,29 +297,49 @@ golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20201204225414-ed752295db88/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210320140829-1e4c9ba3b0c4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210616045830-e2b7044e8c71/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210616094352-59db8d763f22/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210906170528-6f6e22806c34/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20211025201205-69cdffdb9359/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20211116061358-0a5406a5449c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.11.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.12.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.13.0 h1:Af8nKPmuFypiUBjVoU9V20FiaFXOcuZI21p0ycVYYGE= golang.org/x/sys v0.13.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.15.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/sys v0.16.0 h1:xWw16ngr6ZMtmxDyKyIgsE93KNKz5HKmMa3b8ALHidU= +golang.org/x/sys v0.16.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= +golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= +golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k= +golang.org/x/term v0.8.0/go.mod h1:xPskH00ivmX89bAKVGSKKtLOWNx2+17Eiy94tnKShWo= +golang.org/x/term v0.12.0/go.mod h1:owVbMEjm3cBLCHdkQu9b1opXd4ETQWc3BhuQGKgXgvU= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= +golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= +golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= golang.org/x/text v0.13.0 h1:ablQoSUd0tRdKxZewP80B+BaqeKJuVhuRxj/dkrun3k= golang.org/x/text v0.13.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= +golang.org/x/text v0.14.0 h1:ScX5w1eTa3QqT8oi6+ziP7dTV1S2+ALU0bI+0zXKWiQ= +golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= golang.org/x/time v0.3.0 h1:rg5rLMjNzMS1RkNLzCG38eapWhnYLFYXDXj2gOlr8j4= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= +golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= +golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU= golang.org/x/tools v0.7.0 h1:W4OVu8VVOaIO0yzWMNdepAulS7YfoS3Zabrm8DOXXU4= golang.org/x/tools v0.7.0/go.mod h1:4pg6aUX35JBAogB10C9AtvVL+qowtN4pT3CGSQex14s= +golang.org/x/tools v0.10.0 h1:tvDr/iQoUqNdohiYm0LmmKcBk+q86lb9EprIUFhHHGg= +golang.org/x/tools v0.10.0/go.mod h1:UJwyiVBsOA2uwvK/e5OY3GTpDUJriEd+/YlqAwLPmyM= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= @@ -281,8 +348,12 @@ golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2 h1:H2TDz8ibqkAF6YGhCdN3j golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2/go.mod h1:K8+ghG5WaK9qNqU5K3HdILfMLy1f3aNYFI/wnl100a8= google.golang.org/genproto/googleapis/rpc v0.0.0-20230525234030-28d5490b6b19 h1:0nDDozoAU19Qb2HwhXadU8OcsiO/09cnTqhUtq2MEOM= google.golang.org/genproto/googleapis/rpc v0.0.0-20230525234030-28d5490b6b19/go.mod h1:66JfowdXAEgad5O9NnYcsNPLCPZJD++2L9X0PCMODrA= +google.golang.org/genproto/googleapis/rpc v0.0.0-20230711160842-782d3b101e98 h1:bVf09lpb+OJbByTj913DRJioFFAjf/ZGxEz7MajTp2U= +google.golang.org/genproto/googleapis/rpc v0.0.0-20230711160842-782d3b101e98/go.mod h1:TUfxEVdsvPg18p6AslUXFoLdpED4oBnGwyqk3dV1XzM= google.golang.org/grpc v1.57.1 h1:upNTNqv0ES+2ZOOqACwVtS3Il8M12/+Hz41RCPzAjQg= google.golang.org/grpc v1.57.1/go.mod h1:Sd+9RMTACXwmub0zcNY2c4arhtrbBYD1AUHI/dt16Mo= +google.golang.org/grpc v1.58.3 h1:BjnpXut1btbtgN/6sp+brB2Kbm2LjNXnidYujAVbSoQ= +google.golang.org/grpc v1.58.3/go.mod h1:tgX3ZQDlNJGU96V6yHh1T/JeoBQ2TXdr43YbYSsCJk0= google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= google.golang.org/protobuf v1.27.1/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= diff --git a/ldap.go b/ldap.go new file mode 100644 index 0000000..d9db27b --- /dev/null +++ b/ldap.go @@ -0,0 +1,82 @@ +package main + +import ( + "github.com/go-ldap/ldap/v3" + "k8s.io/utils/env" + "log" +) + +type Ldap struct { + Connection *ldap.Conn + Config *LdapConfig +} + +func NewLdap(cfg *LdapConfig, logger appLoggerType) (*Ldap, error) { + conn, err := ldap.DialURL(cfg.Address) + if err != nil { + log.Fatalf("Failed to connect: %s\n", err) + } + + _, err = conn.SimpleBind(&ldap.SimpleBindRequest{ + Username: cfg.BindDN, + Password: env.GetString(cfg.BindPasswordEnvVar, "adminpassword"), + }) + return &Ldap{ + Connection: conn, + Config: cfg, + }, nil +} + +func (source *Ldap) GetUsers() ([]SourceUser, error) { + res, err := source.Connection.Search(&ldap.SearchRequest{ + BaseDN: source.Config.BaseDN, + Filter: source.Config.Users.Filter, + Attributes: []string{"*"}, + Scope: ldap.ScopeWholeSubtree, + }) + if err != nil { + return nil, err + } + + var users []SourceUser + for _, entry := range res.Entries { + username := entry.GetAttributeValue(source.Config.Users.UsernameAttributeType) + uid := entry.GetAttributeValue(source.Config.Users.UidAttributeType) + var firstName string + if source.Config.Users.FirstNameAttributeType != nil { + firstName = entry.GetAttributeValue(*source.Config.Users.FirstNameAttributeType) + } + users = append(users, LdapUser{ + BasicSourceUser: BasicSourceUser{SourceType: LdapSourceType}, + Username: username, + Uid: uid, + FirstName: firstName}) + } + return users, nil +} + +func (source *Ldap) GetGroupsWithMembers() ([]SourceGroupWithMembers, error) { + res, err := source.Connection.Search(&ldap.SearchRequest{ + BaseDN: source.Config.BaseDN, + Filter: source.Config.Groups.Filter, + Attributes: []string{"*"}, + Scope: ldap.ScopeWholeSubtree, + }) + if err != nil { + return nil, err + } + + var groups []SourceGroupWithMembers + for _, entry := range res.Entries { + groupname := entry.GetAttributeValue(source.Config.Groups.GroupnameAttributeType) + members := entry.GetAttributeValues(source.Config.Groups.MemberUidAttributeType) + groups = append(groups, SourceGroupWithMembers{ + SourceGroup: LdapGroup{ + BasicSourceGroup: BasicSourceGroup{SourceType: LdapSourceType}, + Groupname: groupname, + }, + Members: NewStringSetFromItems(members...), + }) + } + return groups, nil +} diff --git a/ldap_config.example.yaml b/ldap_config.example.yaml new file mode 100644 index 0000000..3b5ca10 --- /dev/null +++ b/ldap_config.example.yaml @@ -0,0 +1,40 @@ +app: + sync_interval: 5m + username_replacements: + - from: "@acme.com" + to: "" + - from: "@" + to: ":" + groupname_replacements: + - from: "|all" + to: "" + remove_limit: 10 + ban_before_remove_duration: 168h # 7d + +ldap: + address: "localhost:10210" + bind_dn: "cn=admin,dc=example,dc=org" + bind_password_env_var: "LDAP_PASSWORD" + base_dn: "dc=example,dc=org" + timeout: 1s + users: + filter: "(&(objectClass=posixAccount)(ou=People))" + username_attribute_type: "cn" + uid_attribute_type: "uid" + first_name_attribute_type: "givenName" + groups: + filter: "(objectClass=posixGroup)" + groupname_attribute_type: "cn" + member_uid_attribute_type: "memberUid" + +ytsaurus: + proxy: localhost:10110 + apply_user_changes: true + apply_group_changes: true + apply_member_changes: true + timeout: 1s + log_level: DEBUG + +logging: + level: WARN + is_production: true diff --git a/main.go b/main.go index 67f97af..0b36f24 100644 --- a/main.go +++ b/main.go @@ -2,7 +2,6 @@ package main import ( "fmt" - "github.com/jessevdk/go-flags" "github.com/pkg/errors" "go.uber.org/zap" @@ -36,7 +35,7 @@ func run(configFilePath string) error { return errors.Wrapf(err, "failed to load config %s", configFilePath) } - logger, err := configureLogger(cfg.Logging) + logger, err := configureLogger(&cfg.Logging) if err != nil { return errors.Wrapf(err, "failed to configure logging %+v", cfg.Logging) } diff --git a/migration_integration_test.go b/migration_integration_test.go index d54e763..fded182 100644 --- a/migration_integration_test.go +++ b/migration_integration_test.go @@ -61,7 +61,7 @@ func TestUsersMigration(t *testing.T) { t.Log("Got", len(ytUsersToUpdate), "users to update") for username, user := range ytUsersToUpdate { - attrValue := buildUserAzureAttributeValue(user) + attrValue := user.SourceUser if dryRun { t.Log("[DRY-RUN] will set @azure=", attrValue, "for", username) @@ -73,6 +73,7 @@ func TestUsersMigration(t *testing.T) { context.Background(), yt.client, username, + user.GetSourceAttributeName(), attrValue, ) require.NoError(t, err) @@ -125,7 +126,7 @@ func TestGroupsMigration(t *testing.T) { t.Log("Got", len(ytGroupsToUpdate), "groups to update") for groupname, group := range ytGroupsToUpdate { - attrValue := buildGroupAzureAttributeValue(group) + attrValue := group if dryRun { t.Log("[DRY-RUN] will set @azure=", attrValue, "for", groupname) continue @@ -136,6 +137,7 @@ func TestGroupsMigration(t *testing.T) { context.Background(), yt.client, groupname, + group.GetSourceAttributeName(), attrValue, ) require.NoError(t, err) diff --git a/source_models.go b/source_models.go new file mode 100644 index 0000000..44c6929 --- /dev/null +++ b/source_models.go @@ -0,0 +1,177 @@ +package main + +import ( + "fmt" + "github.com/pkg/errors" + "go.ytsaurus.tech/yt/go/yson" +) + +type ObjectID = string +type SourceType string + +const ( + LdapSourceType SourceType = "ldap" + AzureSourceType SourceType = "azure" +) + +type SourceUser interface { + GetId() ObjectID + GetName() string + GetSourceType() SourceType +} + +func NewSourceUser(attributes map[string]any) (SourceUser, error) { + bytes, err := yson.Marshal(attributes) + if err != nil { + return nil, err + } + + sourceType := attributes["source_type"] + if sourceType == string(LdapSourceType) { + var ldapUser LdapUser + err = yson.Unmarshal(bytes, &ldapUser) + if err != nil { + return nil, err + } + return ldapUser, nil + } else if sourceType == string(AzureSourceType) { + var azureUser AzureUser + err = yson.Unmarshal(bytes, &azureUser) + if err != nil { + return nil, err + } + return azureUser, nil + } else { + return nil, errors.New(fmt.Sprintf("Unknown source type: %v", sourceType)) + } +} + +type BasicSourceUser struct { + SourceType SourceType `yson:"source_type"` +} + +type AzureUser struct { + BasicSourceUser + // PrincipalName is unique human-readable Azure user field, used (possibly with changes) + // for the corresponding YTsaurus user's `name` attribute. + PrincipalName string `yson:"principal_name"` + + AzureID ObjectID `yson:"id"` + Email string `yson:"email"` + FirstName string `yson:"first_name"` + LastName string `yson:"last_name"` + DisplayName string `yson:"display_name"` +} + +func (user AzureUser) GetId() ObjectID { + return user.AzureID +} + +func (user AzureUser) GetName() string { + return user.PrincipalName +} + +func (user AzureUser) GetSourceType() SourceType { + return AzureSourceType +} + +type LdapUser struct { + BasicSourceUser + Username string `yson:"username"` + Uid string `yson:"uid"` + FirstName string `yson:"first_name"` + // TODO(nadya73): Add more fields. +} + +func (user LdapUser) GetId() ObjectID { + return user.Uid +} + +func (user LdapUser) GetName() string { + return user.Username +} + +func (user LdapUser) GetSourceType() SourceType { + return LdapSourceType +} + +type SourceGroup interface { + GetId() ObjectID + GetName() string + GetSourceType() SourceType +} + +type BasicSourceGroup struct { + SourceType SourceType `yson:"source_type"` +} + +func NewSourceGroup(attributes map[string]any) (SourceGroup, error) { + bytes, err := yson.Marshal(attributes) + if err != nil { + return nil, err + } + + sourceType := attributes["source_type"] + if sourceType == string(LdapSourceType) { + var ldapGroup LdapGroup + err = yson.Unmarshal(bytes, &ldapGroup) + if err != nil { + return nil, err + } + return ldapGroup, nil + } else if sourceType == string(AzureSourceType) { + var azureGroup AzureGroup + err = yson.Unmarshal(bytes, &azureGroup) + if err != nil { + return nil, err + } + return azureGroup, nil + } else { + return nil, errors.New(fmt.Sprintf("Unknown source type: %v", sourceType)) + } +} + +type AzureGroup struct { + BasicSourceGroup + // Identity is unique human-readable Source user field, used (possibly with changes) + // for the corresponding YTsaurus user's `name` attribute. + Identity string `yson:"identity"` + + AzureID ObjectID `yson:"id"` + DisplayName string `yson:"display_name"` +} + +func (ag AzureGroup) GetId() ObjectID { + return ag.AzureID +} + +func (ag AzureGroup) GetName() string { + return ag.Identity +} + +func (ag AzureGroup) GetSourceType() SourceType { + return AzureSourceType +} + +type LdapGroup struct { + BasicSourceGroup + Groupname string `yson:"groupname"` +} + +func (lg LdapGroup) GetId() ObjectID { + return lg.Groupname +} + +func (lg LdapGroup) GetName() string { + return lg.Groupname +} + +func (lg LdapGroup) GetSourceType() SourceType { + return LdapSourceType +} + +type SourceGroupWithMembers struct { + SourceGroup SourceGroup + // Members is a set of strings, representing users' ObjectID. + Members StringSet +} diff --git a/testcontainer_openldap.go b/testcontainer_openldap.go new file mode 100644 index 0000000..0af388a --- /dev/null +++ b/testcontainer_openldap.go @@ -0,0 +1,64 @@ +package main + +import ( + "context" + "errors" + "github.com/testcontainers/testcontainers-go" + "github.com/testcontainers/testcontainers-go/modules/openldap" + "go.ytsaurus.tech/library/go/ptr" +) + +type OpenLdapLocal struct { + container *openldap.OpenLDAPContainer +} + +func NewOpenLdapLocal() *OpenLdapLocal { + return &OpenLdapLocal{} +} + +func (y *OpenLdapLocal) Start() error { + ctx := context.Background() + container, err := openldap.RunContainer(ctx, testcontainers.WithImage("bitnami/openldap:2.6.6")) + if err != nil { + return err + } + y.container = container + return y.container.Start(ctx) +} + +func (y *OpenLdapLocal) GetConfig() (*LdapConfig, error) { + connectionString, err := y.container.ConnectionString(context.Background()) + if err != nil { + return nil, err + } + return &LdapConfig{ + Address: connectionString, //"ldap://localhost:1389", + BaseDN: "dc=example,dc=org", + BindDN: "cn=admin,dc=example,dc=org", + BindPasswordEnvVar: "LDAP_PASSWORD", + Users: LdapUsersConfig{ + Filter: "(&(objectClass=posixAccount)(ou=People))", + UsernameAttributeType: "cn", + UidAttributeType: "uid", + FirstNameAttributeType: ptr.String("givenName"), + }, + Groups: LdapGroupsConfig{ + Filter: "(objectClass=posixGroup)", + GroupnameAttributeType: "cn", + MemberUidAttributeType: "memberUid", + }, + }, nil +} + +func (y *OpenLdapLocal) Stop() error { + ctx := context.Background() + if y.container == nil { + return errors.New("container not started") + } + err := y.container.Terminate(ctx) + if err != nil { + return err + } + y.container = nil + return nil +} diff --git a/util.go b/util.go index 1d33be7..1331ff1 100644 --- a/util.go +++ b/util.go @@ -12,7 +12,10 @@ import ( ) const ( - appTimeFormat = "2006-01-02T15:04:05Z0700" + appTimeFormat = "2006-01-02T15:04:05Z0700" + defaultAzureTimeout = 3 * time.Second + defaultAzureSecretEnvVar = "AZURE_CLIENT_SECRET" + defaultAppRemoveLimit = 10 ) type appLoggerType = *zap.SugaredLogger diff --git a/ytsaurus.go b/ytsaurus.go index 20ae4d6..87ef20e 100644 --- a/ytsaurus.go +++ b/ytsaurus.go @@ -109,12 +109,13 @@ func (y *Ytsaurus) CreateUser(user YtsaurusUser) error { defer cancel() y.maybePrintExtraLogs(user.Username, "create_user", "user", user) + return doCreateYtsaurusUser( ctx, y.client, user.Username, map[string]any{ - "azure": buildUserAzureAttributeValue(user), + user.GetSourceAttributeName(): user.SourceUser, }, ) } @@ -233,7 +234,7 @@ func (y *Ytsaurus) CreateGroup(group YtsaurusGroup) error { y.client, group.Name, map[string]any{ - "azure": buildGroupAzureAttributeValue(group), + group.GetSourceAttributeName(): group.SourceGroup, }, ) } @@ -332,12 +333,17 @@ func (y *Ytsaurus) isUserManaged(username string) (bool, error) { ctx, cancel := context.WithTimeout(context.Background(), y.timeout) defer cancel() - attrExists, err := y.client.NodeExists( + attrAzureExists, err := y.client.NodeExists( ctx, ypath.Path("//sys/users/"+username+"/@azure"), nil, ) - return attrExists, err + attrSourceExists, err := y.client.NodeExists( + ctx, + ypath.Path("//sys/users/"+username+"/@source"), + nil, + ) + return attrAzureExists || attrSourceExists, err } func (y *Ytsaurus) ensureUserManaged(username string) error { @@ -355,12 +361,18 @@ func (y *Ytsaurus) isGroupManaged(name string) (bool, error) { ctx, cancel := context.WithTimeout(context.Background(), y.timeout) defer cancel() - attrExists, err := y.client.NodeExists( + attrAzureExists, err := y.client.NodeExists( ctx, ypath.Path("//sys/groups/"+name+"/@azure"), nil, ) - return attrExists, err + attrSourceExists, err := y.client.NodeExists( + ctx, + ypath.Path("//sys/groups/"+name+"/@source"), + nil, + ) + + return attrAzureExists || attrSourceExists, err } func (y *Ytsaurus) ensureGroupManaged(groupname string) error { diff --git a/ytsaurus_helpers.go b/ytsaurus_helpers.go index 6b23953..39d565f 100644 --- a/ytsaurus_helpers.go +++ b/ytsaurus_helpers.go @@ -14,10 +14,11 @@ import ( func doGetAllYtsaurusUsers(ctx context.Context, client yt.Client) ([]YtsaurusUser, error) { type YtsaurusUserResponse struct { - Name string `yson:",value"` - Azure map[string]string `yson:"azure,attr"` - Banned bool `yson:"banned,attr"` - BannedSince string `yson:"banned_since,attr"` + Name string `yson:",value"` + Azure *AzureUser `yson:"azure,attr"` + Source map[string]any `yson:"source,attr"` + Banned bool `yson:"banned,attr"` + BannedSince string `yson:"banned_since,attr"` } var response []YtsaurusUserResponse @@ -30,6 +31,7 @@ func doGetAllYtsaurusUsers(ctx context.Context, client yt.Client) ([]YtsaurusUse "azure", "banned", "banned_since", + "source", }, }, ) @@ -46,15 +48,20 @@ func doGetAllYtsaurusUsers(ctx context.Context, client yt.Client) ([]YtsaurusUse return nil, errors.Wrapf(err, "failed to parse @banned_since. %v", ytUser) } } + var sourceUser SourceUser + if ytUser.Azure != nil { + sourceUser = *ytUser.Azure + } else if ytUser.Source != nil { + sourceUser, err = NewSourceUser(ytUser.Source) + if err != nil { + return nil, errors.Wrapf(err, "failed to create source user. %v", ytUser) + } + } + users = append(users, YtsaurusUser{ - Username: ytUser.Name, - AzureID: ytUser.Azure["id"], - PrincipalName: ytUser.Azure["principal_name"], - Email: ytUser.Azure["email"], - FirstName: ytUser.Azure["first_name"], - LastName: ytUser.Azure["last_name"], - DisplayName: ytUser.Azure["display_name"], - BannedSince: bannedSince, + Username: ytUser.Name, + SourceUser: sourceUser, + BannedSince: bannedSince, }) } return users, nil @@ -62,9 +69,10 @@ func doGetAllYtsaurusUsers(ctx context.Context, client yt.Client) ([]YtsaurusUse func doGetAllYtsaurusGroupsWithMembers(ctx context.Context, client yt.Client) ([]YtsaurusGroupWithMembers, error) { type YtsaurusGroupReponse struct { - Name string `yson:",value"` - Azure map[string]string `yson:"azure,attr"` - Members []string `yson:"members,attr"` + Name string `yson:",value"` + Azure *AzureGroup `yson:"azure,attr"` + Source map[string]any `yson:"source,attr"` + Members []string `yson:"members,attr"` } var response []YtsaurusGroupReponse @@ -73,29 +81,39 @@ func doGetAllYtsaurusGroupsWithMembers(ctx context.Context, client yt.Client) ([ ypath.Path("//sys/groups"), &response, &yt.ListNodeOptions{ - Attributes: []string{"members", "azure"}, + Attributes: []string{"members", "azure", "source"}, }, ) if err != nil { return nil, err } - var users []YtsaurusGroupWithMembers + var groups []YtsaurusGroupWithMembers for _, ytGroup := range response { members := NewStringSet() for _, m := range ytGroup.Members { members.Add(m) } - users = append(users, YtsaurusGroupWithMembers{ + + var sourceGroup SourceGroup + if ytGroup.Azure != nil { + sourceGroup = *ytGroup.Azure + } else if ytGroup.Source != nil { + sourceGroup, err = NewSourceGroup(ytGroup.Source) + if err != nil { + return nil, errors.Wrapf(err, "failed to create source group. %v", ytGroup) + } + } + + groups = append(groups, YtsaurusGroupWithMembers{ YtsaurusGroup: YtsaurusGroup{ Name: ytGroup.Name, - AzureID: ytGroup.Azure["id"], - DisplayName: ytGroup.Azure["display_name"], + SourceGroup: sourceGroup, }, Members: members, }) } - return users, nil + return groups, nil } func doCreateYtsaurusUser(ctx context.Context, client yt.Client, username string, attrs map[string]any) error { @@ -142,48 +160,27 @@ func doRemoveMemberYtsaurusGroup(ctx context.Context, client yt.Client, username ) } -func buildUserAzureAttributeValue(user YtsaurusUser) map[string]string { - return map[string]string{ - "id": user.AzureID, - "email": user.Email, - "principal_name": user.PrincipalName, - "first_name": user.FirstName, - "last_name": user.LastName, - "display_name": user.DisplayName, - } -} - func buildUserAttributes(user YtsaurusUser) map[string]any { return map[string]any{ - "azure": buildUserAzureAttributeValue(user), - "name": user.Username, - "banned_since": user.BannedSinceString(), - "banned": user.IsBanned(), - } -} - -func buildGroupAzureAttributeValue(group YtsaurusGroup) map[string]string { - return map[string]string{ - "id": group.AzureID, - "display_name": group.DisplayName, + "name": user.Username, + "banned_since": user.BannedSinceString(), + "banned": user.IsBanned(), + user.GetSourceAttributeName(): user.SourceUser, } } func buildGroupAttributes(group YtsaurusGroup) map[string]any { return map[string]any{ - "azure": map[string]string{ - "id": group.AzureID, - "display_name": group.DisplayName, - }, - "name": group.Name, + group.GetSourceAttributeName(): group.SourceGroup, + "name": group.Name, } } // nolint: unused -func doSetAzureAttributeForYtsaurusUser(ctx context.Context, client yt.Client, username string, attrValue map[string]string) error { +func doSetAzureAttributeForYtsaurusUser(ctx context.Context, client yt.Client, username string, attrName string, attrValue any) error { return client.SetNode( ctx, - ypath.Path("//sys/users/"+username+"/@azure"), + ypath.Path("//sys/users/"+username+"/@"+attrName), attrValue, nil, ) @@ -210,10 +207,16 @@ func doSetAttributesForYtsaurusUser(ctx context.Context, client yt.Client, usern } // nolint: unused -func doSetAzureAttributeForYtsaurusGroup(ctx context.Context, client yt.Client, groupname string, attrValue map[string]string) error { +func doSetAzureAttributeForYtsaurusGroup( + ctx context.Context, + client yt.Client, + groupname string, + attrName string, + attrValue map[string]string, +) error { return client.SetNode( ctx, - ypath.Path("//sys/groups/"+groupname+"/@azure"), + ypath.Path("//sys/groups/"+groupname+"/@"+attrName), attrValue, nil, ) diff --git a/ytsaurus_models.go b/ytsaurus_models.go index 86585e2..ace5478 100644 --- a/ytsaurus_models.go +++ b/ytsaurus_models.go @@ -6,23 +6,24 @@ import ( type YtsaurusUser struct { // Username is a unique @name attribute of a user. - Username string - // AzureID is non-human readable string like 2cd8a70c-9044-4488-b06a-c8461c39b296. - AzureID string - // PrincipalName is a unique human-readable login. - // It could be in form of email, but doesn't give guarantee that such email exists. - PrincipalName string - // Email is filled if Azure user has email. - Email string - FirstName string - LastName string - DisplayName string + Username string + SourceUser SourceUser BannedSince time.Time } +func (u YtsaurusUser) GetSourceAttributeName() string { + switch u.SourceUser.GetSourceType() { + case AzureSourceType: + return "azure" + case LdapSourceType: + return "source" + } + return "source" +} + // IsManuallyManaged true if user doesn't have @azure attribute (system or manually created user). func (u YtsaurusUser) IsManuallyManaged() bool { - return u.AzureID == "" + return u.SourceUser == nil } func (u YtsaurusUser) IsBanned() bool { @@ -39,8 +40,17 @@ func (u YtsaurusUser) BannedSinceString() string { type YtsaurusGroup struct { // Name is a unique @name attribute of a group. Name string - AzureID string - DisplayName string + SourceGroup SourceGroup +} + +func (g YtsaurusGroup) GetSourceAttributeName() string { + switch g.SourceGroup.GetSourceType() { + case AzureSourceType: + return "azure" + case LdapSourceType: + return "source" + } + return "source" } type YtsaurusGroupWithMembers struct { @@ -60,5 +70,5 @@ type YtsaurusMembership struct { // IsManuallyManaged true if group doesn't have @azure attribute (system or manually created group). func (u YtsaurusGroup) IsManuallyManaged() bool { - return u.AzureID == "" + return u.SourceGroup == nil } diff --git a/ytsaurus_test.go b/ytsaurus_test.go index 00af797..76090c6 100644 --- a/ytsaurus_test.go +++ b/ytsaurus_test.go @@ -41,15 +41,24 @@ func TestUpdateUserFirstName(t *testing.T) { defer func() { require.NoError(t, ytLocal.Stop()) }() yt := getYtsaurus(t, ytLocal) + const azureId = "fake-az-id-old" + managedOleg := YtsaurusUser{ - Username: "oleg", - AzureID: "fake-az-id-oleg", - FirstName: "Lego", + Username: "oleg", + SourceUser: AzureUser{ + AzureID: azureId, + FirstName: "Lego", + }, } err := yt.CreateUser(managedOleg) require.NoError(t, err) - managedOleg.FirstName = "Oleg" + updateSourceUser := AzureUser{ + AzureID: azureId, + FirstName: "Oleg", + } + managedOleg.SourceUser = updateSourceUser + updErr := yt.UpdateUser(managedOleg.Username, managedOleg) ytClient, err := ytLocal.GetClient() @@ -79,15 +88,19 @@ func TestGroups(t *testing.T) { managedOleg := YtsaurusUser{ Username: "oleg", - AzureID: "fake-az-id-oleg", + SourceUser: AzureUser{ + AzureID: "fake-az-id-oleg", + }, } err = yt.CreateUser(managedOleg) require.NoError(t, err) managedOlegsGroup := YtsaurusGroup{ - Name: "olegs", - AzureID: "fake-az-id-olegs", - DisplayName: "This is group is for Olegs only", + Name: "olegs", + SourceGroup: AzureGroup{ + AzureID: "fake-az-id-olegs", + DisplayName: "This is group is for Olegs only", + }, } err = yt.CreateGroup(managedOlegsGroup) require.NoError(t, err) @@ -102,9 +115,11 @@ func TestGroups(t *testing.T) { require.Equal(t, []YtsaurusGroupWithMembers{ { YtsaurusGroup: YtsaurusGroup{ - Name: managedOlegsGroup.Name, - AzureID: managedOlegsGroup.AzureID, - DisplayName: managedOlegsGroup.DisplayName, + Name: managedOlegsGroup.Name, + SourceGroup: AzureGroup{ + AzureID: managedOlegsGroup.SourceGroup.(AzureGroup).AzureID, + DisplayName: managedOlegsGroup.SourceGroup.(AzureGroup).DisplayName, + }, }, Members: members, }, @@ -118,9 +133,11 @@ func TestGroups(t *testing.T) { require.Equal(t, []YtsaurusGroupWithMembers{ { YtsaurusGroup: YtsaurusGroup{ - Name: managedOlegsGroup.Name, - AzureID: managedOlegsGroup.AzureID, - DisplayName: managedOlegsGroup.DisplayName, + Name: managedOlegsGroup.Name, + SourceGroup: AzureGroup{ + AzureID: managedOlegsGroup.SourceGroup.(AzureGroup).AzureID, + DisplayName: managedOlegsGroup.SourceGroup.(AzureGroup).DisplayName, + }, }, Members: NewStringSet(), },