From b775be632022cb2d2ef1bc2103d1ff5a81ad1c27 Mon Sep 17 00:00:00 2001 From: Nadezhda Savchenko Date: Thu, 22 Feb 2024 13:16:29 +0300 Subject: [PATCH 01/21] Introduce sync from ldap --- app.go | 50 +- app_integration_test.go | 2 +- app_test.go | 899 +++++++++++++++++++-------------- azure_fake.go | 12 +- azure_real.go | 61 +-- azure_real_integration_test.go | 4 +- config.go | 71 ++- config_test.go | 9 +- diff.go | 160 +++--- go.mod | 36 +- go.sum | 71 +++ ldap.go | 82 +++ ldap_config.example.yaml | 40 ++ main.go | 3 +- migration_integration_test.go | 6 +- source_models.go | 177 +++++++ testcontainer_openldap.go | 64 +++ util.go | 5 +- ytsaurus.go | 24 +- ytsaurus_helpers.go | 107 ++-- ytsaurus_models.go | 40 +- ytsaurus_test.go | 45 +- 22 files changed, 1315 insertions(+), 653 deletions(-) create mode 100644 ldap.go create mode 100644 ldap_config.example.yaml create mode 100644 source_models.go create mode 100644 testcontainer_openldap.go diff --git a/app.go b/app.go index 2d92cde..a198df9 100644 --- a/app.go +++ b/app.go @@ -1,6 +1,7 @@ package main import ( + "github.com/pkg/errors" "os" "os/signal" "syscall" @@ -9,20 +10,20 @@ import ( "k8s.io/utils/clock" ) -type Azure interface { - GetUsers() ([]AzureUser, error) - GetGroupsWithMembers() ([]AzureGroupWithMembers, error) +type Source interface { + GetUsers() ([]SourceUser, error) + GetGroupsWithMembers() ([]SourceGroupWithMembers, error) } type App struct { - syncInterval time.Duration + syncInterval *time.Duration usernameReplaces []ReplacementPair groupnameReplaces []ReplacementPair - removeLimit int - banDuration time.Duration + removeLimit *int + banDuration *time.Duration ytsaurus *Ytsaurus - azure Azure + source Source stopCh chan struct{} sigCh chan os.Signal @@ -30,17 +31,32 @@ type App struct { } func NewApp(cfg *Config, logger appLoggerType) (*App, error) { - azure, err := NewAzureReal(cfg.Azure, logger) - if err != nil { - return nil, err + if cfg.Azure == nil && cfg.Ldap == nil { + return nil, errors.New("no source (source or ldap) is specified") + } + + var err error + var source Source + if cfg.Azure != nil { + source, err = NewAzureReal(cfg.Azure, logger) + if err != nil { + return nil, err + } + } + + if cfg.Ldap != nil { + source, err = NewLdap(cfg.Ldap, logger) + if err != nil { + return nil, err + } } - return NewAppCustomized(cfg, logger, azure, clock.RealClock{}) + return NewAppCustomized(cfg, logger, source, clock.RealClock{}) } // NewAppCustomized used in tests. -func NewAppCustomized(cfg *Config, logger appLoggerType, azure Azure, clock clock.PassiveClock) (*App, error) { - yt, err := NewYtsaurus(cfg.Ytsaurus, logger, clock) +func NewAppCustomized(cfg *Config, logger appLoggerType, source Source, clock clock.PassiveClock) (*App, error) { + yt, err := NewYtsaurus(&cfg.Ytsaurus, logger, clock) if err != nil { return nil, err } @@ -56,7 +72,7 @@ func NewAppCustomized(cfg *Config, logger appLoggerType, azure Azure, clock cloc banDuration: cfg.App.BanBeforeRemoveDuration, ytsaurus: yt, - azure: azure, + source: source, stopCh: make(chan struct{}), sigCh: sigCh, @@ -66,8 +82,8 @@ func NewAppCustomized(cfg *Config, logger appLoggerType, azure Azure, clock cloc func (a *App) Start() { a.logger.Info("Starting the application") - if a.syncInterval > 0 { - ticker := time.NewTicker(a.syncInterval) + if a.syncInterval != nil && *a.syncInterval > 0 { + ticker := time.NewTicker(*a.syncInterval) for { select { case <-a.stopCh: @@ -83,7 +99,7 @@ func (a *App) Start() { } } else { a.logger.Info( - "app.sync_interval config variable is not greater than zero, " + + "app.sync_interval config variable is not specified or is not greater than zero, " + "auto sync is disabled. Send SIGUSR1 for manual sync.", ) for { diff --git a/app_integration_test.go b/app_integration_test.go index 63a3835..fc42d2c 100644 --- a/app_integration_test.go +++ b/app_integration_test.go @@ -16,7 +16,7 @@ const ( runLocalYtsaurus = false ) -// TestAppIntegration checks sync with real Azure API and local yt +// TestAppIntegration checks sync with real Source API and local yt // It requires AZURE_CLIENT_SECRET to be set. func TestAppIntegration(t *testing.T) { require.NoError(t, os.Setenv(defaultYtsaurusSecretEnvVar, ytDevToken)) diff --git a/app_test.go b/app_test.go index a97c82f..6937815 100644 --- a/app_test.go +++ b/app_test.go @@ -2,7 +2,11 @@ package main import ( "context" + "fmt" + "github.com/go-ldap/ldap/v3" + "go.ytsaurus.tech/library/go/ptr" "os" + "strings" "testing" "time" @@ -16,6 +20,9 @@ import ( const ( ytDevToken = "password" + aliceName = "alice" + bobName = "bob" + carolName = "carol" ) type testCase struct { @@ -23,556 +30,580 @@ type testCase struct { appConfig *AppConfig testTime time.Time - azureUsersSetUp []AzureUser - ytUsersSetUp []YtsaurusUser - ytUsersExpected []YtsaurusUser + sourceType SourceType - azureGroupsSetUp []AzureGroupWithMembers - ytGroupsSetUp []YtsaurusGroupWithMembers - ytGroupsExpected []YtsaurusGroupWithMembers -} + sourceUsersSetUp []SourceUser + ytUsersSetUp []YtsaurusUser + ytUsersExpected []YtsaurusUser -var ( - testTimeStr = "2023-10-20T12:00:00Z" - initialTestTime = parseAppTime(testTimeStr) + sourceGroupsSetUp []SourceGroupWithMembers + ytGroupsSetUp []YtsaurusGroupWithMembers + ytGroupsExpected []YtsaurusGroupWithMembers +} - aliceAzure = AzureUser{ - PrincipalName: "alice@acme.com", - AzureID: "fake-az-id-alice", - Email: "alice@acme.com", - FirstName: "Alice", - LastName: "Henderson", - DisplayName: "Henderson, Alice (ACME)", - } - bobAzure = AzureUser{ - PrincipalName: "Bob@acme.com", - AzureID: "fake-az-id-bob", - Email: "Bob@acme.com", - FirstName: "Bob", - LastName: "Sanders", - DisplayName: "Sanders, Bob (ACME)", - } - carolAzure = AzureUser{ - PrincipalName: "carol@acme.com", - AzureID: "fake-az-id-carol", - Email: "carol@acme.com", - FirstName: "Carol", - LastName: "Sanders", - DisplayName: "Sanders, Carol (ACME)", - } - aliceAzureChangedLastName = AzureUser{ - PrincipalName: aliceAzure.PrincipalName, - AzureID: aliceAzure.AzureID, - Email: aliceAzure.Email, - FirstName: aliceAzure.FirstName, - LastName: "Smith", - DisplayName: aliceAzure.DisplayName, - } - bobAzureChangedEmail = AzureUser{ - PrincipalName: "bobby@example.com", - AzureID: bobAzure.AzureID, - Email: "bobby@example.com", - FirstName: bobAzure.FirstName, - LastName: bobAzure.LastName, - DisplayName: bobAzure.DisplayName, - } - devsAzureGroup = AzureGroup{ - Identity: "acme.devs|all", - AzureID: "fake-az-acme.devs", - DisplayName: "acme.devs|all", - } - hqAzureGroup = AzureGroup{ - Identity: "acme.hq", - AzureID: "fake-az-acme.hq", - DisplayName: "acme.hq", - } - devsAzureGroupChangedDisplayName = AzureGroup{ - Identity: "acme.developers|all", - AzureID: devsAzureGroup.AzureID, - DisplayName: "acme.developers|all", - } - hqAzureGroupChangedBackwardCompatible = AzureGroup{ - Identity: "acme.hq|all", - AzureID: hqAzureGroup.AzureID, - DisplayName: "acme.hq|all", +func getUserId(name string) string { + switch name { + case aliceName: + return "1" + case bobName: + return "2" + case carolName: + return "3" } + return "4" +} - aliceYtsaurus = YtsaurusUser{ - Username: "alice", - AzureID: aliceAzure.AzureID, - PrincipalName: aliceAzure.PrincipalName, - Email: aliceAzure.Email, - FirstName: aliceAzure.FirstName, - LastName: aliceAzure.LastName, - DisplayName: aliceAzure.DisplayName, - } - bobYtsaurus = YtsaurusUser{ - Username: "bob", - AzureID: bobAzure.AzureID, - PrincipalName: bobAzure.PrincipalName, - Email: bobAzure.Email, - FirstName: bobAzure.FirstName, - LastName: bobAzure.LastName, - DisplayName: bobAzure.DisplayName, - } - carolYtsaurus = YtsaurusUser{ - Username: "carol", - AzureID: carolAzure.AzureID, - PrincipalName: carolAzure.PrincipalName, - Email: carolAzure.Email, - FirstName: carolAzure.FirstName, - LastName: carolAzure.LastName, - DisplayName: carolAzure.DisplayName, - } - aliceYtsaurusChangedLastName = YtsaurusUser{ - Username: aliceYtsaurus.Username, - AzureID: aliceYtsaurus.AzureID, - PrincipalName: aliceYtsaurus.PrincipalName, - Email: aliceYtsaurus.Email, - FirstName: aliceYtsaurus.FirstName, - LastName: aliceAzureChangedLastName.LastName, - DisplayName: aliceYtsaurus.DisplayName, - } - bobYtsaurusChangedEmail = YtsaurusUser{ - Username: "bobby:example.com", - AzureID: bobYtsaurus.AzureID, - PrincipalName: bobAzureChangedEmail.PrincipalName, - Email: bobAzureChangedEmail.Email, - FirstName: bobYtsaurus.FirstName, - LastName: bobYtsaurus.LastName, - DisplayName: bobYtsaurus.DisplayName, - } - bobYtsaurusBanned = YtsaurusUser{ - Username: bobYtsaurus.Username, - AzureID: bobYtsaurus.AzureID, - PrincipalName: bobYtsaurus.PrincipalName, - Email: bobYtsaurus.Email, - FirstName: bobYtsaurus.FirstName, - LastName: bobYtsaurus.LastName, - DisplayName: bobYtsaurus.DisplayName, - BannedSince: initialTestTime, - } - carolYtsaurusBanned = YtsaurusUser{ - Username: carolYtsaurus.Username, - AzureID: carolYtsaurus.AzureID, - PrincipalName: carolYtsaurus.PrincipalName, - Email: carolYtsaurus.Email, - FirstName: carolYtsaurus.FirstName, - LastName: carolYtsaurus.LastName, - DisplayName: carolYtsaurus.DisplayName, - BannedSince: initialTestTime.Add(40 * time.Hour), - } - devsYtsaurusGroup = YtsaurusGroup{ - Name: "acme.devs", - AzureID: devsAzureGroup.AzureID, - DisplayName: "acme.devs|all", +func getSourceUser(name string, sourceType SourceType) SourceUser { + switch sourceType { + case LdapSourceType: + return LdapUser{ + BasicSourceUser: BasicSourceUser{SourceType: LdapSourceType}, + Username: fmt.Sprintf("%v@acme.com", name), + Uid: getUserId(name), + FirstName: fmt.Sprintf("%v@acme.com-firstname", name), + } + case AzureSourceType: + return AzureUser{ + BasicSourceUser: BasicSourceUser{SourceType: AzureSourceType}, + PrincipalName: fmt.Sprintf("%v@acme.com", name), + AzureID: fmt.Sprintf("fake-az-id-%v", name), + Email: fmt.Sprintf("%v@acme.com", name), + FirstName: fmt.Sprintf("%v@acme.com-firstname", name), + LastName: fmt.Sprintf("%v-lastname", name), + DisplayName: fmt.Sprintf("Henderson, %v (ACME)", name), + } } - qaYtsaurusGroup = YtsaurusGroup{ - Name: "acme.qa", - AzureID: "fake-az-acme.qa", - DisplayName: "acme.qa|all", + return nil +} + +func getUpdatedSourceUser(name string, sourceType SourceType) SourceUser { + sourceUser := getSourceUser(name, sourceType) + switch sourceType { + case LdapSourceType: + ldapSourceUser := sourceUser.(LdapUser) + return LdapUser{ + BasicSourceUser: ldapSourceUser.BasicSourceUser, + Username: ldapSourceUser.Username, + Uid: ldapSourceUser.Uid, + FirstName: ldapSourceUser.FirstName + "-updated", + } + case AzureSourceType: + azureSourceUser := sourceUser.(AzureUser) + return AzureUser{ + BasicSourceUser: azureSourceUser.BasicSourceUser, + PrincipalName: azureSourceUser.PrincipalName, + AzureID: azureSourceUser.AzureID, + Email: azureSourceUser.Email + "-updated", + FirstName: azureSourceUser.FirstName, + LastName: azureSourceUser.LastName, + DisplayName: azureSourceUser.DisplayName, + } } - hqYtsaurusGroup = YtsaurusGroup{ - Name: "acme.hq", - AzureID: hqAzureGroup.AzureID, - DisplayName: "acme.hq", + return nil +} + +func getYtsaurusUser(sourceUser SourceUser) YtsaurusUser { + name := sourceUser.GetName() + for _, replacement := range defaultUsernameReplacements { + name = strings.Replace(name, replacement.From, replacement.To, -1) } - devsYtsaurusGroupChangedDisplayName = YtsaurusGroup{ - Name: "acme.developers", - AzureID: devsAzureGroup.AzureID, - DisplayName: "acme.developers|all", + return YtsaurusUser{Username: name, SourceUser: sourceUser} +} + +func bannedYtsaurusUser(ytUser YtsaurusUser, bannedSince time.Time) YtsaurusUser { + return YtsaurusUser{Username: ytUser.Username, SourceUser: ytUser.SourceUser, BannedSince: bannedSince} +} + +func getSourceGroup(name string, sourceType SourceType) SourceGroup { + switch sourceType { + case AzureSourceType: + return AzureGroup{ + BasicSourceGroup: BasicSourceGroup{SourceType: AzureSourceType}, + Identity: fmt.Sprintf("acme.%v|all", name), + AzureID: fmt.Sprintf("fake-az-acme.%v", name), + DisplayName: fmt.Sprintf("acme.%v|all", name), + } + case LdapSourceType: + return LdapGroup{ + BasicSourceGroup: BasicSourceGroup{SourceType: LdapSourceType}, + Groupname: fmt.Sprintf("acme.%v|all", name), + } } - hqYtsaurusGroupChangedBackwardCompatible = YtsaurusGroup{ - Name: "acme.hq", - AzureID: hqAzureGroup.AzureID, - DisplayName: "acme.hq|all", + return nil +} + +func getUpdatedSourceGroup(name string, sourceType SourceType) SourceGroup { + sourceGroup := getSourceGroup(name, sourceType) + switch sourceType { + case LdapSourceType: + // TODO(nadya73): add more fields. + ldapSourceGroup := sourceGroup.(LdapGroup) + return LdapGroup{ + BasicSourceGroup: ldapSourceGroup.BasicSourceGroup, + Groupname: ldapSourceGroup.Groupname, + } + case AzureSourceType: + azureSourceGroup := sourceGroup.(AzureGroup) + return AzureGroup{ + BasicSourceGroup: azureSourceGroup.BasicSourceGroup, + Identity: azureSourceGroup.Identity, + AzureID: azureSourceGroup.AzureID, + DisplayName: azureSourceGroup.DisplayName + "-updated", + } } + return nil +} - defaultUsernameReplacements = []ReplacementPair{ - {"@acme.com", ""}, - {"@", ":"}, +func getChangedBackwardCompatibleSourceGroup(name string, sourceType SourceType) SourceGroup { + if sourceType != AzureSourceType { + return nil } - defaultGroupnameReplacements = []ReplacementPair{ - {"|all", ""}, + sourceGroup := getSourceGroup(name, sourceType) + azureSourceGroup := sourceGroup.(AzureGroup) + return AzureGroup{ + BasicSourceGroup: azureSourceGroup.BasicSourceGroup, + Identity: azureSourceGroup.Identity + "-changed", + AzureID: azureSourceGroup.AzureID, + DisplayName: azureSourceGroup.DisplayName + "-updated", } - defaultAppConfig = &AppConfig{ - UsernameReplacements: defaultUsernameReplacements, - GroupnameReplacements: defaultGroupnameReplacements, +} + +func getYtsaurusGroup(sourceGroup SourceGroup) YtsaurusGroup { + name := sourceGroup.GetName() + for _, replacement := range defaultGroupnameReplacements { + name = strings.Replace(name, replacement.From, replacement.To, -1) } + return YtsaurusGroup{Name: name, SourceGroup: sourceGroup} +} - // we test several things in each test case, because of long wait for local ytsaurus - // container start. - testCases = []testCase{ +// We test several things in each test case, because of long wait for local ytsaurus +// container start. +func getTestCases(sourceType SourceType) []testCase { + testCases := []testCase{ { - name: "a-skip-b-create-c-remove", - azureUsersSetUp: []AzureUser{ - aliceAzure, - bobAzure, + name: "a-skip-b-create-c-remove", + sourceType: sourceType, + sourceUsersSetUp: []SourceUser{ + getSourceUser(aliceName, sourceType), + getSourceUser(bobName, sourceType), }, ytUsersSetUp: []YtsaurusUser{ - aliceYtsaurus, - carolYtsaurus, + getYtsaurusUser(getSourceUser(aliceName, sourceType)), + getYtsaurusUser(getSourceUser(carolName, sourceType)), }, ytUsersExpected: []YtsaurusUser{ - aliceYtsaurus, - bobYtsaurus, + getYtsaurusUser(getSourceUser(aliceName, sourceType)), + getYtsaurusUser(getSourceUser(bobName, sourceType)), }, }, { - name: "bob-is-banned", + name: "bob-is-banned", + sourceType: sourceType, appConfig: &AppConfig{ UsernameReplacements: defaultUsernameReplacements, GroupnameReplacements: defaultGroupnameReplacements, - BanBeforeRemoveDuration: 24 * time.Hour, + BanBeforeRemoveDuration: ptr.Duration(24 * time.Hour), }, - ytUsersSetUp: []YtsaurusUser{ - aliceYtsaurus, - bobYtsaurus, + sourceUsersSetUp: []SourceUser{ + getSourceUser(aliceName, sourceType), }, - azureUsersSetUp: []AzureUser{ - aliceAzure, + ytUsersSetUp: []YtsaurusUser{ + getYtsaurusUser(getSourceUser(aliceName, sourceType)), + getYtsaurusUser(getSourceUser(bobName, sourceType)), }, ytUsersExpected: []YtsaurusUser{ - aliceYtsaurus, - bobYtsaurusBanned, + getYtsaurusUser(getSourceUser(aliceName, sourceType)), + bannedYtsaurusUser(getYtsaurusUser(getSourceUser(bobName, sourceType)), initialTestTime), }, }, { - name: "bob-was-banned-now-deleted-carol-was-banned-now-back", + name: "bob-was-banned-now-deleted-carol-was-banned-now-back", + sourceType: sourceType, // Bob was banned at initialTestTime, // 2 days have passed (more than setting allows) —> he should be removed. - // Carol was banned 8 hours ago and has been found in Azure -> she should be restored. + // Carol was banned 8 hours ago and has been found in Source -> she should be restored. testTime: initialTestTime.Add(48 * time.Hour), appConfig: &AppConfig{ UsernameReplacements: defaultUsernameReplacements, GroupnameReplacements: defaultGroupnameReplacements, - BanBeforeRemoveDuration: 24 * time.Hour, + BanBeforeRemoveDuration: ptr.Duration(24 * time.Hour), }, - ytUsersSetUp: []YtsaurusUser{ - aliceYtsaurus, - bobYtsaurusBanned, - carolYtsaurusBanned, + sourceUsersSetUp: []SourceUser{ + getSourceUser(aliceName, sourceType), + getSourceUser(carolName, sourceType), }, - azureUsersSetUp: []AzureUser{ - aliceAzure, - carolAzure, + ytUsersSetUp: []YtsaurusUser{ + getYtsaurusUser(getSourceUser(aliceName, sourceType)), + bannedYtsaurusUser(getYtsaurusUser(getSourceUser(bobName, sourceType)), initialTestTime), + bannedYtsaurusUser(getYtsaurusUser(getSourceUser(carolName, sourceType)), initialTestTime.Add(40*time.Hour)), }, ytUsersExpected: []YtsaurusUser{ - aliceYtsaurus, - carolYtsaurus, + getYtsaurusUser(getSourceUser(aliceName, sourceType)), + getYtsaurusUser(getSourceUser(carolName, sourceType)), }, }, { - name: "remove-limit-users-3", + name: "remove-limit-users-3", + sourceType: sourceType, appConfig: &AppConfig{ UsernameReplacements: defaultUsernameReplacements, GroupnameReplacements: defaultGroupnameReplacements, - RemoveLimit: 3, + RemoveLimit: ptr.Int(3), }, - azureUsersSetUp: []AzureUser{}, + sourceUsersSetUp: []SourceUser{}, ytUsersSetUp: []YtsaurusUser{ - aliceYtsaurus, - bobYtsaurus, - carolYtsaurus, + getYtsaurusUser(getSourceUser(aliceName, sourceType)), + getYtsaurusUser(getSourceUser(bobName, sourceType)), + getYtsaurusUser(getSourceUser(carolName, sourceType)), }, // no one is deleted: limitation works ytUsersExpected: []YtsaurusUser{ - aliceYtsaurus, - bobYtsaurus, - carolYtsaurus, + getYtsaurusUser(getSourceUser(aliceName, sourceType)), + getYtsaurusUser(getSourceUser(bobName, sourceType)), + getYtsaurusUser(getSourceUser(carolName, sourceType)), }, }, { - name: "remove-limit-groups-3", + name: "remove-limit-groups-3", + sourceType: sourceType, appConfig: &AppConfig{ UsernameReplacements: defaultUsernameReplacements, GroupnameReplacements: defaultGroupnameReplacements, - RemoveLimit: 3, + RemoveLimit: ptr.Int(3), }, - azureGroupsSetUp: []AzureGroupWithMembers{}, + sourceGroupsSetUp: []SourceGroupWithMembers{}, ytGroupsSetUp: []YtsaurusGroupWithMembers{ - NewEmptyYtsaurusGroupWithMembers(devsYtsaurusGroup), - NewEmptyYtsaurusGroupWithMembers(qaYtsaurusGroup), - NewEmptyYtsaurusGroupWithMembers(hqYtsaurusGroup), + NewEmptyYtsaurusGroupWithMembers(getYtsaurusGroup(getSourceGroup("dev", sourceType))), + NewEmptyYtsaurusGroupWithMembers(getYtsaurusGroup(getSourceGroup("qa", sourceType))), + NewEmptyYtsaurusGroupWithMembers(getYtsaurusGroup(getSourceGroup("hq", sourceType))), }, // no group is deleted: limitation works ytGroupsExpected: []YtsaurusGroupWithMembers{ - NewEmptyYtsaurusGroupWithMembers(devsYtsaurusGroup), - NewEmptyYtsaurusGroupWithMembers(qaYtsaurusGroup), - NewEmptyYtsaurusGroupWithMembers(hqYtsaurusGroup), + NewEmptyYtsaurusGroupWithMembers(getYtsaurusGroup(getSourceGroup("dev", sourceType))), + NewEmptyYtsaurusGroupWithMembers(getYtsaurusGroup(getSourceGroup("qa", sourceType))), + NewEmptyYtsaurusGroupWithMembers(getYtsaurusGroup(getSourceGroup("hq", sourceType))), }, }, { - name: "a-changed-name-b-changed-email", - azureUsersSetUp: []AzureUser{ - aliceAzureChangedLastName, - bobAzureChangedEmail, + name: "a-changed-name-b-changed-email", + sourceType: sourceType, + sourceUsersSetUp: []SourceUser{ + getUpdatedSourceUser(aliceName, sourceType), + getUpdatedSourceUser(bobName, sourceType), }, ytUsersSetUp: []YtsaurusUser{ - aliceYtsaurus, - bobYtsaurus, + getYtsaurusUser(getSourceUser(aliceName, sourceType)), + getYtsaurusUser(getSourceUser(bobName, sourceType)), }, ytUsersExpected: []YtsaurusUser{ - aliceYtsaurusChangedLastName, - bobYtsaurusChangedEmail, + getYtsaurusUser(getUpdatedSourceUser(aliceName, sourceType)), + getYtsaurusUser(getUpdatedSourceUser(bobName, sourceType)), }, }, { - name: "skip-create-remove-group-no-members-change-correct-name-replace", - azureUsersSetUp: []AzureUser{ - aliceAzure, - bobAzure, - carolAzure, + name: "skip-create-remove-group-no-members-change-correct-name-replace", + sourceType: sourceType, + sourceUsersSetUp: []SourceUser{ + getSourceUser(aliceName, sourceType), + getSourceUser(bobName, sourceType), + getSourceUser(carolName, sourceType), }, ytUsersSetUp: []YtsaurusUser{ - aliceYtsaurus, - bobYtsaurus, - carolYtsaurus, + getYtsaurusUser(getSourceUser(aliceName, sourceType)), + getYtsaurusUser(getSourceUser(bobName, sourceType)), + getYtsaurusUser(getSourceUser(carolName, sourceType)), }, ytUsersExpected: []YtsaurusUser{ - aliceYtsaurus, - bobYtsaurus, - carolYtsaurus, + getYtsaurusUser(getSourceUser(aliceName, sourceType)), + getYtsaurusUser(getSourceUser(bobName, sourceType)), + getYtsaurusUser(getSourceUser(carolName, sourceType)), }, ytGroupsSetUp: []YtsaurusGroupWithMembers{ { - YtsaurusGroup: devsYtsaurusGroup, - Members: NewStringSetFromItems(aliceYtsaurus.Username), + YtsaurusGroup: getYtsaurusGroup(getSourceGroup("devs", sourceType)), + Members: NewStringSetFromItems(aliceName), }, { - YtsaurusGroup: qaYtsaurusGroup, - Members: NewStringSetFromItems(bobYtsaurus.Username), + YtsaurusGroup: getYtsaurusGroup(getSourceGroup("qa", sourceType)), + Members: NewStringSetFromItems(bobName), }, }, - azureGroupsSetUp: []AzureGroupWithMembers{ + sourceGroupsSetUp: []SourceGroupWithMembers{ { - AzureGroup: devsAzureGroup, - Members: NewStringSetFromItems(aliceAzure.AzureID), + SourceGroup: getSourceGroup("devs", sourceType), + Members: NewStringSetFromItems(getSourceUser(aliceName, sourceType).GetId()), }, { - AzureGroup: hqAzureGroup, - Members: NewStringSetFromItems(carolAzure.AzureID), + SourceGroup: getSourceGroup("hq", sourceType), + Members: NewStringSetFromItems(getSourceUser(carolName, sourceType).GetId()), }, }, ytGroupsExpected: []YtsaurusGroupWithMembers{ { - YtsaurusGroup: devsYtsaurusGroup, - Members: NewStringSetFromItems(aliceYtsaurus.Username), + YtsaurusGroup: getYtsaurusGroup(getSourceGroup("devs", sourceType)), + Members: NewStringSetFromItems(aliceName), }, { - YtsaurusGroup: hqYtsaurusGroup, - Members: NewStringSetFromItems(carolYtsaurus.Username), + YtsaurusGroup: getYtsaurusGroup(getSourceGroup("hq", sourceType)), + Members: NewStringSetFromItems(carolName), }, }, }, { - name: "memberships-add-remove", - azureUsersSetUp: []AzureUser{ - aliceAzure, - bobAzure, - carolAzure, + name: "memberships-add-remove", + sourceType: sourceType, + sourceUsersSetUp: []SourceUser{ + getSourceUser(aliceName, sourceType), + getSourceUser(bobName, sourceType), + getSourceUser(carolName, sourceType), }, ytUsersSetUp: []YtsaurusUser{ - aliceYtsaurus, - bobYtsaurus, - carolYtsaurus, + getYtsaurusUser(getSourceUser(aliceName, sourceType)), + getYtsaurusUser(getSourceUser(bobName, sourceType)), + getYtsaurusUser(getSourceUser(carolName, sourceType)), }, ytUsersExpected: []YtsaurusUser{ - aliceYtsaurus, - bobYtsaurus, - carolYtsaurus, + getYtsaurusUser(getSourceUser(aliceName, sourceType)), + getYtsaurusUser(getSourceUser(bobName, sourceType)), + getYtsaurusUser(getSourceUser(carolName, sourceType)), }, ytGroupsSetUp: []YtsaurusGroupWithMembers{ { - YtsaurusGroup: devsYtsaurusGroup, + YtsaurusGroup: getYtsaurusGroup(getSourceGroup("devs", sourceType)), Members: NewStringSetFromItems( - aliceYtsaurus.Username, - bobYtsaurus.Username, + aliceName, + bobName, ), }, }, - azureGroupsSetUp: []AzureGroupWithMembers{ + sourceGroupsSetUp: []SourceGroupWithMembers{ { - AzureGroup: devsAzureGroup, + SourceGroup: getSourceGroup("devs", sourceType), Members: NewStringSetFromItems( - aliceAzure.AzureID, - carolAzure.AzureID, + getSourceUser(aliceName, sourceType).GetId(), + getSourceUser(carolName, sourceType).GetId(), ), }, }, ytGroupsExpected: []YtsaurusGroupWithMembers{ { - YtsaurusGroup: devsYtsaurusGroup, + YtsaurusGroup: getYtsaurusGroup(getSourceGroup("devs", sourceType)), Members: NewStringSetFromItems( - aliceYtsaurus.Username, - carolYtsaurus.Username, + aliceName, + carolName, ), }, }, }, - { - name: "display-name-changes", - azureUsersSetUp: []AzureUser{ - aliceAzure, - bobAzure, - carolAzure, + } + + if sourceType == AzureSourceType { + testCases = append(testCases, testCase{ + name: "display-name-changes", + sourceType: sourceType, + sourceUsersSetUp: []SourceUser{ + getSourceUser(aliceName, sourceType), + getSourceUser(bobName, sourceType), + getSourceUser(carolName, sourceType), }, ytUsersSetUp: []YtsaurusUser{ - aliceYtsaurus, - bobYtsaurus, - carolYtsaurus, + getYtsaurusUser(getSourceUser(aliceName, sourceType)), + getYtsaurusUser(getSourceUser(bobName, sourceType)), + getYtsaurusUser(getSourceUser(carolName, sourceType)), }, ytUsersExpected: []YtsaurusUser{ - aliceYtsaurus, - bobYtsaurus, - carolYtsaurus, + getYtsaurusUser(getSourceUser(aliceName, sourceType)), + getYtsaurusUser(getSourceUser(bobName, sourceType)), + getYtsaurusUser(getSourceUser(carolName, sourceType)), }, ytGroupsSetUp: []YtsaurusGroupWithMembers{ { - YtsaurusGroup: devsYtsaurusGroup, + YtsaurusGroup: getYtsaurusGroup(getSourceGroup("devs", sourceType)), Members: NewStringSetFromItems( - aliceYtsaurus.Username, - bobYtsaurus.Username, + aliceName, + bobName, ), }, { - YtsaurusGroup: hqYtsaurusGroup, + YtsaurusGroup: getYtsaurusGroup(getSourceGroup("hq", sourceType)), Members: NewStringSetFromItems( - aliceYtsaurus.Username, - bobYtsaurus.Username, + aliceName, + bobName, ), }, }, - azureGroupsSetUp: []AzureGroupWithMembers{ + sourceGroupsSetUp: []SourceGroupWithMembers{ { // This group should be updated. - AzureGroup: devsAzureGroupChangedDisplayName, + SourceGroup: getUpdatedSourceGroup("devs", sourceType), // Members list are also updated. Members: NewStringSetFromItems( - aliceAzure.AzureID, - carolAzure.AzureID, + getSourceUser(aliceName, sourceType).GetId(), + getSourceUser(carolName, sourceType).GetId(), ), }, { // for this group only displayName should be updated - AzureGroup: hqAzureGroupChangedBackwardCompatible, + SourceGroup: getChangedBackwardCompatibleSourceGroup("hq", sourceType), // members also changed Members: NewStringSetFromItems( - aliceAzure.AzureID, - carolAzure.AzureID, + getSourceUser(aliceName, sourceType).GetId(), + getSourceUser(carolName, sourceType).GetId(), ), }, }, ytGroupsExpected: []YtsaurusGroupWithMembers{ { - YtsaurusGroup: devsYtsaurusGroupChangedDisplayName, + YtsaurusGroup: getYtsaurusGroup(getUpdatedSourceGroup("devs", sourceType)), Members: NewStringSetFromItems( - aliceYtsaurus.Username, - carolYtsaurus.Username, + aliceName, + carolName, ), }, { - YtsaurusGroup: hqYtsaurusGroupChangedBackwardCompatible, + YtsaurusGroup: getYtsaurusGroup(getChangedBackwardCompatibleSourceGroup("hq", sourceType)), Members: NewStringSetFromItems( - aliceYtsaurus.Username, - carolYtsaurus.Username, + aliceName, + carolName, ), }, }, - }, + }) + } + return testCases +} + +var ( + testTimeStr = "2023-10-20T12:00:00Z" + initialTestTime = parseAppTime(testTimeStr) + + defaultUsernameReplacements = []ReplacementPair{ + {"@acme.com", ""}, + {"@", ":"}, + } + defaultGroupnameReplacements = []ReplacementPair{ + {"|all", ""}, + } + defaultAppConfig = &AppConfig{ + UsernameReplacements: defaultUsernameReplacements, + GroupnameReplacements: defaultGroupnameReplacements, } ) -// TestAppSync uses local YTsaurus container and fake Azure to test all the cases: -// [x] If Azure user not in YTsaurus -> created; -// [x] If Azure user already in YTsaurus no changes -> skipped; -// [x] If Azure user already in YTsaurus with changes -> updated; -// [x] If user in YTsaurus but not in Azure (and ban_before_remove_duration=0) -> removed; -// [x] If user in YTsaurus but not in Azure (and ban_before_remove_duration != 0) -> banned -> removed; +// TestAppSync uses local YTsaurus container and fake Source to test all the cases: +// [x] If Source user not in YTsaurus -> created; +// [x] If Source user already in YTsaurus no changes -> skipped; +// [x] If Source user already in YTsaurus with changes -> updated; +// [x] If user in YTsaurus but not in Source (and ban_before_remove_duration=0) -> removed; +// [x] If user in YTsaurus but not in Source (and ban_before_remove_duration != 0) -> banned -> removed; // [x] If Azure user without @azure attribute in YTsaurus —> ignored; -// [x] Azure user field updates is reflected in YTsaurus user; +// [x] Source user field updates is reflected in YTsaurus user; // [x] YTsaurus username is built according to config; // [x] YTsaurus username is in lowercase; -// [x] If Azure group is not exist in YTsaurus -> creating YTsaurus with members; -// [x] If YTsaurus group is not exist in Azure -> delete YTsaurus group; -// [x] If Azure group membership changed -> update in YTsaurus group membership; -// [x] If Azure group fields (though there are none extra fields) changed -> update YTsaurus group; -// [x] If Azure group displayName changed -> recreate YTsaurus group; -// [x] If Azure group displayName changed AND Azure members changed -> recreate YTsaurus group with actual members set; +// [x] If Source group is not exist in YTsaurus -> creating YTsaurus with members; +// [x] If YTsaurus group is not exist in Source -> delete YTsaurus group; +// [x] If Source group membership changed -> update in YTsaurus group membership; +// [x] If Source group fields (though there are none extra fields) changed -> update YTsaurus group; +// [x] If Source group displayName changed -> recreate YTsaurus group; +// [x] If Source group displayName changed AND Source members changed -> recreate YTsaurus group with actual members set; // [x] YTsaurus group name is built according to config; // [x] Remove limits config option works. func TestAppSync(t *testing.T) { require.NoError(t, os.Setenv(defaultYtsaurusSecretEnvVar, ytDevToken)) - for _, tc := range testCases { - t.Run( - tc.name, - func(tc testCase) func(t *testing.T) { - return func(t *testing.T) { - if tc.testTime.IsZero() { - tc.testTime = initialTestTime - } - clock := testclock.NewFakePassiveClock(initialTestTime) - - ytLocal := NewYtsaurusLocal() - defer func() { require.NoError(t, ytLocal.Stop()) }() - require.NoError(t, ytLocal.Start()) - - azure := NewAzureFake() - azure.setUsers(tc.azureUsersSetUp) - azure.setGroups(tc.azureGroupsSetUp) - - ytClient, err := ytLocal.GetClient() - require.NoError(t, err) - - initialYtUsers, initialYtGroups := getAllYtsaurusObjects(t, ytClient) - setupYtsaurusObjects(t, ytClient, tc.ytUsersSetUp, tc.ytGroupsSetUp) - - if tc.appConfig == nil { - tc.appConfig = defaultAppConfig - } - app, err := NewAppCustomized( - &Config{ - App: tc.appConfig, - Azure: &AzureConfig{}, - Ytsaurus: &YtsaurusConfig{ - Proxy: ytLocal.GetProxy(), - ApplyUserChanges: true, - ApplyGroupChanges: true, - ApplyMemberChanges: true, - LogLevel: "DEBUG", + for _, sourceType := range []SourceType{LdapSourceType, AzureSourceType} { + for _, tc := range getTestCases(sourceType) { + t.Run( + fmt.Sprintf("%v-%v", sourceType, tc.name), + func(tc testCase) func(t *testing.T) { + return func(t *testing.T) { + if tc.testTime.IsZero() { + tc.testTime = initialTestTime + } + clock := testclock.NewFakePassiveClock(initialTestTime) + + ytLocal := NewYtsaurusLocal() + defer func() { require.NoError(t, ytLocal.Stop()) }() + require.NoError(t, ytLocal.Start()) + + var source Source + + switch tc.sourceType { + case AzureSourceType: + azure := NewAzureFake() + azure.setUsers(tc.sourceUsersSetUp) + azure.setGroups(tc.sourceGroupsSetUp) + + source = azure + case LdapSourceType: + ldapLocal := NewOpenLdapLocal() + + defer func() { require.NoError(t, ldapLocal.Stop()) }() + require.NoError(t, ldapLocal.Start()) + + ldapConfig, err := ldapLocal.GetConfig() + require.NoError(t, err) + ldapSource, err := NewLdap(ldapConfig, getDevelopmentLogger()) + require.NoError(t, err) + + setupLdapObjects(t, ldapSource.Connection, tc.sourceUsersSetUp, tc.sourceGroupsSetUp) + source = ldapSource + } + + ytClient, err := ytLocal.GetClient() + require.NoError(t, err) + + initialYtUsers, initialYtGroups := getAllYtsaurusObjects(t, ytClient) + setupYtsaurusObjects(t, ytClient, tc.ytUsersSetUp, tc.ytGroupsSetUp) + + if tc.appConfig == nil { + tc.appConfig = defaultAppConfig + } + app, err := NewAppCustomized( + &Config{ + App: *tc.appConfig, + Azure: &AzureConfig{}, + Ldap: &LdapConfig{}, + Ytsaurus: YtsaurusConfig{ + Proxy: ytLocal.GetProxy(), + ApplyUserChanges: true, + ApplyGroupChanges: true, + ApplyMemberChanges: true, + LogLevel: "DEBUG", + }, }, - }, getDevelopmentLogger(), - azure, - clock, - ) - require.NoError(t, err) - - app.syncOnce() - - // we have eventually here, because user removal takes some time. - require.Eventually( - t, - func() bool { - udiff, gdiff := diffYtsaurusObjects(t, ytClient, tc.ytUsersExpected, initialYtUsers, tc.ytGroupsExpected, initialYtGroups) - actualUsers, actualGroups := getAllYtsaurusObjects(t, ytClient) - if udiff != "" { - t.Log("Users diff is not empty yet:", udiff) - t.Log("expected users", tc.ytUsersExpected) - t.Log("actual users", actualUsers) - } - if gdiff != "" { - t.Log("Groups diff is not empty yet:", gdiff) - t.Log("expected groups", tc.ytGroupsExpected) - t.Log("actual groups", actualGroups) - } - return udiff == "" && gdiff == "" - }, - 3*time.Second, - 300*time.Millisecond, - ) - } - }(tc), - ) + getDevelopmentLogger(), + source, + clock, + ) + require.NoError(t, err) + + app.syncOnce() + + // We have eventually here, because user removal takes some time. + require.Eventually( + t, + func() bool { + udiff, gdiff := diffYtsaurusObjects(t, ytClient, tc.ytUsersExpected, initialYtUsers, tc.ytGroupsExpected, initialYtGroups) + actualUsers, actualGroups := getAllYtsaurusObjects(t, ytClient) + if udiff != "" { + t.Log("Users diff is not empty yet:", udiff) + t.Log("expected users", tc.ytUsersExpected) + t.Log("actual users", actualUsers) + } + if gdiff != "" { + t.Log("Groups diff is not empty yet:", gdiff) + t.Log("expected groups", tc.ytGroupsExpected) + t.Log("actual groups", actualGroups) + } + return udiff == "" && gdiff == "" + }, + 3*time.Second, + 300*time.Millisecond, + ) + } + }(tc), + ) + } } } @@ -615,7 +646,7 @@ func TestManageUnmanagedUsersIsForbidden(t *testing.T) { "Prevented attempt to change manual managed user", ) require.ErrorContains(t, - ytsaurus.UpdateUser(username, YtsaurusUser{Username: username, Email: "dummy@acme.com"}), + ytsaurus.UpdateUser(username, YtsaurusUser{Username: username, SourceUser: AzureUser{Email: "dummy@acme.com"}}), "Prevented attempt to change manual managed user", ) } @@ -629,6 +660,114 @@ func getAllYtsaurusObjects(t *testing.T, client yt.Client) (users []YtsaurusUser return allUsers, allGroups } +func setupLdapObjects(t *testing.T, conn *ldap.Conn, users []SourceUser, groups []SourceGroupWithMembers) { + require.NoError(t, conn.Add(&ldap.AddRequest{ + DN: "ou=People,dc=example,dc=org", + Attributes: []ldap.Attribute{ + { + Type: "objectClass", + Vals: []string{"organizationalUnit"}, + }, + { + Type: "ou", + Vals: []string{"People"}, + }, + }, + })) + + require.NoError(t, conn.Add(&ldap.AddRequest{ + DN: "ou=Group,dc=example,dc=org", + Attributes: []ldap.Attribute{ + { + Type: "objectClass", + Vals: []string{"organizationalUnit"}, + }, + { + Type: "ou", + Vals: []string{"Group"}, + }, + }, + })) + + for _, user := range users { + ldapUser := user.(LdapUser) + addRequest := ldap.AddRequest{ + DN: fmt.Sprintf("uid=%s,ou=People,dc=example,dc=org", user.GetId()), + Attributes: []ldap.Attribute{ + { + Type: "objectClass", + Vals: []string{"top", "posixAccount", "inetOrgPerson"}, + }, + { + Type: "ou", + Vals: []string{"People"}, + }, + { + Type: "cn", + Vals: []string{user.GetName()}, + }, + { + Type: "uid", + Vals: []string{user.GetId()}, + }, + { + Type: "uidNumber", + Vals: []string{user.GetId()}, + }, + { + Type: "gidNumber", + Vals: []string{user.GetId()}, + }, + { + Type: "givenName", + Vals: []string{ldapUser.FirstName}, + }, + { + Type: "homeDirectory", + Vals: []string{ldapUser.GetId()}, + }, + { + Type: "sn", + Vals: []string{ldapUser.GetName() + "-surname"}, + }, + }, + } + require.NoError(t, conn.Add(&addRequest)) + } + + for groupId, group := range groups { + ldapGroup := group.SourceGroup.(LdapGroup) + + members := make([]string, 0) + for member := range group.Members.Iter() { + members = append(members, member) + } + + addRequest := ldap.AddRequest{ + DN: fmt.Sprintf("cn=%s,ou=Group,dc=example,dc=org", ldapGroup.GetId()), + Attributes: []ldap.Attribute{ + { + Type: "objectClass", + Vals: []string{"top", "posixGroup"}, + }, + { + Type: "cn", + Vals: []string{ldapGroup.GetName()}, + }, + { + Type: "gidNumber", + Vals: []string{fmt.Sprint(groupId)}, + }, + { + Type: "memberUid", + Vals: members, + }, + }, + } + require.NoError(t, conn.Add(&addRequest)) + } +} + func setupYtsaurusObjects(t *testing.T, client yt.Client, users []YtsaurusUser, groups []YtsaurusGroupWithMembers) { t.Log("Setting up yt for test") for _, user := range users { diff --git a/azure_fake.go b/azure_fake.go index 2ac1e06..937261c 100644 --- a/azure_fake.go +++ b/azure_fake.go @@ -1,26 +1,26 @@ package main type AzureFake struct { - users []AzureUser - groups []AzureGroupWithMembers + users []SourceUser + groups []SourceGroupWithMembers } func NewAzureFake() *AzureFake { return &AzureFake{} } -func (a *AzureFake) setUsers(users []AzureUser) { +func (a *AzureFake) setUsers(users []SourceUser) { a.users = users } -func (a *AzureFake) setGroups(groups []AzureGroupWithMembers) { +func (a *AzureFake) setGroups(groups []SourceGroupWithMembers) { a.groups = groups } -func (a *AzureFake) GetUsers() ([]AzureUser, error) { +func (a *AzureFake) GetUsers() ([]SourceUser, error) { return a.users, nil } -func (a *AzureFake) GetGroupsWithMembers() ([]AzureGroupWithMembers, error) { +func (a *AzureFake) GetGroupsWithMembers() ([]SourceGroupWithMembers, error) { return a.groups, nil } diff --git a/azure_real.go b/azure_real.go index 4b9c19c..5ecf5a4 100644 --- a/azure_real.go +++ b/azure_real.go @@ -17,14 +17,10 @@ import ( ) const ( - scope = "https://graph.microsoft.com/.default" - defaultAzureTimeout = 3 * time.Second - defaultAzureSecretEnvVar = "AZURE_CLIENT_SECRET" - msgraphExpandLimit = 20 + scope = "https://graph.microsoft.com/.default" + msgraphExpandLimit = 20 ) -type AzureID = string - var ( defaultUserFieldsToSelect = []string{ "userPrincipalName", @@ -41,33 +37,6 @@ var ( } ) -type AzureUser struct { - // PrincipalName is unique human-readable Azure user field, used (possibly with changes) - // for the corresponding YTsaurus user's `name` attribute. - PrincipalName string - - AzureID AzureID - Email string - FirstName string - LastName string - DisplayName string -} - -type AzureGroup struct { - // Identity is unique human-readable Azure user field, used (possibly with changes) - // for the corresponding YTsaurus user's `name` attribute. - Identity string - - AzureID AzureID - DisplayName string -} - -type AzureGroupWithMembers struct { - AzureGroup - // Members is a set of strings, representing users' AzureID. - Members StringSet -} - type AzureReal struct { graphClient *msgraphsdk.GraphServiceClient @@ -98,7 +67,7 @@ func NewAzureReal(cfg *AzureConfig, logger appLoggerType) (*AzureReal, error) { nil, ) if err != nil { - return nil, errors.Wrap(err, "failed to create Azure secret credentials") + return nil, errors.Wrap(err, "failed to create Source secret credentials") } graphClient, err := msgraphsdk.NewGraphServiceClientWithCredentials(cred, []string{scope}) @@ -129,7 +98,7 @@ func handleNil[T any](s *T) T { return result } -func (a *AzureReal) GetUsers() ([]AzureUser, error) { +func (a *AzureReal) GetUsers() ([]SourceUser, error) { ctx, cancel := context.WithTimeout(context.Background(), a.timeout) defer cancel() @@ -139,7 +108,7 @@ func (a *AzureReal) GetUsers() ([]AzureUser, error) { } usersSkipped := 0 - var users []AzureUser + var users []SourceUser for _, user := range usersRaw { principalName := handleNil(user.GetUserPrincipalName()) id := handleNil(user.GetId()) @@ -173,11 +142,11 @@ func (a *AzureReal) GetUsers() ([]AzureUser, error) { } } - a.logger.Infow("Fetched users from Azure AD", "got", len(usersRaw), "skipped", usersSkipped) + a.logger.Infow("Fetched users from Source AD", "got", len(usersRaw), "skipped", usersSkipped) return users, nil } -func (a *AzureReal) GetGroupsWithMembers() ([]AzureGroupWithMembers, error) { +func (a *AzureReal) GetGroupsWithMembers() ([]SourceGroupWithMembers, error) { ctx, cancel := context.WithTimeout(context.Background(), a.timeout) defer cancel() @@ -187,7 +156,7 @@ func (a *AzureReal) GetGroupsWithMembers() ([]AzureGroupWithMembers, error) { } groupsSkipped := 0 - var groups []AzureGroupWithMembers + var groups []SourceGroupWithMembers for _, group := range groupsRaw { displayName := handleNil(group.GetDisplayName()) id := handleNil(group.GetId()) @@ -225,8 +194,8 @@ func (a *AzureReal) GetGroupsWithMembers() ([]AzureGroupWithMembers, error) { a.maybePrintDebugLogs(id, "azure_members_count", len(memberIDs.ToSlice())) groups = append(groups, - AzureGroupWithMembers{ - AzureGroup: AzureGroup{ + SourceGroupWithMembers{ + SourceGroup: AzureGroup{ Identity: displayName, AzureID: id, DisplayName: displayName, @@ -235,11 +204,11 @@ func (a *AzureReal) GetGroupsWithMembers() ([]AzureGroupWithMembers, error) { }) } - a.logger.Infow("Fetched groups from Azure AD", "got", len(groupsRaw), "skipped", groupsSkipped) + a.logger.Infow("Fetched groups from Source AD", "got", len(groupsRaw), "skipped", groupsSkipped) return groups, nil } -func (a *AzureReal) maybePrintDebugLogs(id AzureID, args ...any) { +func (a *AzureReal) maybePrintDebugLogs(id ObjectID, args ...any) { args = append([]any{"id", id}, args...) for _, debugID := range a.debugAzureIDs { if id == debugID { @@ -285,7 +254,7 @@ func (a *AzureReal) getUsersRaw(ctx context.Context, fieldsToSelect []string, fi return true }) if err != nil { - return nil, errors.Wrap(err, "failed to iterate over Azure users") + return nil, errors.Wrap(err, "failed to iterate over Source users") } return rawUsers, nil } @@ -326,7 +295,7 @@ func (a *AzureReal) getGroupsWithMembersRaw(ctx context.Context, fieldsToSelect return true }) if err != nil { - return nil, errors.Wrap(err, "failed to iterate over Azure groups") + return nil, errors.Wrap(err, "failed to iterate over Source groups") } return rawGroups, nil } @@ -364,7 +333,7 @@ func (a *AzureReal) getGroupMembers(ctx context.Context, groupID string) ([]mode return true }) if err != nil { - return nil, errors.Wrap(err, "failed to iterate over Azure group members") + return nil, errors.Wrap(err, "failed to iterate over Source group members") } return rawMembers, nil diff --git a/azure_real_integration_test.go b/azure_real_integration_test.go index 18037b2..5864cc6 100644 --- a/azure_real_integration_test.go +++ b/azure_real_integration_test.go @@ -14,7 +14,7 @@ import ( //go:embed config.local.yaml var _localConfig embed.FS -// TestPrintAzureUsersIntegration tests nothing, but can be used to debug Azure users retrieved from ms graph api. +// TestPrintAzureUsersIntegration tests nothing, but can be used to debug Source users retrieved from ms graph api. // In particular, it can be used to tune userFilter for production use. // It requires AZURE_CLIENT_SECRET env var and `config.local.yaml` file (which is .gitignored). func TestPrintAzureUsersIntegration(t *testing.T) { @@ -79,7 +79,7 @@ func TestPrintAzureUsersIntegration(t *testing.T) { require.NotEmpty(t, usersRaw) } -// TestPrintAzureGroupsIntegration tests nothing, but can be used to debug Azure groups retrieved from ms graph api. +// TestPrintAzureGroupsIntegration tests nothing, but can be used to debug Source groups retrieved from ms graph api. // In particular, it can be used to tune groupsFilter for production use. // It requires AZURE_CLIENT_SECRET env var and `config.local.yaml` file (which is .gitignored). func TestPrintAzureGroupsIntegrationRaw(t *testing.T) { diff --git a/config.go b/config.go index 00a2835..d3d7fad 100644 --- a/config.go +++ b/config.go @@ -5,19 +5,21 @@ import ( ) type Config struct { - App *AppConfig `yaml:"app"` - Azure *AzureConfig `yaml:"azure"` - Ytsaurus *YtsaurusConfig `yaml:"ytsaurus"` - Logging *LoggingConfig `yaml:"logging"` + App AppConfig `yaml:"app"` + Ytsaurus YtsaurusConfig `yaml:"ytsaurus"` + Logging LoggingConfig `yaml:"logging"` + + // One of them should be specified. + Azure *AzureConfig `yaml:"azure,omitempty"` + Ldap *LdapConfig `yaml:"ldap,omitempty"` } type AppConfig struct { // SyncInterval is the interval between full synchronizations. - // Zero value means that auto-sync disabled (sync can be invoked only manually). - SyncInterval time.Duration `yaml:"sync_interval"` + // If it is not speciied or value is zero than auto-sync disabled (sync can be invoked only manually). + SyncInterval *time.Duration `yaml:"sync_interval"` - // UsernameReplacements is a list of replaces which will be applied to the userPrincipalName Azure field before - // using as username in Ytsaurus. + // UsernameReplacements is a list of replaces which will be applied to a username for source (Source or Ldap). // For example, you may use it to strip off characters like @ which are not recommended for use // in usernames as they are required to be escaped in YPath. UsernameReplacements []ReplacementPair `yaml:"username_replacements"` @@ -25,11 +27,12 @@ type AppConfig struct { // If count users or groups for planned delete in on sync cycle reaches RemoveLimit // app will fail that sync cycle. - RemoveLimit int `yaml:"remove_limit"` + // No limit if it is not specified. + RemoveLimit *int `yaml:"remove_limit,omitempty"` // BanBeforeRemoveDuration is a duration of a graceful ban before finally removing the user from YTsaurus. - // Default value is 0s, which means remove straight after user was found to be missing from Azure,. - BanBeforeRemoveDuration time.Duration `yaml:"ban_before_remove_duration"` + // If it is not specified, user will be removed straight after user was found to be missing from source (Source or Ldap). + BanBeforeRemoveDuration *time.Duration `yaml:"ban_before_remove_duration"` } type ReplacementPair struct { @@ -41,6 +44,7 @@ type AzureConfig struct { Tenant string `yaml:"tenant"` ClientID string `yaml:"client_id"` ClientSecretEnvVar string `yaml:"client_secret_env_var"` // default: "AZURE_CLIENT_SECRET" + // UsersFilter is MS Graph $filter value used for user fetching requests. // See https://learn.microsoft.com/en-us/graph/api/user-list?#optional-query-parameters UsersFilter string `yaml:"users_filter"` @@ -48,13 +52,54 @@ type AzureConfig struct { // See https://learn.microsoft.com/en-us/graph/api/group-list GroupsFilter string `yaml:"groups_filter"` // GroupsDisplayNameSuffixPostFilter applied to the fetched groups display names. - GroupsDisplayNameSuffixPostFilter string `yaml:"groups_display_name_suffix_post_filter"` - Timeout time.Duration `yaml:"timeout"` + Timeout time.Duration `yaml:"timeout"` + + // TODO(nadya73): support for ldap also, but with other name. + GroupsDisplayNameSuffixPostFilter string `yaml:"groups_display_name_suffix_post_filter"` + // TODO(nadya73): support for ldap also, but with other name. // DebugAzureIDs is a list of ids for which app will print more debug info in logs. DebugAzureIDs []string `yaml:"debug_azure_ids"` } +type LdapUsersConfig struct { + // A filter for getting users. + // For example, `(objectClass=account)`. + Filter string `yaml:"filter"` + // An attribute type which will be used as @name attribute. + // For example, `cn`. + UsernameAttributeType string `yaml:"username_attribute_type"` + // For example, `uid`. + UidAttributeType string `yaml:"uid_attribute_type"` + FirstNameAttributeType *string `yaml:"first_name_attribute_type"` + // A list of usernames for which app will print more debug info in logs. + DebugUsernames []string `yaml:"debug_usernames"` +} + +type LdapGroupsConfig struct { + // A filter for getting groups. + // For example, `(objectClass=posixGroup)`. + Filter string `yaml:"filter"` + // An attribute type which will be used as @name attribute. + // For example, `cn`. + GroupnameAttributeType string `yaml:"groupname_attribute_type"` + // An attribute type which will be used for getting group members. + // For example, `memberUid`. + MemberUidAttributeType string `yaml:"member_uid_attribute_type"` + + // A list of groupnames for which app will print more debug info in logs. + DebugGroupnames []string `yaml:"debug_groupnames"` +} + +type LdapConfig struct { + Address string `yaml:"address"` + BindDN string `yaml:"bind_dn"` + BindPasswordEnvVar string `yaml:"bind_password_env_var"` + Users LdapUsersConfig `yaml:"users"` + Groups LdapGroupsConfig `yaml:"groups"` + BaseDN string `yaml:"base_dn"` +} + type YtsaurusConfig struct { Proxy string `yaml:"proxy"` // SecretEnvVar is a name of env variable with YTsaurus token. Default: "YT_TOKEN". diff --git a/config_test.go b/config_test.go index 53c502e..6d075a2 100644 --- a/config_test.go +++ b/config_test.go @@ -2,6 +2,7 @@ package main import ( "embed" + "go.ytsaurus.tech/library/go/ptr" "testing" "time" @@ -17,7 +18,7 @@ func TestConfig(t *testing.T) { cfg, err := loadConfig(configPath) require.NoError(t, err) - require.Equal(t, 5*time.Minute, cfg.App.SyncInterval) + require.Equal(t, ptr.Duration(5*time.Minute), cfg.App.SyncInterval) require.Equal(t, []ReplacementPair{ {From: "@acme.com", To: ""}, {From: "@", To: ":"}, @@ -25,8 +26,8 @@ func TestConfig(t *testing.T) { require.Equal(t, []ReplacementPair{ {From: "|all", To: ""}, }, cfg.App.GroupnameReplacements) - require.Equal(t, 10, cfg.App.RemoveLimit) - require.Equal(t, 7*24*time.Hour, cfg.App.BanBeforeRemoveDuration) + require.Equal(t, ptr.Int(10), cfg.App.RemoveLimit) + require.Equal(t, ptr.Duration(7*24*time.Hour), cfg.App.BanBeforeRemoveDuration) require.Equal(t, "acme.onmicrosoft.com", cfg.Azure.Tenant) require.Equal(t, "abcdefgh-a000-b111-c222-abcdef123456", cfg.Azure.ClientID) @@ -45,7 +46,7 @@ func TestConfig(t *testing.T) { require.Equal(t, "WARN", cfg.Logging.Level) require.Equal(t, true, cfg.Logging.IsProduction) - logger, err := configureLogger(cfg.Logging) + logger, err := configureLogger(&cfg.Logging) require.NoError(t, err) logger.Debugw("test logging message", "key", "val") } diff --git a/diff.go b/diff.go index d5802c0..771f662 100644 --- a/diff.go +++ b/diff.go @@ -25,36 +25,42 @@ func (a *App) syncOnce() { } func (a *App) isRemoveLimitReached(objectsCount int) bool { - if a.removeLimit <= 0 { + if a.removeLimit == nil || *a.removeLimit <= 0 { return false } - return objectsCount >= a.removeLimit + return objectsCount >= *a.removeLimit } -// syncUsers syncs AD users with YTsaurus cluster and returns /actual/ map[AzureID]YtsaurusUser +// syncUsers syncs AD users with YTsaurus cluster and returns /actual/ map[ObjectID]YtsaurusUser // after applying changes. -func (a *App) syncUsers() (map[AzureID]YtsaurusUser, error) { +func (a *App) syncUsers() (map[ObjectID]YtsaurusUser, error) { a.logger.Info("Start syncing users") - azureUsers, err := a.azure.GetUsers() - if err != nil { - return nil, errors.Wrap(err, "failed to get Azure users") + var err error + var sourceUsers []SourceUser + + if a.source != nil { + sourceUsers, err = a.source.GetUsers() + if err != nil { + return nil, errors.Wrap(err, "failed to get Source users") + } } + ytUsers, err := a.ytsaurus.GetUsers() if err != nil { return nil, errors.Wrap(err, "failed to get YTsaurus users") } - azureUsersMap := make(map[AzureID]AzureUser) - ytUsersMap := make(map[AzureID]YtsaurusUser) + sourceUsersMap := make(map[ObjectID]SourceUser) + ytUsersMap := make(map[ObjectID]YtsaurusUser) - for _, user := range azureUsers { - azureUsersMap[user.AzureID] = user + for _, user := range sourceUsers { + sourceUsersMap[user.GetId()] = user } for _, user := range ytUsers { - ytUsersMap[user.AzureID] = user + ytUsersMap[user.SourceUser.GetId()] = user } - diff := a.diffUsers(azureUsersMap, ytUsersMap) + diff := a.diffUsers(sourceUsersMap, ytUsersMap) if a.isRemoveLimitReached(len(diff.remove)) { return nil, fmt.Errorf("delete limit in one cycle reached: %d %v", len(diff.remove), diff) } @@ -74,7 +80,7 @@ func (a *App) syncUsers() (map[AzureID]YtsaurusUser, error) { removedCount++ } // Actualizing user map for group sync later. - delete(ytUsersMap, user.AzureID) + delete(ytUsersMap, user.SourceUser.GetId()) } for _, user := range diff.create { err = a.ytsaurus.CreateUser(user) @@ -83,7 +89,7 @@ func (a *App) syncUsers() (map[AzureID]YtsaurusUser, error) { a.logger.Errorw("failed to create user", zap.Error(err), "user", user) } // Actualizing user map for group sync later. - ytUsersMap[user.AzureID] = user + ytUsersMap[user.SourceUser.GetId()] = user } for _, updatedUser := range diff.update { err = a.ytsaurus.UpdateUser(updatedUser.OldUsername, updatedUser.YtsaurusUser) @@ -92,7 +98,7 @@ func (a *App) syncUsers() (map[AzureID]YtsaurusUser, error) { a.logger.Errorw("failed to update user", zap.Error(err), "user", updatedUser) } // Actualizing user map for group sync later. - ytUsersMap[updatedUser.AzureID] = updatedUser.YtsaurusUser + ytUsersMap[updatedUser.SourceUser.GetId()] = updatedUser.YtsaurusUser } a.logger.Infow("Finish syncing users", "created", len(diff.create)-createErrCount, @@ -106,11 +112,11 @@ func (a *App) syncUsers() (map[AzureID]YtsaurusUser, error) { return ytUsersMap, nil } -func (a *App) syncGroups(usersMap map[AzureID]YtsaurusUser) error { +func (a *App) syncGroups(usersMap map[ObjectID]YtsaurusUser) error { a.logger.Info("Start syncing groups") - azureGroups, err := a.azure.GetGroupsWithMembers() + azureGroups, err := a.source.GetGroupsWithMembers() if err != nil { - return errors.Wrap(err, "failed to get Azure groups") + return errors.Wrap(err, "failed to get Source groups") } ytGroups, err := a.ytsaurus.GetGroupsWithMembers() if err != nil { @@ -189,30 +195,30 @@ type groupDiff struct { } func (a *App) diffGroups( - azureGroups []AzureGroupWithMembers, + sourceGroups []SourceGroupWithMembers, ytGroups []YtsaurusGroupWithMembers, - usersMap map[AzureID]YtsaurusUser, + usersMap map[ObjectID]YtsaurusUser, ) groupDiff { var groupsToCreate, groupsToRemove []YtsaurusGroup var groupsToUpdate []UpdatedYtsaurusGroup var membersToAdd, membersToRemove []YtsaurusMembership - azureGroupsWithMembersMap := make(map[AzureID]AzureGroupWithMembers) - ytGroupsWithMembersMap := make(map[AzureID]YtsaurusGroupWithMembers) + sourceGroupsWithMembersMap := make(map[ObjectID]SourceGroupWithMembers) + ytGroupsWithMembersMap := make(map[ObjectID]YtsaurusGroupWithMembers) - for _, group := range azureGroups { - azureGroupsWithMembersMap[group.AzureID] = group + for _, group := range sourceGroups { + sourceGroupsWithMembersMap[group.SourceGroup.GetId()] = group } for _, group := range ytGroups { - ytGroupsWithMembersMap[group.AzureID] = group + ytGroupsWithMembersMap[group.SourceGroup.GetId()] = group } - // Collecting groups to create (the ones that exist in Azure but not in YTsaurus). - for azureID, azureGroupWithMembers := range azureGroupsWithMembersMap { - if _, ok := ytGroupsWithMembersMap[azureID]; !ok { - newYtsaurusGroup := a.buildYtsaurusGroup(azureGroupWithMembers.AzureGroup) + // Collecting groups to create (the ones that exist in Source but not in YTsaurus). + for objectID, sourceGroupWithMembers := range sourceGroupsWithMembersMap { + if _, ok := ytGroupsWithMembersMap[objectID]; !ok { + newYtsaurusGroup := a.buildYtsaurusGroup(sourceGroupWithMembers.SourceGroup) groupsToCreate = append(groupsToCreate, newYtsaurusGroup) - for username := range a.buildYtsaurusGroupMembers(azureGroupWithMembers, usersMap).Iter() { + for username := range a.buildYtsaurusGroupMembers(sourceGroupWithMembers, usersMap).Iter() { membersToAdd = append(membersToAdd, YtsaurusMembership{ GroupName: newYtsaurusGroup.Name, Username: username, @@ -221,17 +227,17 @@ func (a *App) diffGroups( } } - for azureID, ytGroupWithMembers := range ytGroupsWithMembersMap { - // Collecting groups to remove (the ones that exist in YTsaurus and not in Azure). - azureGroupWithMembers, ok := azureGroupsWithMembersMap[azureID] + for objectID, ytGroupWithMembers := range ytGroupsWithMembersMap { + // Collecting groups to remove (the ones that exist in YTsaurus and not in Source). + sourceGroupWithMembers, ok := sourceGroupsWithMembersMap[objectID] if !ok { groupsToRemove = append(groupsToRemove, ytGroupWithMembers.YtsaurusGroup) continue } - // Collecting groups with changed Azure fields (actually we have only displayName for now which + // Collecting groups with changed Source fields (actually we have only displayName for now which // should change, though we still handle that just in case). - groupChanged, updatedYtGroup := a.isGroupChanged(azureGroupWithMembers.AzureGroup, ytGroupWithMembers.YtsaurusGroup) + groupChanged, updatedYtGroup := a.isGroupChanged(sourceGroupWithMembers.SourceGroup, ytGroupWithMembers.YtsaurusGroup) // Group name can change after update, so we ensure that correct one is used for membership updates. actualGroupname := ytGroupWithMembers.YtsaurusGroup.Name if groupChanged { @@ -244,7 +250,7 @@ func (a *App) diffGroups( actualGroupname = updatedYtGroup.YtsaurusGroup.Name } - membersCreate, membersRemove := a.isGroupMembersChanged(azureGroupWithMembers, ytGroupWithMembers, usersMap) + membersCreate, membersRemove := a.isGroupMembersChanged(sourceGroupWithMembers, ytGroupWithMembers, usersMap) for _, username := range membersCreate { membersToAdd = append(membersToAdd, YtsaurusMembership{ GroupName: actualGroupname, @@ -275,24 +281,24 @@ type usersDiff struct { } func (a *App) diffUsers( - azureUsersMap map[AzureID]AzureUser, - ytUsersMap map[AzureID]YtsaurusUser, + sourceUsersMap map[ObjectID]SourceUser, + ytUsersMap map[ObjectID]YtsaurusUser, ) usersDiff { var create, remove []YtsaurusUser var update []UpdatedYtsaurusUser - for azureID, azureUser := range azureUsersMap { - if _, ok := ytUsersMap[azureID]; !ok { - create = append(create, a.buildYtsaurusUser(azureUser)) + for objectID, sourceUser := range sourceUsersMap { + if _, ok := ytUsersMap[objectID]; !ok { + create = append(create, a.buildYtsaurusUser(sourceUser)) } } - for azureID, ytUser := range ytUsersMap { - azureUser, ok := azureUsersMap[azureID] + for objectID, ytUser := range ytUsersMap { + sourceUser, ok := sourceUsersMap[objectID] if !ok { remove = append(remove, ytUser) continue } - userChanged, updatedYtUser := a.isUserChanged(azureUser, ytUser) + userChanged, updatedYtUser := a.isUserChanged(sourceUser, ytUser) if !userChanged { continue } @@ -305,49 +311,47 @@ func (a *App) diffUsers( } } -func (a *App) buildUsername(azureUser AzureUser) string { - username := azureUser.PrincipalName - for _, replace := range a.usernameReplaces { - username = strings.Replace(username, replace.From, replace.To, -1) +func (a *App) buildUsername(sourceUser SourceUser) string { + username := sourceUser.GetName() + if a.usernameReplaces != nil { + for _, replace := range a.usernameReplaces { + username = strings.Replace(username, replace.From, replace.To, -1) + } } username = strings.ToLower(username) return username } -func (a *App) buildGroupName(azureGroup AzureGroup) string { - name := azureGroup.Identity - for _, replace := range a.groupnameReplaces { - name = strings.Replace(name, replace.From, replace.To, -1) +func (a *App) buildGroupName(sourceGroup SourceGroup) string { + name := sourceGroup.GetName() + if a.groupnameReplaces != nil { + for _, replace := range a.groupnameReplaces { + name = strings.Replace(name, replace.From, replace.To, -1) + } } name = strings.ToLower(name) return name } -func (a *App) buildYtsaurusUser(azureUser AzureUser) YtsaurusUser { +func (a *App) buildYtsaurusUser(sourceUser SourceUser) YtsaurusUser { return YtsaurusUser{ - Username: a.buildUsername(azureUser), - AzureID: azureUser.AzureID, - PrincipalName: azureUser.PrincipalName, - Email: azureUser.Email, - FirstName: azureUser.FirstName, - LastName: azureUser.LastName, - DisplayName: azureUser.DisplayName, - // If we have Azure user —> he is not banned. + Username: a.buildUsername(sourceUser), + SourceUser: sourceUser, + // If we have Source user —> he is not banned. BannedSince: time.Time{}, } } -func (a *App) buildYtsaurusGroup(azureGroup AzureGroup) YtsaurusGroup { +func (a *App) buildYtsaurusGroup(sourceGroup SourceGroup) YtsaurusGroup { return YtsaurusGroup{ - Name: a.buildGroupName(azureGroup), - AzureID: azureGroup.AzureID, - DisplayName: azureGroup.DisplayName, + Name: a.buildGroupName(sourceGroup), + SourceGroup: sourceGroup, } } -func (a *App) buildYtsaurusGroupMembers(azureGroupWithMembers AzureGroupWithMembers, usersMap map[AzureID]YtsaurusUser) StringSet { +func (a *App) buildYtsaurusGroupMembers(sourceGroupWithMembers SourceGroupWithMembers, usersMap map[ObjectID]YtsaurusUser) StringSet { members := NewStringSet() - for azureID := range azureGroupWithMembers.Members.Iter() { + for azureID := range sourceGroupWithMembers.Members.Iter() { ytUser, ok := usersMap[azureID] if !ok { // User is unknown to the YTsaurus (can be accountEnabled=false). @@ -366,8 +370,8 @@ type UpdatedYtsaurusUser struct { } // If isUserChanged detects that user is changed, it returns UpdatedYtsaurusUser. -func (a *App) isUserChanged(azureUser AzureUser, ytUser YtsaurusUser) (bool, UpdatedYtsaurusUser) { - newUser := a.buildYtsaurusUser(azureUser) +func (a *App) isUserChanged(sourceUser SourceUser, ytUser YtsaurusUser) (bool, UpdatedYtsaurusUser) { + newUser := a.buildYtsaurusUser(sourceUser) if newUser == ytUser { return false, UpdatedYtsaurusUser{} } @@ -382,17 +386,17 @@ type UpdatedYtsaurusGroup struct { } // If isGroupChanged detects that group itself (not members) is changed, it returns UpdatedYtsaurusGroup. -func (a *App) isGroupChanged(azureGroup AzureGroup, ytGroup YtsaurusGroup) (bool, UpdatedYtsaurusGroup) { - newGroup := a.buildYtsaurusGroup(azureGroup) - if newGroup.Name == ytGroup.Name && newGroup.DisplayName == ytGroup.DisplayName { +func (a *App) isGroupChanged(sourceGroup SourceGroup, ytGroup YtsaurusGroup) (bool, UpdatedYtsaurusGroup) { + newGroup := a.buildYtsaurusGroup(sourceGroup) + if newGroup == ytGroup { return false, UpdatedYtsaurusGroup{} } return true, UpdatedYtsaurusGroup{YtsaurusGroup: newGroup, OldName: ytGroup.Name} } // If isGroupMembersChanged detects that group members are changed, it returns lists of usernames to create and remove. -func (a *App) isGroupMembersChanged(azureGroup AzureGroupWithMembers, ytGroup YtsaurusGroupWithMembers, usersMap map[AzureID]YtsaurusUser) (create, remove []string) { - newMembers := a.buildYtsaurusGroupMembers(azureGroup, usersMap) +func (a *App) isGroupMembersChanged(sourceGroup SourceGroupWithMembers, ytGroup YtsaurusGroupWithMembers, usersMap map[ObjectID]YtsaurusUser) (create, remove []string) { + newMembers := a.buildYtsaurusGroupMembers(sourceGroup, usersMap) oldMembers := ytGroup.Members create = newMembers.Difference(oldMembers).ToSlice() @@ -402,15 +406,15 @@ func (a *App) isGroupMembersChanged(azureGroup AzureGroupWithMembers, ytGroup Yt func (a *App) banOrRemoveUser(user YtsaurusUser) (wasBanned, wasRemoved bool, err error) { // Ban settings is disabled. - if a.banDuration == 0 { + if a.banDuration == nil { return false, true, a.ytsaurus.RemoveUser(user.Username) } // If user is not already banned we should do it. - if !user.IsBanned() && a.banDuration != 0 { + if !user.IsBanned() && *a.banDuration != 0 { return true, false, a.ytsaurus.BanUser(user.Username) } // If user was banned longer than setting permits, we remove it. - if user.IsBanned() && time.Since(user.BannedSince) > a.banDuration { + if user.IsBanned() && time.Since(user.BannedSince) > *a.banDuration { return false, true, a.ytsaurus.RemoveUser(user.Username) } a.logger.Debugw("user is banned, but not yet removed", "user", user.Username, "since", user.BannedSince) diff --git a/go.mod b/go.mod index f340080..c29f748 100644 --- a/go.mod +++ b/go.mod @@ -5,14 +5,14 @@ go 1.20 require ( github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.4.0 github.com/deckarep/golang-set/v2 v2.3.1 - github.com/google/go-cmp v0.5.9 + github.com/google/go-cmp v0.6.0 github.com/jessevdk/go-flags v1.5.0 github.com/microsoft/kiota-abstractions-go v1.3.0 github.com/microsoftgraph/msgraph-sdk-go v1.24.0 github.com/microsoftgraph/msgraph-sdk-go-core v1.0.0 github.com/pkg/errors v0.9.1 github.com/stretchr/testify v1.8.4 - github.com/testcontainers/testcontainers-go v0.26.0 + github.com/testcontainers/testcontainers-go v0.28.0 go.uber.org/zap v1.26.0 go.ytsaurus.tech/yt/go v0.0.13 gopkg.in/yaml.v3 v3.0.1 @@ -24,20 +24,25 @@ require ( github.com/Azure/azure-sdk-for-go/sdk/azcore v1.8.0 // indirect github.com/Azure/azure-sdk-for-go/sdk/internal v1.3.0 // indirect github.com/Azure/go-ansiterm v0.0.0-20210617225240-d185dfc1b5a1 // indirect + github.com/Azure/go-ntlmssp v0.0.0-20221128193559-754e69321358 // indirect github.com/AzureAD/microsoft-authentication-library-for-go v1.2.0 // indirect github.com/Microsoft/go-winio v0.6.1 // indirect - github.com/Microsoft/hcsshim v0.11.1 // indirect + github.com/Microsoft/hcsshim v0.11.4 // indirect github.com/andybalholm/brotli v1.0.4 // indirect github.com/cenkalti/backoff/v4 v4.2.1 // indirect github.com/cjlapao/common-go v0.0.39 // indirect - github.com/containerd/containerd v1.7.7 // indirect + github.com/containerd/containerd v1.7.12 // indirect github.com/containerd/log v0.1.0 // indirect github.com/cpuguy83/dockercfg v0.3.1 // indirect github.com/davecgh/go-spew v1.1.1 // indirect + github.com/distribution/reference v0.5.0 // indirect github.com/docker/distribution v2.8.2+incompatible // indirect - github.com/docker/docker v24.0.6+incompatible // indirect - github.com/docker/go-connections v0.4.0 // indirect + github.com/docker/docker v25.0.2+incompatible // indirect + github.com/docker/go-connections v0.5.0 // indirect github.com/docker/go-units v0.5.0 // indirect + github.com/felixge/httpsnoop v1.0.3 // indirect + github.com/go-asn1-ber/asn1-ber v1.5.5 // indirect + github.com/go-ldap/ldap/v3 v3.4.6 // indirect github.com/go-logr/logr v1.2.4 // indirect github.com/go-logr/stdr v1.2.2 // indirect github.com/go-ole/go-ole v1.2.6 // indirect @@ -47,7 +52,7 @@ require ( github.com/golang/protobuf v1.5.3 // indirect github.com/golang/snappy v0.0.4 // indirect github.com/google/tink/go v1.7.0 // indirect - github.com/google/uuid v1.4.0 // indirect + github.com/google/uuid v1.6.0 // indirect github.com/klauspost/compress v1.16.7 // indirect github.com/kylelemons/godebug v1.1.0 // indirect github.com/lufia/plan9stats v0.0.0-20211012122336-39d0f177ccd0 // indirect @@ -62,6 +67,7 @@ require ( github.com/mitchellh/reflectwalk v1.0.2 // indirect github.com/moby/patternmatcher v0.6.0 // indirect github.com/moby/sys/sequential v0.5.0 // indirect + github.com/moby/sys/user v0.1.0 // indirect github.com/moby/term v0.5.0 // indirect github.com/morikuni/aec v1.0.0 // indirect github.com/opencontainers/go-digest v1.0.0 // indirect @@ -72,13 +78,15 @@ require ( github.com/pkg/browser v0.0.0-20210911075715-681adbf594b8 // indirect github.com/pmezard/go-difflib v1.0.0 // indirect github.com/power-devops/perfstat v0.0.0-20210106213030-5aafc221ea8c // indirect - github.com/shirou/gopsutil/v3 v3.23.9 // indirect + github.com/shirou/gopsutil/v3 v3.23.12 // indirect github.com/shoenig/go-m1cpu v0.1.6 // indirect github.com/sirupsen/logrus v1.9.3 // indirect github.com/std-uritemplate/std-uritemplate/go v0.0.42 // indirect + github.com/testcontainers/testcontainers-go/modules/openldap v0.28.0 // indirect github.com/tklauser/go-sysconf v0.3.12 // indirect github.com/tklauser/numcpus v0.6.1 // indirect github.com/yusufpapurcu/wmi v1.2.3 // indirect + go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.45.0 // indirect go.opentelemetry.io/otel v1.19.0 // indirect go.opentelemetry.io/otel/metric v1.19.0 // indirect go.opentelemetry.io/otel/trace v1.19.0 // indirect @@ -90,15 +98,15 @@ require ( go.ytsaurus.tech/library/go/ptr v0.0.1 // indirect go.ytsaurus.tech/library/go/x/xreflect v0.0.2 // indirect go.ytsaurus.tech/library/go/x/xruntime v0.0.3 // indirect - golang.org/x/crypto v0.14.0 // indirect + golang.org/x/crypto v0.17.0 // indirect golang.org/x/exp v0.0.0-20230725093048-515e97ebf090 // indirect golang.org/x/mod v0.11.0 // indirect golang.org/x/net v0.17.0 // indirect - golang.org/x/sys v0.13.0 // indirect - golang.org/x/text v0.13.0 // indirect - golang.org/x/tools v0.7.0 // indirect + golang.org/x/sys v0.16.0 // indirect + golang.org/x/text v0.14.0 // indirect + golang.org/x/tools v0.10.0 // indirect golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2 // indirect - google.golang.org/genproto/googleapis/rpc v0.0.0-20230525234030-28d5490b6b19 // indirect - google.golang.org/grpc v1.57.1 // indirect + google.golang.org/genproto/googleapis/rpc v0.0.0-20230711160842-782d3b101e98 // indirect + google.golang.org/grpc v1.58.3 // indirect google.golang.org/protobuf v1.31.0 // indirect ) diff --git a/go.sum b/go.sum index 2eb4fa1..8df8c6e 100644 --- a/go.sum +++ b/go.sum @@ -9,6 +9,8 @@ github.com/Azure/azure-sdk-for-go/sdk/internal v1.3.0 h1:sXr+ck84g/ZlZUOZiNELInm github.com/Azure/azure-sdk-for-go/sdk/internal v1.3.0/go.mod h1:okt5dMMTOFjX/aovMlrjvvXoPMBVSPzk9185BT0+eZM= github.com/Azure/go-ansiterm v0.0.0-20210617225240-d185dfc1b5a1 h1:UQHMgLO+TxOElx5B5HZ4hJQsoJ/PvUvKRhJHDQXO8P8= github.com/Azure/go-ansiterm v0.0.0-20210617225240-d185dfc1b5a1/go.mod h1:xomTg63KZ2rFqZQzSB4Vz2SUXa1BpHTVz9L5PTmPC4E= +github.com/Azure/go-ntlmssp v0.0.0-20221128193559-754e69321358 h1:mFRzDkZVAjdal+s7s0MwaRv9igoPqLRdzOLzw/8Xvq8= +github.com/Azure/go-ntlmssp v0.0.0-20221128193559-754e69321358/go.mod h1:chxPXzSsl7ZWRAuOIE23GDNzjWuZquvFlgA8xmpunjU= github.com/AzureAD/microsoft-authentication-library-for-go v1.2.0 h1:hVeq+yCyUi+MsoO/CU95yqCIcdzra5ovzk8Q2BBpV2M= github.com/AzureAD/microsoft-authentication-library-for-go v1.2.0/go.mod h1:wP83P5OoQ5p6ip3ScPr0BAq0BvuPAvacpEuSzyouqAI= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= @@ -16,6 +18,9 @@ github.com/Microsoft/go-winio v0.6.1 h1:9/kr64B9VUZrLm5YYwbGtUJnMgqWVOdUAXu6Migc github.com/Microsoft/go-winio v0.6.1/go.mod h1:LRdKpFKfdobln8UmuiYcKPot9D2v6svN5+sAH+4kjUM= github.com/Microsoft/hcsshim v0.11.1 h1:hJ3s7GbWlGK4YVV92sO88BQSyF4ZLVy7/awqOlPxFbA= github.com/Microsoft/hcsshim v0.11.1/go.mod h1:nFJmaO4Zr5Y7eADdFOpYswDDlNVbvcIJJNJLECr5JQg= +github.com/Microsoft/hcsshim v0.11.4 h1:68vKo2VN8DE9AdN4tnkWnmdhqdbpUFM8OF3Airm7fz8= +github.com/Microsoft/hcsshim v0.11.4/go.mod h1:smjE4dvqPX9Zldna+t5FG3rnoHhaB7QYxPRqGcpAD9w= +github.com/alexbrainman/sspi v0.0.0-20210105120005-909beea2cc74/go.mod h1:cEWa1LVoE5KvSD9ONXsZrj0z6KqySlCCNKHlLzbqAt4= github.com/andybalholm/brotli v1.0.4 h1:V7DdXeJtZscaqfNuAdSRuRFzuiKlHSC/Zh3zl9qY3JY= github.com/andybalholm/brotli v1.0.4/go.mod h1:fO7iG3H7G2nSZ7m0zPUDn85XEX2GTukHGRSepvi9Eig= github.com/cenkalti/backoff/v4 v4.2.1 h1:y4OZtCnogmCPw98Zjyt5a6+QwPLGkiQsYW5oUqylYbM= @@ -27,6 +32,8 @@ github.com/cjlapao/common-go v0.0.39/go.mod h1:M3dzazLjTjEtZJbbxoA5ZDiGCiHmpwqW9 github.com/containerd/console v1.0.3/go.mod h1:7LqA/THxQ86k76b8c/EMSiaJ3h1eZkMkXar0TQ1gf3U= github.com/containerd/containerd v1.7.7 h1:QOC2K4A42RQpcrZyptP6z9EJZnlHfHJUfZrAAHe15q4= github.com/containerd/containerd v1.7.7/go.mod h1:3c4XZv6VeT9qgf9GMTxNTMFxGJrGpI2vz1yk4ye+YY8= +github.com/containerd/containerd v1.7.12 h1:+KQsnv4VnzyxWcfO9mlxxELaoztsDEjOuCMPAuPqgU0= +github.com/containerd/containerd v1.7.12/go.mod h1:/5OMpE1p0ylxtEUGY8kuCYkDRzJm9NO1TFMWjUpdevk= github.com/containerd/log v0.1.0 h1:TCJt7ioM2cr/tfR8GPbGf9/VRAX8D2B4PjzCpfX540I= github.com/containerd/log v0.1.0/go.mod h1:VRRf09a7mHDIRezVKTRCrOq78v577GXq3bSa3EhrzVo= github.com/coreos/go-systemd/v22 v22.3.2/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc= @@ -40,18 +47,30 @@ github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/deckarep/golang-set/v2 v2.3.1 h1:vjmkvJt/IV27WXPyYQpAh4bRyWJc5Y435D17XQ9QU5A= github.com/deckarep/golang-set/v2 v2.3.1/go.mod h1:VAky9rY/yGXJOLEDv3OMci+7wtDpOF4IN+y82NBOac4= +github.com/distribution/reference v0.5.0 h1:/FUIFXtfc/x2gpa5/VGfiGLuOIdYa1t65IKK2OFGvA0= +github.com/distribution/reference v0.5.0/go.mod h1:BbU0aIcezP1/5jX/8MP0YiH4SdvB5Y4f/wlDRiLyi3E= github.com/dnaeon/go-vcr v1.2.0 h1:zHCHvJYTMh1N7xnV7zf1m1GPBF9Ad0Jk/whtQ1663qI= github.com/docker/distribution v2.8.2+incompatible h1:T3de5rq0dB1j30rp0sA2rER+m322EBzniBPB6ZIzuh8= github.com/docker/distribution v2.8.2+incompatible/go.mod h1:J2gT2udsDAN96Uj4KfcMRqY0/ypR+oyYUYmja8H+y+w= github.com/docker/docker v24.0.6+incompatible h1:hceabKCtUgDqPu+qm0NgsaXf28Ljf4/pWFL7xjWWDgE= github.com/docker/docker v24.0.6+incompatible/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk= +github.com/docker/docker v25.0.2+incompatible h1:/OaKeauroa10K4Nqavw4zlhcDq/WBcPMc5DbjOGgozY= +github.com/docker/docker v25.0.2+incompatible/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk= github.com/docker/go-connections v0.4.0 h1:El9xVISelRB7BuFusrZozjnkIM5YnzCViNKohAFqRJQ= github.com/docker/go-connections v0.4.0/go.mod h1:Gbd7IOopHjR8Iph03tsViu4nIes5XhDvyHbTtUxmeec= +github.com/docker/go-connections v0.5.0 h1:USnMq7hx7gwdVZq1L49hLXaFtUdTADjXGp+uj1Br63c= +github.com/docker/go-connections v0.5.0/go.mod h1:ov60Kzw0kKElRwhNs9UlUHAE/F9Fe6GLaXnqyDdmEXc= github.com/docker/go-units v0.4.0/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk= github.com/docker/go-units v0.5.0 h1:69rxXcBk27SvSaaxTtLh/8llcHD8vYHT7WSdRZ/jvr4= github.com/docker/go-units v0.5.0/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk= +github.com/felixge/httpsnoop v1.0.3 h1:s/nj+GCswXYzN5v2DpNMuMQYe+0DDwt5WVCU6CWBdXk= +github.com/felixge/httpsnoop v1.0.3/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U= github.com/frankban/quicktest v1.11.3/go.mod h1:wRf/ReqHper53s+kmmSZizM8NamnL3IM0I9ntUbOk+k= github.com/frankban/quicktest v1.14.5 h1:dfYrrRyLtiqT9GyKXgdh+k4inNeTvmGbuSgZ3lx3GhA= +github.com/go-asn1-ber/asn1-ber v1.5.5 h1:MNHlNMBDgEKD4TcKr36vQN68BA00aDfjIt3/bD50WnA= +github.com/go-asn1-ber/asn1-ber v1.5.5/go.mod h1:hEBeB/ic+5LoWskz+yKT7vGhhPYkProFKoKdwZRWMe0= +github.com/go-ldap/ldap/v3 v3.4.6 h1:ert95MdbiG7aWo/oPYp9btL3KJlMPKnP58r09rI8T+A= +github.com/go-ldap/ldap/v3 v3.4.6/go.mod h1:IGMQANNtxpsOzj7uUAMjpGBaOVTC4DYyIy8VsTdxmtc= github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= github.com/go-logr/logr v1.2.4 h1:g01GSCwiDw2xSZfjJ2/T9M+S6pFdcNtFYsp+Y43HYDQ= github.com/go-logr/logr v1.2.4/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= @@ -77,10 +96,15 @@ github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/ github.com/google/go-cmp v0.5.6/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38= github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= +github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= +github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/google/tink/go v1.7.0 h1:6Eox8zONGebBFcCBqkVmt60LaWZa6xg1cl/DwAh/J1w= github.com/google/tink/go v1.7.0/go.mod h1:GAUOd+QE3pgj9q8VKIGTCP33c/B7eb4NhxLcgTJZStM= +github.com/google/uuid v1.3.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/google/uuid v1.4.0 h1:MtMxsa51/r9yyhkyLsVeVt0B+BGQZzpQiTQ4eHZ8bc4= github.com/google/uuid v1.4.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= +github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/jessevdk/go-flags v1.5.0 h1:1jKYvbxEjfUl0fmqTCOfonvskHHXMjBySTLW4y9LFvc= github.com/jessevdk/go-flags v1.5.0/go.mod h1:Fw0T6WPc1dYxT4mKEZRfG5kJhaTDP9pj1c2EWnYs/m4= github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= @@ -125,6 +149,8 @@ github.com/moby/patternmatcher v0.6.0/go.mod h1:hDPoyOpDY7OrrMDLaYoY3hf52gNCR/YO github.com/moby/sys/mountinfo v0.5.0/go.mod h1:3bMD3Rg+zkqx8MRYPi7Pyb0Ie97QEBmdxbhnCLlSvSU= github.com/moby/sys/sequential v0.5.0 h1:OPvI35Lzn9K04PBbCLW0g4LcFAJgHsvXsRyewg5lXtc= github.com/moby/sys/sequential v0.5.0/go.mod h1:tH2cOOs5V9MlPiXcQzRC+eEyab644PWKGRYaaV5ZZlo= +github.com/moby/sys/user v0.1.0 h1:WmZ93f5Ux6het5iituh9x2zAG7NFY9Aqi49jjE1PaQg= +github.com/moby/sys/user v0.1.0/go.mod h1:fKJhFOnsCN6xZ5gSfbM6zaHGgDJMrqt9/reuj4T7MmU= github.com/moby/term v0.5.0 h1:xt8Q1nalod/v7BqbG21f8mQPqH+xAaC9C3N3wfWbVP0= github.com/moby/term v0.5.0/go.mod h1:8FzsFHVUBGZdbDsJw/ot+X+d5HLUbvklYLJ9uGfcI3Y= github.com/morikuni/aec v1.0.0 h1:nP9CBfwrvYnBRgY6qfDQkygYDmYwOilePFkwzv4dU8A= @@ -155,6 +181,8 @@ github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQD github.com/seccomp/libseccomp-golang v0.9.2-0.20220502022130-f33da4d89646/go.mod h1:JA8cRccbGaA1s33RQf7Y1+q9gHmZX1yB/z9WDN1C6fg= github.com/shirou/gopsutil/v3 v3.23.9 h1:ZI5bWVeu2ep4/DIxB4U9okeYJ7zp/QLTO4auRb/ty/E= github.com/shirou/gopsutil/v3 v3.23.9/go.mod h1:x/NWSb71eMcjFIO0vhyGW5nZ7oSIgVjrCnADckb85GA= +github.com/shirou/gopsutil/v3 v3.23.12 h1:z90NtUkp3bMtmICZKpC4+WaknU1eXtp5vtbQ11DgpE4= +github.com/shirou/gopsutil/v3 v3.23.12/go.mod h1:1FrWgea594Jp7qmjHUUPlJDTPgcsb9mGnXDxavtikzM= github.com/shoenig/go-m1cpu v0.1.6 h1:nxdKQNcEB6vzgA2E2bvzKIYRuNj7XNJ4S/aRSwKzFtM= github.com/shoenig/go-m1cpu v0.1.6/go.mod h1:1JJMcUBvfNwpq05QDQVAnx3gUHr9IYF7GNg9SUEw2VQ= github.com/shoenig/test v0.6.4 h1:kVTaSd7WLz5WZ2IaoM0RSzRsUD+m8wRR+5qvntpn4LU= @@ -178,6 +206,10 @@ github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXl github.com/syndtr/gocapability v0.0.0-20200815063812-42c35b437635/go.mod h1:hkRG7XYTFWNJGYcbNJQlaLq0fg1yr4J4t/NcTQtrfww= github.com/testcontainers/testcontainers-go v0.26.0 h1:uqcYdoOHBy1ca7gKODfBd9uTHVK3a7UL848z09MVZ0c= github.com/testcontainers/testcontainers-go v0.26.0/go.mod h1:ICriE9bLX5CLxL9OFQ2N+2N+f+803LNJ1utJb1+Inx0= +github.com/testcontainers/testcontainers-go v0.28.0 h1:1HLm9qm+J5VikzFDYhOd+Zw12NtOl+8drH2E8nTY1r8= +github.com/testcontainers/testcontainers-go v0.28.0/go.mod h1:COlDpUXbwW3owtpMkEB1zo9gwb1CoKVKlyrVPejF4AU= +github.com/testcontainers/testcontainers-go/modules/openldap v0.28.0 h1:hR3BtDFSPUXvMZNuVbw7YcqAFsvjwt/VNA6ZTjtDZiM= +github.com/testcontainers/testcontainers-go/modules/openldap v0.28.0/go.mod h1:8/VlUrL0D5j0G3Vy3c+APgdBH3RZJcIETkualsRA8W4= github.com/tklauser/go-sysconf v0.3.12 h1:0QaGUFOdQaIVdPgfITYzaTegZvdCjmYO52cSFAEVmqU= github.com/tklauser/go-sysconf v0.3.12/go.mod h1:Ho14jnntGE1fpdOqQEEaiKRpvIavV0hSfmBq8nJbHYI= github.com/tklauser/numcpus v0.6.1 h1:ng9scYS7az0Bk4OZLvrNXNSAO2Pxr1XXRAPyjhIx+Fk= @@ -187,8 +219,11 @@ github.com/vishvananda/netlink v1.1.0/go.mod h1:cTgwzPIzzgDAYoQrMm0EdrjRUBkTqKYp github.com/vishvananda/netns v0.0.0-20191106174202-0a2b9b5464df/go.mod h1:JP3t17pCcGlemwknint6hfoeCVQrEMVwxRLRjXpq+BU= github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= github.com/yusufpapurcu/wmi v1.2.3 h1:E1ctvB7uKFMOJw3fdOW32DwGE9I7t++CRUEMKvFoFiw= github.com/yusufpapurcu/wmi v1.2.3/go.mod h1:SBZ9tNy3G9/m5Oi98Zks0QjeHVDvuK0qfxQmPyzfmi0= +go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.45.0 h1:x8Z78aZx8cOF0+Kkazoc7lwUNMGy0LrzEMxTm4BbTxg= +go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.45.0/go.mod h1:62CPTSry9QZtOaSsE3tOzhx6LzDhHnXJ6xHeMNNiM6Q= go.opentelemetry.io/otel v1.19.0 h1:MuS/TNf4/j4IXsZuJegVzI1cwut7Qc00344rgH7p8bs= go.opentelemetry.io/otel v1.19.0/go.mod h1:i0QyjOq3UPoTzff0PJB2N66fb4S0+rSbSB15/oyH9fY= go.opentelemetry.io/otel/metric v1.19.0 h1:aTzpGtV0ar9wlV4Sna9sdJyII5jTVJEvKETPiOKwvpE= @@ -220,12 +255,18 @@ go.ytsaurus.tech/yt/go v0.0.13/go.mod h1:r3puwhqLgS/TD9VZ8/3rhuxtdD02bQ+yuZWncNu golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= +golang.org/x/crypto v0.13.0/go.mod h1:y6Z2r+Rw4iayiXXAIxJIDAJ1zMW4yaTpebo8fPOliYc= golang.org/x/crypto v0.14.0 h1:wBqGXzWJW6m1XrIKlAH0Hs1JJ7+9KBwnIO8v66Q9cHc= golang.org/x/crypto v0.14.0/go.mod h1:MVFd36DqK4CsrnJYDkBA3VC4m2GkXAM0PvzMCn4JQf4= +golang.org/x/crypto v0.17.0 h1:r8bRNjWL3GshPW3gkd+RpvzWrZAwPS49OmTGZ/uhM4k= +golang.org/x/crypto v0.17.0/go.mod h1:gCAAfMLgwOJRpTjQ2zCCt2OcSfYMTeZVSRtQlPC7Nq4= golang.org/x/exp v0.0.0-20230725093048-515e97ebf090 h1:Di6/M8l0O2lCLc6VVRWhgCiApHV8MnQurBnFSHsQtNY= golang.org/x/exp v0.0.0-20230725093048-515e97ebf090/go.mod h1:FXUEEKJgO7OQYeo8N01OfiKP8RXMtf6e8aTskBGqWdc= golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= +golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= golang.org/x/mod v0.11.0 h1:bUO06HqtnRcc/7l71XBe4WcqTZ+3AH1J59zWDDwLKgU= golang.org/x/mod v0.11.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= @@ -233,11 +274,17 @@ golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLL golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/net v0.0.0-20201224014010-6772e930b67b/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= +golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= +golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= +golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= +golang.org/x/net v0.10.0/go.mod h1:0qNGK6F8kojg2nk9dLZ2mShWaEBan6FAoqfSigmmuDg= golang.org/x/net v0.17.0 h1:pVaXccu2ozPjCXewfr1S7xza/zcXTity9cCdXQYSjIM= golang.org/x/net v0.17.0/go.mod h1:NxSsAGuq816PNPmqtQdLE42eU2Fs7NoRIZrHJAlaCOE= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.3.0 h1:ftCYgMx6zT/asHUrPw8BLLscYtGznsLAnjq5RH9P66E= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -250,29 +297,49 @@ golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20201204225414-ed752295db88/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210320140829-1e4c9ba3b0c4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210616045830-e2b7044e8c71/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210616094352-59db8d763f22/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210906170528-6f6e22806c34/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20211025201205-69cdffdb9359/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20211116061358-0a5406a5449c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.11.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.12.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.13.0 h1:Af8nKPmuFypiUBjVoU9V20FiaFXOcuZI21p0ycVYYGE= golang.org/x/sys v0.13.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.15.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/sys v0.16.0 h1:xWw16ngr6ZMtmxDyKyIgsE93KNKz5HKmMa3b8ALHidU= +golang.org/x/sys v0.16.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= +golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= +golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k= +golang.org/x/term v0.8.0/go.mod h1:xPskH00ivmX89bAKVGSKKtLOWNx2+17Eiy94tnKShWo= +golang.org/x/term v0.12.0/go.mod h1:owVbMEjm3cBLCHdkQu9b1opXd4ETQWc3BhuQGKgXgvU= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= +golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= +golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= golang.org/x/text v0.13.0 h1:ablQoSUd0tRdKxZewP80B+BaqeKJuVhuRxj/dkrun3k= golang.org/x/text v0.13.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= +golang.org/x/text v0.14.0 h1:ScX5w1eTa3QqT8oi6+ziP7dTV1S2+ALU0bI+0zXKWiQ= +golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= golang.org/x/time v0.3.0 h1:rg5rLMjNzMS1RkNLzCG38eapWhnYLFYXDXj2gOlr8j4= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= +golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= +golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU= golang.org/x/tools v0.7.0 h1:W4OVu8VVOaIO0yzWMNdepAulS7YfoS3Zabrm8DOXXU4= golang.org/x/tools v0.7.0/go.mod h1:4pg6aUX35JBAogB10C9AtvVL+qowtN4pT3CGSQex14s= +golang.org/x/tools v0.10.0 h1:tvDr/iQoUqNdohiYm0LmmKcBk+q86lb9EprIUFhHHGg= +golang.org/x/tools v0.10.0/go.mod h1:UJwyiVBsOA2uwvK/e5OY3GTpDUJriEd+/YlqAwLPmyM= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= @@ -281,8 +348,12 @@ golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2 h1:H2TDz8ibqkAF6YGhCdN3j golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2/go.mod h1:K8+ghG5WaK9qNqU5K3HdILfMLy1f3aNYFI/wnl100a8= google.golang.org/genproto/googleapis/rpc v0.0.0-20230525234030-28d5490b6b19 h1:0nDDozoAU19Qb2HwhXadU8OcsiO/09cnTqhUtq2MEOM= google.golang.org/genproto/googleapis/rpc v0.0.0-20230525234030-28d5490b6b19/go.mod h1:66JfowdXAEgad5O9NnYcsNPLCPZJD++2L9X0PCMODrA= +google.golang.org/genproto/googleapis/rpc v0.0.0-20230711160842-782d3b101e98 h1:bVf09lpb+OJbByTj913DRJioFFAjf/ZGxEz7MajTp2U= +google.golang.org/genproto/googleapis/rpc v0.0.0-20230711160842-782d3b101e98/go.mod h1:TUfxEVdsvPg18p6AslUXFoLdpED4oBnGwyqk3dV1XzM= google.golang.org/grpc v1.57.1 h1:upNTNqv0ES+2ZOOqACwVtS3Il8M12/+Hz41RCPzAjQg= google.golang.org/grpc v1.57.1/go.mod h1:Sd+9RMTACXwmub0zcNY2c4arhtrbBYD1AUHI/dt16Mo= +google.golang.org/grpc v1.58.3 h1:BjnpXut1btbtgN/6sp+brB2Kbm2LjNXnidYujAVbSoQ= +google.golang.org/grpc v1.58.3/go.mod h1:tgX3ZQDlNJGU96V6yHh1T/JeoBQ2TXdr43YbYSsCJk0= google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= google.golang.org/protobuf v1.27.1/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= diff --git a/ldap.go b/ldap.go new file mode 100644 index 0000000..d9db27b --- /dev/null +++ b/ldap.go @@ -0,0 +1,82 @@ +package main + +import ( + "github.com/go-ldap/ldap/v3" + "k8s.io/utils/env" + "log" +) + +type Ldap struct { + Connection *ldap.Conn + Config *LdapConfig +} + +func NewLdap(cfg *LdapConfig, logger appLoggerType) (*Ldap, error) { + conn, err := ldap.DialURL(cfg.Address) + if err != nil { + log.Fatalf("Failed to connect: %s\n", err) + } + + _, err = conn.SimpleBind(&ldap.SimpleBindRequest{ + Username: cfg.BindDN, + Password: env.GetString(cfg.BindPasswordEnvVar, "adminpassword"), + }) + return &Ldap{ + Connection: conn, + Config: cfg, + }, nil +} + +func (source *Ldap) GetUsers() ([]SourceUser, error) { + res, err := source.Connection.Search(&ldap.SearchRequest{ + BaseDN: source.Config.BaseDN, + Filter: source.Config.Users.Filter, + Attributes: []string{"*"}, + Scope: ldap.ScopeWholeSubtree, + }) + if err != nil { + return nil, err + } + + var users []SourceUser + for _, entry := range res.Entries { + username := entry.GetAttributeValue(source.Config.Users.UsernameAttributeType) + uid := entry.GetAttributeValue(source.Config.Users.UidAttributeType) + var firstName string + if source.Config.Users.FirstNameAttributeType != nil { + firstName = entry.GetAttributeValue(*source.Config.Users.FirstNameAttributeType) + } + users = append(users, LdapUser{ + BasicSourceUser: BasicSourceUser{SourceType: LdapSourceType}, + Username: username, + Uid: uid, + FirstName: firstName}) + } + return users, nil +} + +func (source *Ldap) GetGroupsWithMembers() ([]SourceGroupWithMembers, error) { + res, err := source.Connection.Search(&ldap.SearchRequest{ + BaseDN: source.Config.BaseDN, + Filter: source.Config.Groups.Filter, + Attributes: []string{"*"}, + Scope: ldap.ScopeWholeSubtree, + }) + if err != nil { + return nil, err + } + + var groups []SourceGroupWithMembers + for _, entry := range res.Entries { + groupname := entry.GetAttributeValue(source.Config.Groups.GroupnameAttributeType) + members := entry.GetAttributeValues(source.Config.Groups.MemberUidAttributeType) + groups = append(groups, SourceGroupWithMembers{ + SourceGroup: LdapGroup{ + BasicSourceGroup: BasicSourceGroup{SourceType: LdapSourceType}, + Groupname: groupname, + }, + Members: NewStringSetFromItems(members...), + }) + } + return groups, nil +} diff --git a/ldap_config.example.yaml b/ldap_config.example.yaml new file mode 100644 index 0000000..3b5ca10 --- /dev/null +++ b/ldap_config.example.yaml @@ -0,0 +1,40 @@ +app: + sync_interval: 5m + username_replacements: + - from: "@acme.com" + to: "" + - from: "@" + to: ":" + groupname_replacements: + - from: "|all" + to: "" + remove_limit: 10 + ban_before_remove_duration: 168h # 7d + +ldap: + address: "localhost:10210" + bind_dn: "cn=admin,dc=example,dc=org" + bind_password_env_var: "LDAP_PASSWORD" + base_dn: "dc=example,dc=org" + timeout: 1s + users: + filter: "(&(objectClass=posixAccount)(ou=People))" + username_attribute_type: "cn" + uid_attribute_type: "uid" + first_name_attribute_type: "givenName" + groups: + filter: "(objectClass=posixGroup)" + groupname_attribute_type: "cn" + member_uid_attribute_type: "memberUid" + +ytsaurus: + proxy: localhost:10110 + apply_user_changes: true + apply_group_changes: true + apply_member_changes: true + timeout: 1s + log_level: DEBUG + +logging: + level: WARN + is_production: true diff --git a/main.go b/main.go index 67f97af..0b36f24 100644 --- a/main.go +++ b/main.go @@ -2,7 +2,6 @@ package main import ( "fmt" - "github.com/jessevdk/go-flags" "github.com/pkg/errors" "go.uber.org/zap" @@ -36,7 +35,7 @@ func run(configFilePath string) error { return errors.Wrapf(err, "failed to load config %s", configFilePath) } - logger, err := configureLogger(cfg.Logging) + logger, err := configureLogger(&cfg.Logging) if err != nil { return errors.Wrapf(err, "failed to configure logging %+v", cfg.Logging) } diff --git a/migration_integration_test.go b/migration_integration_test.go index d54e763..fded182 100644 --- a/migration_integration_test.go +++ b/migration_integration_test.go @@ -61,7 +61,7 @@ func TestUsersMigration(t *testing.T) { t.Log("Got", len(ytUsersToUpdate), "users to update") for username, user := range ytUsersToUpdate { - attrValue := buildUserAzureAttributeValue(user) + attrValue := user.SourceUser if dryRun { t.Log("[DRY-RUN] will set @azure=", attrValue, "for", username) @@ -73,6 +73,7 @@ func TestUsersMigration(t *testing.T) { context.Background(), yt.client, username, + user.GetSourceAttributeName(), attrValue, ) require.NoError(t, err) @@ -125,7 +126,7 @@ func TestGroupsMigration(t *testing.T) { t.Log("Got", len(ytGroupsToUpdate), "groups to update") for groupname, group := range ytGroupsToUpdate { - attrValue := buildGroupAzureAttributeValue(group) + attrValue := group if dryRun { t.Log("[DRY-RUN] will set @azure=", attrValue, "for", groupname) continue @@ -136,6 +137,7 @@ func TestGroupsMigration(t *testing.T) { context.Background(), yt.client, groupname, + group.GetSourceAttributeName(), attrValue, ) require.NoError(t, err) diff --git a/source_models.go b/source_models.go new file mode 100644 index 0000000..44c6929 --- /dev/null +++ b/source_models.go @@ -0,0 +1,177 @@ +package main + +import ( + "fmt" + "github.com/pkg/errors" + "go.ytsaurus.tech/yt/go/yson" +) + +type ObjectID = string +type SourceType string + +const ( + LdapSourceType SourceType = "ldap" + AzureSourceType SourceType = "azure" +) + +type SourceUser interface { + GetId() ObjectID + GetName() string + GetSourceType() SourceType +} + +func NewSourceUser(attributes map[string]any) (SourceUser, error) { + bytes, err := yson.Marshal(attributes) + if err != nil { + return nil, err + } + + sourceType := attributes["source_type"] + if sourceType == string(LdapSourceType) { + var ldapUser LdapUser + err = yson.Unmarshal(bytes, &ldapUser) + if err != nil { + return nil, err + } + return ldapUser, nil + } else if sourceType == string(AzureSourceType) { + var azureUser AzureUser + err = yson.Unmarshal(bytes, &azureUser) + if err != nil { + return nil, err + } + return azureUser, nil + } else { + return nil, errors.New(fmt.Sprintf("Unknown source type: %v", sourceType)) + } +} + +type BasicSourceUser struct { + SourceType SourceType `yson:"source_type"` +} + +type AzureUser struct { + BasicSourceUser + // PrincipalName is unique human-readable Azure user field, used (possibly with changes) + // for the corresponding YTsaurus user's `name` attribute. + PrincipalName string `yson:"principal_name"` + + AzureID ObjectID `yson:"id"` + Email string `yson:"email"` + FirstName string `yson:"first_name"` + LastName string `yson:"last_name"` + DisplayName string `yson:"display_name"` +} + +func (user AzureUser) GetId() ObjectID { + return user.AzureID +} + +func (user AzureUser) GetName() string { + return user.PrincipalName +} + +func (user AzureUser) GetSourceType() SourceType { + return AzureSourceType +} + +type LdapUser struct { + BasicSourceUser + Username string `yson:"username"` + Uid string `yson:"uid"` + FirstName string `yson:"first_name"` + // TODO(nadya73): Add more fields. +} + +func (user LdapUser) GetId() ObjectID { + return user.Uid +} + +func (user LdapUser) GetName() string { + return user.Username +} + +func (user LdapUser) GetSourceType() SourceType { + return LdapSourceType +} + +type SourceGroup interface { + GetId() ObjectID + GetName() string + GetSourceType() SourceType +} + +type BasicSourceGroup struct { + SourceType SourceType `yson:"source_type"` +} + +func NewSourceGroup(attributes map[string]any) (SourceGroup, error) { + bytes, err := yson.Marshal(attributes) + if err != nil { + return nil, err + } + + sourceType := attributes["source_type"] + if sourceType == string(LdapSourceType) { + var ldapGroup LdapGroup + err = yson.Unmarshal(bytes, &ldapGroup) + if err != nil { + return nil, err + } + return ldapGroup, nil + } else if sourceType == string(AzureSourceType) { + var azureGroup AzureGroup + err = yson.Unmarshal(bytes, &azureGroup) + if err != nil { + return nil, err + } + return azureGroup, nil + } else { + return nil, errors.New(fmt.Sprintf("Unknown source type: %v", sourceType)) + } +} + +type AzureGroup struct { + BasicSourceGroup + // Identity is unique human-readable Source user field, used (possibly with changes) + // for the corresponding YTsaurus user's `name` attribute. + Identity string `yson:"identity"` + + AzureID ObjectID `yson:"id"` + DisplayName string `yson:"display_name"` +} + +func (ag AzureGroup) GetId() ObjectID { + return ag.AzureID +} + +func (ag AzureGroup) GetName() string { + return ag.Identity +} + +func (ag AzureGroup) GetSourceType() SourceType { + return AzureSourceType +} + +type LdapGroup struct { + BasicSourceGroup + Groupname string `yson:"groupname"` +} + +func (lg LdapGroup) GetId() ObjectID { + return lg.Groupname +} + +func (lg LdapGroup) GetName() string { + return lg.Groupname +} + +func (lg LdapGroup) GetSourceType() SourceType { + return LdapSourceType +} + +type SourceGroupWithMembers struct { + SourceGroup SourceGroup + // Members is a set of strings, representing users' ObjectID. + Members StringSet +} diff --git a/testcontainer_openldap.go b/testcontainer_openldap.go new file mode 100644 index 0000000..0af388a --- /dev/null +++ b/testcontainer_openldap.go @@ -0,0 +1,64 @@ +package main + +import ( + "context" + "errors" + "github.com/testcontainers/testcontainers-go" + "github.com/testcontainers/testcontainers-go/modules/openldap" + "go.ytsaurus.tech/library/go/ptr" +) + +type OpenLdapLocal struct { + container *openldap.OpenLDAPContainer +} + +func NewOpenLdapLocal() *OpenLdapLocal { + return &OpenLdapLocal{} +} + +func (y *OpenLdapLocal) Start() error { + ctx := context.Background() + container, err := openldap.RunContainer(ctx, testcontainers.WithImage("bitnami/openldap:2.6.6")) + if err != nil { + return err + } + y.container = container + return y.container.Start(ctx) +} + +func (y *OpenLdapLocal) GetConfig() (*LdapConfig, error) { + connectionString, err := y.container.ConnectionString(context.Background()) + if err != nil { + return nil, err + } + return &LdapConfig{ + Address: connectionString, //"ldap://localhost:1389", + BaseDN: "dc=example,dc=org", + BindDN: "cn=admin,dc=example,dc=org", + BindPasswordEnvVar: "LDAP_PASSWORD", + Users: LdapUsersConfig{ + Filter: "(&(objectClass=posixAccount)(ou=People))", + UsernameAttributeType: "cn", + UidAttributeType: "uid", + FirstNameAttributeType: ptr.String("givenName"), + }, + Groups: LdapGroupsConfig{ + Filter: "(objectClass=posixGroup)", + GroupnameAttributeType: "cn", + MemberUidAttributeType: "memberUid", + }, + }, nil +} + +func (y *OpenLdapLocal) Stop() error { + ctx := context.Background() + if y.container == nil { + return errors.New("container not started") + } + err := y.container.Terminate(ctx) + if err != nil { + return err + } + y.container = nil + return nil +} diff --git a/util.go b/util.go index 1d33be7..1331ff1 100644 --- a/util.go +++ b/util.go @@ -12,7 +12,10 @@ import ( ) const ( - appTimeFormat = "2006-01-02T15:04:05Z0700" + appTimeFormat = "2006-01-02T15:04:05Z0700" + defaultAzureTimeout = 3 * time.Second + defaultAzureSecretEnvVar = "AZURE_CLIENT_SECRET" + defaultAppRemoveLimit = 10 ) type appLoggerType = *zap.SugaredLogger diff --git a/ytsaurus.go b/ytsaurus.go index 20ae4d6..87ef20e 100644 --- a/ytsaurus.go +++ b/ytsaurus.go @@ -109,12 +109,13 @@ func (y *Ytsaurus) CreateUser(user YtsaurusUser) error { defer cancel() y.maybePrintExtraLogs(user.Username, "create_user", "user", user) + return doCreateYtsaurusUser( ctx, y.client, user.Username, map[string]any{ - "azure": buildUserAzureAttributeValue(user), + user.GetSourceAttributeName(): user.SourceUser, }, ) } @@ -233,7 +234,7 @@ func (y *Ytsaurus) CreateGroup(group YtsaurusGroup) error { y.client, group.Name, map[string]any{ - "azure": buildGroupAzureAttributeValue(group), + group.GetSourceAttributeName(): group.SourceGroup, }, ) } @@ -332,12 +333,17 @@ func (y *Ytsaurus) isUserManaged(username string) (bool, error) { ctx, cancel := context.WithTimeout(context.Background(), y.timeout) defer cancel() - attrExists, err := y.client.NodeExists( + attrAzureExists, err := y.client.NodeExists( ctx, ypath.Path("//sys/users/"+username+"/@azure"), nil, ) - return attrExists, err + attrSourceExists, err := y.client.NodeExists( + ctx, + ypath.Path("//sys/users/"+username+"/@source"), + nil, + ) + return attrAzureExists || attrSourceExists, err } func (y *Ytsaurus) ensureUserManaged(username string) error { @@ -355,12 +361,18 @@ func (y *Ytsaurus) isGroupManaged(name string) (bool, error) { ctx, cancel := context.WithTimeout(context.Background(), y.timeout) defer cancel() - attrExists, err := y.client.NodeExists( + attrAzureExists, err := y.client.NodeExists( ctx, ypath.Path("//sys/groups/"+name+"/@azure"), nil, ) - return attrExists, err + attrSourceExists, err := y.client.NodeExists( + ctx, + ypath.Path("//sys/groups/"+name+"/@source"), + nil, + ) + + return attrAzureExists || attrSourceExists, err } func (y *Ytsaurus) ensureGroupManaged(groupname string) error { diff --git a/ytsaurus_helpers.go b/ytsaurus_helpers.go index 6b23953..39d565f 100644 --- a/ytsaurus_helpers.go +++ b/ytsaurus_helpers.go @@ -14,10 +14,11 @@ import ( func doGetAllYtsaurusUsers(ctx context.Context, client yt.Client) ([]YtsaurusUser, error) { type YtsaurusUserResponse struct { - Name string `yson:",value"` - Azure map[string]string `yson:"azure,attr"` - Banned bool `yson:"banned,attr"` - BannedSince string `yson:"banned_since,attr"` + Name string `yson:",value"` + Azure *AzureUser `yson:"azure,attr"` + Source map[string]any `yson:"source,attr"` + Banned bool `yson:"banned,attr"` + BannedSince string `yson:"banned_since,attr"` } var response []YtsaurusUserResponse @@ -30,6 +31,7 @@ func doGetAllYtsaurusUsers(ctx context.Context, client yt.Client) ([]YtsaurusUse "azure", "banned", "banned_since", + "source", }, }, ) @@ -46,15 +48,20 @@ func doGetAllYtsaurusUsers(ctx context.Context, client yt.Client) ([]YtsaurusUse return nil, errors.Wrapf(err, "failed to parse @banned_since. %v", ytUser) } } + var sourceUser SourceUser + if ytUser.Azure != nil { + sourceUser = *ytUser.Azure + } else if ytUser.Source != nil { + sourceUser, err = NewSourceUser(ytUser.Source) + if err != nil { + return nil, errors.Wrapf(err, "failed to create source user. %v", ytUser) + } + } + users = append(users, YtsaurusUser{ - Username: ytUser.Name, - AzureID: ytUser.Azure["id"], - PrincipalName: ytUser.Azure["principal_name"], - Email: ytUser.Azure["email"], - FirstName: ytUser.Azure["first_name"], - LastName: ytUser.Azure["last_name"], - DisplayName: ytUser.Azure["display_name"], - BannedSince: bannedSince, + Username: ytUser.Name, + SourceUser: sourceUser, + BannedSince: bannedSince, }) } return users, nil @@ -62,9 +69,10 @@ func doGetAllYtsaurusUsers(ctx context.Context, client yt.Client) ([]YtsaurusUse func doGetAllYtsaurusGroupsWithMembers(ctx context.Context, client yt.Client) ([]YtsaurusGroupWithMembers, error) { type YtsaurusGroupReponse struct { - Name string `yson:",value"` - Azure map[string]string `yson:"azure,attr"` - Members []string `yson:"members,attr"` + Name string `yson:",value"` + Azure *AzureGroup `yson:"azure,attr"` + Source map[string]any `yson:"source,attr"` + Members []string `yson:"members,attr"` } var response []YtsaurusGroupReponse @@ -73,29 +81,39 @@ func doGetAllYtsaurusGroupsWithMembers(ctx context.Context, client yt.Client) ([ ypath.Path("//sys/groups"), &response, &yt.ListNodeOptions{ - Attributes: []string{"members", "azure"}, + Attributes: []string{"members", "azure", "source"}, }, ) if err != nil { return nil, err } - var users []YtsaurusGroupWithMembers + var groups []YtsaurusGroupWithMembers for _, ytGroup := range response { members := NewStringSet() for _, m := range ytGroup.Members { members.Add(m) } - users = append(users, YtsaurusGroupWithMembers{ + + var sourceGroup SourceGroup + if ytGroup.Azure != nil { + sourceGroup = *ytGroup.Azure + } else if ytGroup.Source != nil { + sourceGroup, err = NewSourceGroup(ytGroup.Source) + if err != nil { + return nil, errors.Wrapf(err, "failed to create source group. %v", ytGroup) + } + } + + groups = append(groups, YtsaurusGroupWithMembers{ YtsaurusGroup: YtsaurusGroup{ Name: ytGroup.Name, - AzureID: ytGroup.Azure["id"], - DisplayName: ytGroup.Azure["display_name"], + SourceGroup: sourceGroup, }, Members: members, }) } - return users, nil + return groups, nil } func doCreateYtsaurusUser(ctx context.Context, client yt.Client, username string, attrs map[string]any) error { @@ -142,48 +160,27 @@ func doRemoveMemberYtsaurusGroup(ctx context.Context, client yt.Client, username ) } -func buildUserAzureAttributeValue(user YtsaurusUser) map[string]string { - return map[string]string{ - "id": user.AzureID, - "email": user.Email, - "principal_name": user.PrincipalName, - "first_name": user.FirstName, - "last_name": user.LastName, - "display_name": user.DisplayName, - } -} - func buildUserAttributes(user YtsaurusUser) map[string]any { return map[string]any{ - "azure": buildUserAzureAttributeValue(user), - "name": user.Username, - "banned_since": user.BannedSinceString(), - "banned": user.IsBanned(), - } -} - -func buildGroupAzureAttributeValue(group YtsaurusGroup) map[string]string { - return map[string]string{ - "id": group.AzureID, - "display_name": group.DisplayName, + "name": user.Username, + "banned_since": user.BannedSinceString(), + "banned": user.IsBanned(), + user.GetSourceAttributeName(): user.SourceUser, } } func buildGroupAttributes(group YtsaurusGroup) map[string]any { return map[string]any{ - "azure": map[string]string{ - "id": group.AzureID, - "display_name": group.DisplayName, - }, - "name": group.Name, + group.GetSourceAttributeName(): group.SourceGroup, + "name": group.Name, } } // nolint: unused -func doSetAzureAttributeForYtsaurusUser(ctx context.Context, client yt.Client, username string, attrValue map[string]string) error { +func doSetAzureAttributeForYtsaurusUser(ctx context.Context, client yt.Client, username string, attrName string, attrValue any) error { return client.SetNode( ctx, - ypath.Path("//sys/users/"+username+"/@azure"), + ypath.Path("//sys/users/"+username+"/@"+attrName), attrValue, nil, ) @@ -210,10 +207,16 @@ func doSetAttributesForYtsaurusUser(ctx context.Context, client yt.Client, usern } // nolint: unused -func doSetAzureAttributeForYtsaurusGroup(ctx context.Context, client yt.Client, groupname string, attrValue map[string]string) error { +func doSetAzureAttributeForYtsaurusGroup( + ctx context.Context, + client yt.Client, + groupname string, + attrName string, + attrValue map[string]string, +) error { return client.SetNode( ctx, - ypath.Path("//sys/groups/"+groupname+"/@azure"), + ypath.Path("//sys/groups/"+groupname+"/@"+attrName), attrValue, nil, ) diff --git a/ytsaurus_models.go b/ytsaurus_models.go index 86585e2..ace5478 100644 --- a/ytsaurus_models.go +++ b/ytsaurus_models.go @@ -6,23 +6,24 @@ import ( type YtsaurusUser struct { // Username is a unique @name attribute of a user. - Username string - // AzureID is non-human readable string like 2cd8a70c-9044-4488-b06a-c8461c39b296. - AzureID string - // PrincipalName is a unique human-readable login. - // It could be in form of email, but doesn't give guarantee that such email exists. - PrincipalName string - // Email is filled if Azure user has email. - Email string - FirstName string - LastName string - DisplayName string + Username string + SourceUser SourceUser BannedSince time.Time } +func (u YtsaurusUser) GetSourceAttributeName() string { + switch u.SourceUser.GetSourceType() { + case AzureSourceType: + return "azure" + case LdapSourceType: + return "source" + } + return "source" +} + // IsManuallyManaged true if user doesn't have @azure attribute (system or manually created user). func (u YtsaurusUser) IsManuallyManaged() bool { - return u.AzureID == "" + return u.SourceUser == nil } func (u YtsaurusUser) IsBanned() bool { @@ -39,8 +40,17 @@ func (u YtsaurusUser) BannedSinceString() string { type YtsaurusGroup struct { // Name is a unique @name attribute of a group. Name string - AzureID string - DisplayName string + SourceGroup SourceGroup +} + +func (g YtsaurusGroup) GetSourceAttributeName() string { + switch g.SourceGroup.GetSourceType() { + case AzureSourceType: + return "azure" + case LdapSourceType: + return "source" + } + return "source" } type YtsaurusGroupWithMembers struct { @@ -60,5 +70,5 @@ type YtsaurusMembership struct { // IsManuallyManaged true if group doesn't have @azure attribute (system or manually created group). func (u YtsaurusGroup) IsManuallyManaged() bool { - return u.AzureID == "" + return u.SourceGroup == nil } diff --git a/ytsaurus_test.go b/ytsaurus_test.go index 00af797..76090c6 100644 --- a/ytsaurus_test.go +++ b/ytsaurus_test.go @@ -41,15 +41,24 @@ func TestUpdateUserFirstName(t *testing.T) { defer func() { require.NoError(t, ytLocal.Stop()) }() yt := getYtsaurus(t, ytLocal) + const azureId = "fake-az-id-old" + managedOleg := YtsaurusUser{ - Username: "oleg", - AzureID: "fake-az-id-oleg", - FirstName: "Lego", + Username: "oleg", + SourceUser: AzureUser{ + AzureID: azureId, + FirstName: "Lego", + }, } err := yt.CreateUser(managedOleg) require.NoError(t, err) - managedOleg.FirstName = "Oleg" + updateSourceUser := AzureUser{ + AzureID: azureId, + FirstName: "Oleg", + } + managedOleg.SourceUser = updateSourceUser + updErr := yt.UpdateUser(managedOleg.Username, managedOleg) ytClient, err := ytLocal.GetClient() @@ -79,15 +88,19 @@ func TestGroups(t *testing.T) { managedOleg := YtsaurusUser{ Username: "oleg", - AzureID: "fake-az-id-oleg", + SourceUser: AzureUser{ + AzureID: "fake-az-id-oleg", + }, } err = yt.CreateUser(managedOleg) require.NoError(t, err) managedOlegsGroup := YtsaurusGroup{ - Name: "olegs", - AzureID: "fake-az-id-olegs", - DisplayName: "This is group is for Olegs only", + Name: "olegs", + SourceGroup: AzureGroup{ + AzureID: "fake-az-id-olegs", + DisplayName: "This is group is for Olegs only", + }, } err = yt.CreateGroup(managedOlegsGroup) require.NoError(t, err) @@ -102,9 +115,11 @@ func TestGroups(t *testing.T) { require.Equal(t, []YtsaurusGroupWithMembers{ { YtsaurusGroup: YtsaurusGroup{ - Name: managedOlegsGroup.Name, - AzureID: managedOlegsGroup.AzureID, - DisplayName: managedOlegsGroup.DisplayName, + Name: managedOlegsGroup.Name, + SourceGroup: AzureGroup{ + AzureID: managedOlegsGroup.SourceGroup.(AzureGroup).AzureID, + DisplayName: managedOlegsGroup.SourceGroup.(AzureGroup).DisplayName, + }, }, Members: members, }, @@ -118,9 +133,11 @@ func TestGroups(t *testing.T) { require.Equal(t, []YtsaurusGroupWithMembers{ { YtsaurusGroup: YtsaurusGroup{ - Name: managedOlegsGroup.Name, - AzureID: managedOlegsGroup.AzureID, - DisplayName: managedOlegsGroup.DisplayName, + Name: managedOlegsGroup.Name, + SourceGroup: AzureGroup{ + AzureID: managedOlegsGroup.SourceGroup.(AzureGroup).AzureID, + DisplayName: managedOlegsGroup.SourceGroup.(AzureGroup).DisplayName, + }, }, Members: NewStringSet(), }, From 840198652f1590ce11bd4ff3a945c7eddaf4bc75 Mon Sep 17 00:00:00 2001 From: Nadezhda Savchenko Date: Thu, 22 Feb 2024 13:18:13 +0300 Subject: [PATCH 02/21] Update go.mod and go.sum --- go.mod | 8 +++---- go.sum | 76 +++++++--------------------------------------------------- 2 files changed, 11 insertions(+), 73 deletions(-) diff --git a/go.mod b/go.mod index c29f748..7cece01 100644 --- a/go.mod +++ b/go.mod @@ -5,6 +5,7 @@ go 1.20 require ( github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.4.0 github.com/deckarep/golang-set/v2 v2.3.1 + github.com/go-ldap/ldap/v3 v3.4.6 github.com/google/go-cmp v0.6.0 github.com/jessevdk/go-flags v1.5.0 github.com/microsoft/kiota-abstractions-go v1.3.0 @@ -13,7 +14,9 @@ require ( github.com/pkg/errors v0.9.1 github.com/stretchr/testify v1.8.4 github.com/testcontainers/testcontainers-go v0.28.0 + github.com/testcontainers/testcontainers-go/modules/openldap v0.28.0 go.uber.org/zap v1.26.0 + go.ytsaurus.tech/library/go/ptr v0.0.1 go.ytsaurus.tech/yt/go v0.0.13 gopkg.in/yaml.v3 v3.0.1 k8s.io/utils v0.0.0-20230220204549-a5ecb0141aa5 @@ -36,13 +39,11 @@ require ( github.com/cpuguy83/dockercfg v0.3.1 // indirect github.com/davecgh/go-spew v1.1.1 // indirect github.com/distribution/reference v0.5.0 // indirect - github.com/docker/distribution v2.8.2+incompatible // indirect github.com/docker/docker v25.0.2+incompatible // indirect github.com/docker/go-connections v0.5.0 // indirect github.com/docker/go-units v0.5.0 // indirect github.com/felixge/httpsnoop v1.0.3 // indirect github.com/go-asn1-ber/asn1-ber v1.5.5 // indirect - github.com/go-ldap/ldap/v3 v3.4.6 // indirect github.com/go-logr/logr v1.2.4 // indirect github.com/go-logr/stdr v1.2.2 // indirect github.com/go-ole/go-ole v1.2.6 // indirect @@ -72,7 +73,6 @@ require ( github.com/morikuni/aec v1.0.0 // indirect github.com/opencontainers/go-digest v1.0.0 // indirect github.com/opencontainers/image-spec v1.1.0-rc5 // indirect - github.com/opencontainers/runc v1.1.5 // indirect github.com/opentracing/opentracing-go v1.2.0 // indirect github.com/pierrec/lz4 v2.6.1+incompatible // indirect github.com/pkg/browser v0.0.0-20210911075715-681adbf594b8 // indirect @@ -82,7 +82,6 @@ require ( github.com/shoenig/go-m1cpu v0.1.6 // indirect github.com/sirupsen/logrus v1.9.3 // indirect github.com/std-uritemplate/std-uritemplate/go v0.0.42 // indirect - github.com/testcontainers/testcontainers-go/modules/openldap v0.28.0 // indirect github.com/tklauser/go-sysconf v0.3.12 // indirect github.com/tklauser/numcpus v0.6.1 // indirect github.com/yusufpapurcu/wmi v1.2.3 // indirect @@ -95,7 +94,6 @@ require ( go.ytsaurus.tech/library/go/blockcodecs v0.0.2 // indirect go.ytsaurus.tech/library/go/core/log v0.0.3 // indirect go.ytsaurus.tech/library/go/core/xerrors v0.0.3 // indirect - go.ytsaurus.tech/library/go/ptr v0.0.1 // indirect go.ytsaurus.tech/library/go/x/xreflect v0.0.2 // indirect go.ytsaurus.tech/library/go/x/xruntime v0.0.3 // indirect golang.org/x/crypto v0.17.0 // indirect diff --git a/go.sum b/go.sum index 8df8c6e..6073294 100644 --- a/go.sum +++ b/go.sum @@ -13,35 +13,25 @@ github.com/Azure/go-ntlmssp v0.0.0-20221128193559-754e69321358 h1:mFRzDkZVAjdal+ github.com/Azure/go-ntlmssp v0.0.0-20221128193559-754e69321358/go.mod h1:chxPXzSsl7ZWRAuOIE23GDNzjWuZquvFlgA8xmpunjU= github.com/AzureAD/microsoft-authentication-library-for-go v1.2.0 h1:hVeq+yCyUi+MsoO/CU95yqCIcdzra5ovzk8Q2BBpV2M= github.com/AzureAD/microsoft-authentication-library-for-go v1.2.0/go.mod h1:wP83P5OoQ5p6ip3ScPr0BAq0BvuPAvacpEuSzyouqAI= -github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= github.com/Microsoft/go-winio v0.6.1 h1:9/kr64B9VUZrLm5YYwbGtUJnMgqWVOdUAXu6Migciow= github.com/Microsoft/go-winio v0.6.1/go.mod h1:LRdKpFKfdobln8UmuiYcKPot9D2v6svN5+sAH+4kjUM= -github.com/Microsoft/hcsshim v0.11.1 h1:hJ3s7GbWlGK4YVV92sO88BQSyF4ZLVy7/awqOlPxFbA= -github.com/Microsoft/hcsshim v0.11.1/go.mod h1:nFJmaO4Zr5Y7eADdFOpYswDDlNVbvcIJJNJLECr5JQg= github.com/Microsoft/hcsshim v0.11.4 h1:68vKo2VN8DE9AdN4tnkWnmdhqdbpUFM8OF3Airm7fz8= github.com/Microsoft/hcsshim v0.11.4/go.mod h1:smjE4dvqPX9Zldna+t5FG3rnoHhaB7QYxPRqGcpAD9w= +github.com/alexbrainman/sspi v0.0.0-20210105120005-909beea2cc74 h1:Kk6a4nehpJ3UuJRqlA3JxYxBZEqCeOmATOvrbT4p9RA= github.com/alexbrainman/sspi v0.0.0-20210105120005-909beea2cc74/go.mod h1:cEWa1LVoE5KvSD9ONXsZrj0z6KqySlCCNKHlLzbqAt4= github.com/andybalholm/brotli v1.0.4 h1:V7DdXeJtZscaqfNuAdSRuRFzuiKlHSC/Zh3zl9qY3JY= github.com/andybalholm/brotli v1.0.4/go.mod h1:fO7iG3H7G2nSZ7m0zPUDn85XEX2GTukHGRSepvi9Eig= github.com/cenkalti/backoff/v4 v4.2.1 h1:y4OZtCnogmCPw98Zjyt5a6+QwPLGkiQsYW5oUqylYbM= github.com/cenkalti/backoff/v4 v4.2.1/go.mod h1:Y3VNntkOUPxTVeUxJ/G5vcM//AlwfmyYozVcomhLiZE= -github.com/checkpoint-restore/go-criu/v5 v5.3.0/go.mod h1:E/eQpaFtUKGOOSEBZgmKAcn+zUUwWxqcaKZlF54wK8E= -github.com/cilium/ebpf v0.7.0/go.mod h1:/oI2+1shJiTGAMgl6/RgJr36Eo1jzrRcAWbcXO2usCA= github.com/cjlapao/common-go v0.0.39 h1:bAAUrj2B9v0kMzbAOhzjSmiyDy+rd56r2sy7oEiQLlA= github.com/cjlapao/common-go v0.0.39/go.mod h1:M3dzazLjTjEtZJbbxoA5ZDiGCiHmpwqW9l4UWaddwOA= -github.com/containerd/console v1.0.3/go.mod h1:7LqA/THxQ86k76b8c/EMSiaJ3h1eZkMkXar0TQ1gf3U= -github.com/containerd/containerd v1.7.7 h1:QOC2K4A42RQpcrZyptP6z9EJZnlHfHJUfZrAAHe15q4= -github.com/containerd/containerd v1.7.7/go.mod h1:3c4XZv6VeT9qgf9GMTxNTMFxGJrGpI2vz1yk4ye+YY8= github.com/containerd/containerd v1.7.12 h1:+KQsnv4VnzyxWcfO9mlxxELaoztsDEjOuCMPAuPqgU0= github.com/containerd/containerd v1.7.12/go.mod h1:/5OMpE1p0ylxtEUGY8kuCYkDRzJm9NO1TFMWjUpdevk= github.com/containerd/log v0.1.0 h1:TCJt7ioM2cr/tfR8GPbGf9/VRAX8D2B4PjzCpfX540I= github.com/containerd/log v0.1.0/go.mod h1:VRRf09a7mHDIRezVKTRCrOq78v577GXq3bSa3EhrzVo= -github.com/coreos/go-systemd/v22 v22.3.2/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc= github.com/cpuguy83/dockercfg v0.3.1 h1:/FpZ+JaygUR/lZP2NlFI2DVfrOEMAIKP5wWEJdoYe9E= github.com/cpuguy83/dockercfg v0.3.1/go.mod h1:sugsbF4//dDlL/i+S+rtpIWp+5h0BHJHfjj5/jFyUJc= -github.com/cpuguy83/go-md2man/v2 v2.0.0-20190314233015-f79a8a8ca69d/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU= github.com/creack/pty v1.1.18 h1:n56/Zwd5o6whRC5PMGretI4IdRLlmBXYNjScPaBgsbY= -github.com/cyphar/filepath-securejoin v0.2.3/go.mod h1:aPGpWjXOXUn2NCNjFvBE6aRxGGx79pTxQpKOJNYHHl4= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= @@ -50,22 +40,14 @@ github.com/deckarep/golang-set/v2 v2.3.1/go.mod h1:VAky9rY/yGXJOLEDv3OMci+7wtDpO github.com/distribution/reference v0.5.0 h1:/FUIFXtfc/x2gpa5/VGfiGLuOIdYa1t65IKK2OFGvA0= github.com/distribution/reference v0.5.0/go.mod h1:BbU0aIcezP1/5jX/8MP0YiH4SdvB5Y4f/wlDRiLyi3E= github.com/dnaeon/go-vcr v1.2.0 h1:zHCHvJYTMh1N7xnV7zf1m1GPBF9Ad0Jk/whtQ1663qI= -github.com/docker/distribution v2.8.2+incompatible h1:T3de5rq0dB1j30rp0sA2rER+m322EBzniBPB6ZIzuh8= -github.com/docker/distribution v2.8.2+incompatible/go.mod h1:J2gT2udsDAN96Uj4KfcMRqY0/ypR+oyYUYmja8H+y+w= -github.com/docker/docker v24.0.6+incompatible h1:hceabKCtUgDqPu+qm0NgsaXf28Ljf4/pWFL7xjWWDgE= -github.com/docker/docker v24.0.6+incompatible/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk= github.com/docker/docker v25.0.2+incompatible h1:/OaKeauroa10K4Nqavw4zlhcDq/WBcPMc5DbjOGgozY= github.com/docker/docker v25.0.2+incompatible/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk= -github.com/docker/go-connections v0.4.0 h1:El9xVISelRB7BuFusrZozjnkIM5YnzCViNKohAFqRJQ= -github.com/docker/go-connections v0.4.0/go.mod h1:Gbd7IOopHjR8Iph03tsViu4nIes5XhDvyHbTtUxmeec= github.com/docker/go-connections v0.5.0 h1:USnMq7hx7gwdVZq1L49hLXaFtUdTADjXGp+uj1Br63c= github.com/docker/go-connections v0.5.0/go.mod h1:ov60Kzw0kKElRwhNs9UlUHAE/F9Fe6GLaXnqyDdmEXc= -github.com/docker/go-units v0.4.0/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk= github.com/docker/go-units v0.5.0 h1:69rxXcBk27SvSaaxTtLh/8llcHD8vYHT7WSdRZ/jvr4= github.com/docker/go-units v0.5.0/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk= github.com/felixge/httpsnoop v1.0.3 h1:s/nj+GCswXYzN5v2DpNMuMQYe+0DDwt5WVCU6CWBdXk= github.com/felixge/httpsnoop v1.0.3/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U= -github.com/frankban/quicktest v1.11.3/go.mod h1:wRf/ReqHper53s+kmmSZizM8NamnL3IM0I9ntUbOk+k= github.com/frankban/quicktest v1.14.5 h1:dfYrrRyLtiqT9GyKXgdh+k4inNeTvmGbuSgZ3lx3GhA= github.com/go-asn1-ber/asn1-ber v1.5.5 h1:MNHlNMBDgEKD4TcKr36vQN68BA00aDfjIt3/bD50WnA= github.com/go-asn1-ber/asn1-ber v1.5.5/go.mod h1:hEBeB/ic+5LoWskz+yKT7vGhhPYkProFKoKdwZRWMe0= @@ -78,8 +60,6 @@ github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag= github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE= github.com/go-ole/go-ole v1.2.6 h1:/Fpf6oFPoeFik9ty7siob0G6Ke8QvQEuVcuChpwXzpY= github.com/go-ole/go-ole v1.2.6/go.mod h1:pprOEPIfldk/42T2oK7lQ4v4JSDwmV0As9GaiUsvbm0= -github.com/godbus/dbus/v5 v5.0.4/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA= -github.com/godbus/dbus/v5 v5.0.6/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA= github.com/gofrs/uuid v4.2.0+incompatible h1:yyYWMnhkhrKwwr8gAOcOCYxOOscHgDS9yZgBrnJfGa0= github.com/gofrs/uuid v4.2.0+incompatible/go.mod h1:b2aQJv3Z4Fp6yNu3cdSllBxTCLRxnplIgP/c0N/04lM= github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= @@ -91,30 +71,24 @@ github.com/golang/protobuf v1.5.3 h1:KhyjKVUg7Usr/dYsdSqoFveMYd5ko72D+zANwlG1mmg github.com/golang/protobuf v1.5.3/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= github.com/golang/snappy v0.0.4 h1:yAGX7huGHXlcLOEtBnF4w7FQwA26wojNCwOYAEhLjQM= github.com/golang/snappy v0.0.4/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= -github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.6/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38= github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/google/tink/go v1.7.0 h1:6Eox8zONGebBFcCBqkVmt60LaWZa6xg1cl/DwAh/J1w= github.com/google/tink/go v1.7.0/go.mod h1:GAUOd+QE3pgj9q8VKIGTCP33c/B7eb4NhxLcgTJZStM= github.com/google/uuid v1.3.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= -github.com/google/uuid v1.4.0 h1:MtMxsa51/r9yyhkyLsVeVt0B+BGQZzpQiTQ4eHZ8bc4= -github.com/google/uuid v1.4.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/grpc-ecosystem/grpc-gateway/v2 v2.16.0 h1:YBftPWNWd4WwGqtY2yeZL2ef8rHAxPBD8KFhJpmcqms= github.com/jessevdk/go-flags v1.5.0 h1:1jKYvbxEjfUl0fmqTCOfonvskHHXMjBySTLW4y9LFvc= github.com/jessevdk/go-flags v1.5.0/go.mod h1:Fw0T6WPc1dYxT4mKEZRfG5kJhaTDP9pj1c2EWnYs/m4= github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= github.com/klauspost/compress v1.16.7 h1:2mk3MPGNzKyxErAw8YaohYh69+pa4sIQSC0fPGCFR9I= github.com/klauspost/compress v1.16.7/go.mod h1:ntbaceVETuRiXiv4DpjP66DpAtAGkEQskQzEyD//IeE= -github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= -github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= -github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0SNc= github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw= @@ -146,7 +120,6 @@ github.com/mitchellh/reflectwalk v1.0.2 h1:G2LzWKi524PWgd3mLHV8Y5k7s6XUvT0Gef6zx github.com/mitchellh/reflectwalk v1.0.2/go.mod h1:mSTlrgnPZtwu0c4WaC2kGObEpuNDbx0jmZXqmk4esnw= github.com/moby/patternmatcher v0.6.0 h1:GmP9lR19aU5GqSSFko+5pRqHi+Ohk1O69aFiKkVGiPk= github.com/moby/patternmatcher v0.6.0/go.mod h1:hDPoyOpDY7OrrMDLaYoY3hf52gNCR/YOUYxkhApJIxc= -github.com/moby/sys/mountinfo v0.5.0/go.mod h1:3bMD3Rg+zkqx8MRYPi7Pyb0Ie97QEBmdxbhnCLlSvSU= github.com/moby/sys/sequential v0.5.0 h1:OPvI35Lzn9K04PBbCLW0g4LcFAJgHsvXsRyewg5lXtc= github.com/moby/sys/sequential v0.5.0/go.mod h1:tH2cOOs5V9MlPiXcQzRC+eEyab644PWKGRYaaV5ZZlo= github.com/moby/sys/user v0.1.0 h1:WmZ93f5Ux6het5iituh9x2zAG7NFY9Aqi49jjE1PaQg= @@ -155,15 +128,10 @@ github.com/moby/term v0.5.0 h1:xt8Q1nalod/v7BqbG21f8mQPqH+xAaC9C3N3wfWbVP0= github.com/moby/term v0.5.0/go.mod h1:8FzsFHVUBGZdbDsJw/ot+X+d5HLUbvklYLJ9uGfcI3Y= github.com/morikuni/aec v1.0.0 h1:nP9CBfwrvYnBRgY6qfDQkygYDmYwOilePFkwzv4dU8A= github.com/morikuni/aec v1.0.0/go.mod h1:BbKIizmSmc5MMPqRYbxO4ZU0S0+P200+tUnFx7PXmsc= -github.com/mrunalp/fileutils v0.5.0/go.mod h1:M1WthSahJixYnrXQl/DFQuteStB1weuxD2QJNHXfbSQ= github.com/opencontainers/go-digest v1.0.0 h1:apOUWs51W5PlhuyGyz9FCeeBIOUDA/6nW8Oi/yOhh5U= github.com/opencontainers/go-digest v1.0.0/go.mod h1:0JzlMkj0TRzQZfJkVvzbP0HBR3IKzErnv2BNG4W4MAM= github.com/opencontainers/image-spec v1.1.0-rc5 h1:Ygwkfw9bpDvs+c9E34SdgGOj41dX/cbdlwvlWt0pnFI= github.com/opencontainers/image-spec v1.1.0-rc5/go.mod h1:X4pATf0uXsnn3g5aiGIsVnJBR4mxhKzfwmvK/B2NTm8= -github.com/opencontainers/runc v1.1.5 h1:L44KXEpKmfWDcS02aeGm8QNTFXTo2D+8MYGDIJ/GDEs= -github.com/opencontainers/runc v1.1.5/go.mod h1:1J5XiS+vdZ3wCyZybsuxXZWGrgSr8fFJHLXuG2PsnNg= -github.com/opencontainers/runtime-spec v1.0.3-0.20210326190908-1c3f411f0417/go.mod h1:jwyrGlmzljRJv/Fgzds9SsS/C5hL+LL3ko9hs6T5lQ0= -github.com/opencontainers/selinux v1.10.0/go.mod h1:2i0OySw99QjzBBQByd1Gr9gSjvuho1lHsJxIJ3gGbJI= github.com/opentracing/opentracing-go v1.2.0 h1:uEJPy/1a5RIPAJ0Ov+OIO8OxWu77jEv+1B0VhjKrZUs= github.com/opentracing/opentracing-go v1.2.0/go.mod h1:GxEUsuufX4nBwe+T+Wl9TAgYrxe9dPLANfrWvHYVTgc= github.com/pierrec/lz4 v2.6.1+incompatible h1:9UY3+iC23yxF0UfGaYrGplQ+79Rg+h/q9FV9ix19jjM= @@ -177,18 +145,12 @@ github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZN github.com/power-devops/perfstat v0.0.0-20210106213030-5aafc221ea8c h1:ncq/mPwQF4JjgDlrVEn3C11VoGHZN7m8qihwgMEtzYw= github.com/power-devops/perfstat v0.0.0-20210106213030-5aafc221ea8c/go.mod h1:OmDBASR4679mdNQnz2pUhc2G8CO2JrUAVFDRBDP/hJE= github.com/rogpeppe/go-internal v1.10.0 h1:TMyTOH3F/DB16zRVcYyreMH6GnZZrwQVAoYjRBZyWFQ= -github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= -github.com/seccomp/libseccomp-golang v0.9.2-0.20220502022130-f33da4d89646/go.mod h1:JA8cRccbGaA1s33RQf7Y1+q9gHmZX1yB/z9WDN1C6fg= -github.com/shirou/gopsutil/v3 v3.23.9 h1:ZI5bWVeu2ep4/DIxB4U9okeYJ7zp/QLTO4auRb/ty/E= -github.com/shirou/gopsutil/v3 v3.23.9/go.mod h1:x/NWSb71eMcjFIO0vhyGW5nZ7oSIgVjrCnADckb85GA= github.com/shirou/gopsutil/v3 v3.23.12 h1:z90NtUkp3bMtmICZKpC4+WaknU1eXtp5vtbQ11DgpE4= github.com/shirou/gopsutil/v3 v3.23.12/go.mod h1:1FrWgea594Jp7qmjHUUPlJDTPgcsb9mGnXDxavtikzM= github.com/shoenig/go-m1cpu v0.1.6 h1:nxdKQNcEB6vzgA2E2bvzKIYRuNj7XNJ4S/aRSwKzFtM= github.com/shoenig/go-m1cpu v0.1.6/go.mod h1:1JJMcUBvfNwpq05QDQVAnx3gUHr9IYF7GNg9SUEw2VQ= github.com/shoenig/test v0.6.4 h1:kVTaSd7WLz5WZ2IaoM0RSzRsUD+m8wRR+5qvntpn4LU= github.com/shoenig/test v0.6.4/go.mod h1:byHiCGXqrVaflBLAMq/srcZIHynQPQgeyvkvXnjqq0k= -github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc= -github.com/sirupsen/logrus v1.8.1/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0= github.com/sirupsen/logrus v1.9.3 h1:dueUQJ1C2q9oE3F7wvmSGAaVtTmUizReu6fjN8uqzbQ= github.com/sirupsen/logrus v1.9.3/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ= github.com/std-uritemplate/std-uritemplate/go v0.0.42 h1:rG+XlE4drkVWs2NLfGS15N+vg+CUcjXElQKvJ0fctlI= @@ -196,16 +158,12 @@ github.com/std-uritemplate/std-uritemplate/go v0.0.42/go.mod h1:Qov4Ay4U83j37Xjg github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= -github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk= github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= -github.com/syndtr/gocapability v0.0.0-20200815063812-42c35b437635/go.mod h1:hkRG7XYTFWNJGYcbNJQlaLq0fg1yr4J4t/NcTQtrfww= -github.com/testcontainers/testcontainers-go v0.26.0 h1:uqcYdoOHBy1ca7gKODfBd9uTHVK3a7UL848z09MVZ0c= -github.com/testcontainers/testcontainers-go v0.26.0/go.mod h1:ICriE9bLX5CLxL9OFQ2N+2N+f+803LNJ1utJb1+Inx0= github.com/testcontainers/testcontainers-go v0.28.0 h1:1HLm9qm+J5VikzFDYhOd+Zw12NtOl+8drH2E8nTY1r8= github.com/testcontainers/testcontainers-go v0.28.0/go.mod h1:COlDpUXbwW3owtpMkEB1zo9gwb1CoKVKlyrVPejF4AU= github.com/testcontainers/testcontainers-go/modules/openldap v0.28.0 h1:hR3BtDFSPUXvMZNuVbw7YcqAFsvjwt/VNA6ZTjtDZiM= @@ -214,9 +172,6 @@ github.com/tklauser/go-sysconf v0.3.12 h1:0QaGUFOdQaIVdPgfITYzaTegZvdCjmYO52cSFA github.com/tklauser/go-sysconf v0.3.12/go.mod h1:Ho14jnntGE1fpdOqQEEaiKRpvIavV0hSfmBq8nJbHYI= github.com/tklauser/numcpus v0.6.1 h1:ng9scYS7az0Bk4OZLvrNXNSAO2Pxr1XXRAPyjhIx+Fk= github.com/tklauser/numcpus v0.6.1/go.mod h1:1XfjsgE2zo8GVw7POkMbHENHzVg3GzmoZ9fESEdAacY= -github.com/urfave/cli v1.22.1/go.mod h1:Gos4lmkARVdJ6EkW0WaNv/tZAAMe9V7XWyB60NtXRu0= -github.com/vishvananda/netlink v1.1.0/go.mod h1:cTgwzPIzzgDAYoQrMm0EdrjRUBkTqKYppBueQtXaqoE= -github.com/vishvananda/netns v0.0.0-20191106174202-0a2b9b5464df/go.mod h1:JP3t17pCcGlemwknint6hfoeCVQrEMVwxRLRjXpq+BU= github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= @@ -226,10 +181,14 @@ go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.45.0 h1:x8Z78aZ go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.45.0/go.mod h1:62CPTSry9QZtOaSsE3tOzhx6LzDhHnXJ6xHeMNNiM6Q= go.opentelemetry.io/otel v1.19.0 h1:MuS/TNf4/j4IXsZuJegVzI1cwut7Qc00344rgH7p8bs= go.opentelemetry.io/otel v1.19.0/go.mod h1:i0QyjOq3UPoTzff0PJB2N66fb4S0+rSbSB15/oyH9fY= +go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.19.0 h1:Mne5On7VWdx7omSrSSZvM4Kw7cS7NQkOOmLcgscI51U= +go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.19.0 h1:IeMeyr1aBvBiPVYihXIaeIZba6b8E1bYp7lbdxK8CQg= go.opentelemetry.io/otel/metric v1.19.0 h1:aTzpGtV0ar9wlV4Sna9sdJyII5jTVJEvKETPiOKwvpE= go.opentelemetry.io/otel/metric v1.19.0/go.mod h1:L5rUsV9kM1IxCj1MmSdS+JQAcVm319EUrDVLrt7jqt8= +go.opentelemetry.io/otel/sdk v1.19.0 h1:6USY6zH+L8uMH8L3t1enZPR3WFEmSTADlqldyHtJi3o= go.opentelemetry.io/otel/trace v1.19.0 h1:DFVQmlVbfVeOuBRrwdtaehRrWiL1JoVs9CPIQ1Dzxpg= go.opentelemetry.io/otel/trace v1.19.0/go.mod h1:mfaSyvGyEJEI0nyV2I4qhNQnbBOUUmYZpYojqMnX2vo= +go.opentelemetry.io/proto/otlp v1.0.0 h1:T0TX0tmXU8a3CbNXzEKGeU5mIVOdf0oykP+u2lIVU/I= go.uber.org/atomic v1.11.0 h1:ZvwS0R+56ePWxUNi+Atn9dWONBPp/AUETXlHW0DxSjE= go.uber.org/atomic v1.11.0/go.mod h1:LUxbIzbOniOlMKjJjyPfpl4v+PKK2cNJn91OQbhoJI0= go.uber.org/goleak v1.2.1 h1:NBol2c7O1ZokfZ0LEU9K6Whx/KnwvepVetCUhtKja4A= @@ -257,8 +216,6 @@ golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8U golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.13.0/go.mod h1:y6Z2r+Rw4iayiXXAIxJIDAJ1zMW4yaTpebo8fPOliYc= -golang.org/x/crypto v0.14.0 h1:wBqGXzWJW6m1XrIKlAH0Hs1JJ7+9KBwnIO8v66Q9cHc= -golang.org/x/crypto v0.14.0/go.mod h1:MVFd36DqK4CsrnJYDkBA3VC4m2GkXAM0PvzMCn4JQf4= golang.org/x/crypto v0.17.0 h1:r8bRNjWL3GshPW3gkd+RpvzWrZAwPS49OmTGZ/uhM4k= golang.org/x/crypto v0.17.0/go.mod h1:gCAAfMLgwOJRpTjQ2zCCt2OcSfYMTeZVSRtQlPC7Nq4= golang.org/x/exp v0.0.0-20230725093048-515e97ebf090 h1:Di6/M8l0O2lCLc6VVRWhgCiApHV8MnQurBnFSHsQtNY= @@ -273,7 +230,6 @@ golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= -golang.org/x/net v0.0.0-20201224014010-6772e930b67b/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= @@ -288,21 +244,14 @@ golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.3.0 h1:ftCYgMx6zT/asHUrPw8BLLscYtGznsLAnjq5RH9P66E= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190606203320-7fc4e5ec1444/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190916202348-b4ddaad3f8a3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20191115151921-52ab43148777/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201204225414-ed752295db88/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210320140829-1e4c9ba3b0c4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210616045830-e2b7044e8c71/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210616094352-59db8d763f22/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20210906170528-6f6e22806c34/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20211025201205-69cdffdb9359/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20211116061358-0a5406a5449c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= @@ -310,8 +259,6 @@ golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.11.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.12.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.13.0 h1:Af8nKPmuFypiUBjVoU9V20FiaFXOcuZI21p0ycVYYGE= -golang.org/x/sys v0.13.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.15.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/sys v0.16.0 h1:xWw16ngr6ZMtmxDyKyIgsE93KNKz5HKmMa3b8ALHidU= golang.org/x/sys v0.16.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= @@ -325,7 +272,6 @@ golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= -golang.org/x/text v0.13.0 h1:ablQoSUd0tRdKxZewP80B+BaqeKJuVhuRxj/dkrun3k= golang.org/x/text v0.13.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= golang.org/x/text v0.14.0 h1:ScX5w1eTa3QqT8oi6+ziP7dTV1S2+ALU0bI+0zXKWiQ= golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= @@ -336,8 +282,6 @@ golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roY golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU= -golang.org/x/tools v0.7.0 h1:W4OVu8VVOaIO0yzWMNdepAulS7YfoS3Zabrm8DOXXU4= -golang.org/x/tools v0.7.0/go.mod h1:4pg6aUX35JBAogB10C9AtvVL+qowtN4pT3CGSQex14s= golang.org/x/tools v0.10.0 h1:tvDr/iQoUqNdohiYm0LmmKcBk+q86lb9EprIUFhHHGg= golang.org/x/tools v0.10.0/go.mod h1:UJwyiVBsOA2uwvK/e5OY3GTpDUJriEd+/YlqAwLPmyM= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= @@ -346,22 +290,18 @@ golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8T golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2 h1:H2TDz8ibqkAF6YGhCdN3jS9O0/s90v0rJh3X/OLHEUk= golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2/go.mod h1:K8+ghG5WaK9qNqU5K3HdILfMLy1f3aNYFI/wnl100a8= -google.golang.org/genproto/googleapis/rpc v0.0.0-20230525234030-28d5490b6b19 h1:0nDDozoAU19Qb2HwhXadU8OcsiO/09cnTqhUtq2MEOM= -google.golang.org/genproto/googleapis/rpc v0.0.0-20230525234030-28d5490b6b19/go.mod h1:66JfowdXAEgad5O9NnYcsNPLCPZJD++2L9X0PCMODrA= +google.golang.org/genproto v0.0.0-20230711160842-782d3b101e98 h1:Z0hjGZePRE0ZBWotvtrwxFNrNE9CUAGtplaDK5NNI/g= +google.golang.org/genproto/googleapis/api v0.0.0-20230711160842-782d3b101e98 h1:FmF5cCW94Ij59cfpoLiwTgodWmm60eEV0CjlsVg2fuw= google.golang.org/genproto/googleapis/rpc v0.0.0-20230711160842-782d3b101e98 h1:bVf09lpb+OJbByTj913DRJioFFAjf/ZGxEz7MajTp2U= google.golang.org/genproto/googleapis/rpc v0.0.0-20230711160842-782d3b101e98/go.mod h1:TUfxEVdsvPg18p6AslUXFoLdpED4oBnGwyqk3dV1XzM= -google.golang.org/grpc v1.57.1 h1:upNTNqv0ES+2ZOOqACwVtS3Il8M12/+Hz41RCPzAjQg= -google.golang.org/grpc v1.57.1/go.mod h1:Sd+9RMTACXwmub0zcNY2c4arhtrbBYD1AUHI/dt16Mo= google.golang.org/grpc v1.58.3 h1:BjnpXut1btbtgN/6sp+brB2Kbm2LjNXnidYujAVbSoQ= google.golang.org/grpc v1.58.3/go.mod h1:tgX3ZQDlNJGU96V6yHh1T/JeoBQ2TXdr43YbYSsCJk0= google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= -google.golang.org/protobuf v1.27.1/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= google.golang.org/protobuf v1.31.0 h1:g0LDEJHgrBl9N9r17Ru3sqWhkIx2NB67okBHPwC7hs8= google.golang.org/protobuf v1.31.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= -gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= From d08d635bed1415fc3f69ce4068d9360c76397a5e Mon Sep 17 00:00:00 2001 From: Nadezhda Savchenko Date: Thu, 22 Feb 2024 14:03:16 +0300 Subject: [PATCH 03/21] Linter fixes --- .gitignore | 1 + app.go | 3 ++- app_test.go | 44 ++++++++++++++++++++------------------- config.go | 4 ++-- config_test.go | 3 ++- diff.go | 14 ++++++------- ldap.go | 14 +++++++++---- main.go | 1 + source_models.go | 24 ++++++++++----------- testcontainer_openldap.go | 6 ++++-- ytsaurus.go | 16 ++++++++++++-- ytsaurus_models.go | 10 ++++----- ytsaurus_test.go | 6 +++--- 13 files changed, 85 insertions(+), 61 deletions(-) diff --git a/.gitignore b/.gitignore index 979da68..0ac30f5 100644 --- a/.gitignore +++ b/.gitignore @@ -1,2 +1,3 @@ *.local.yaml specs +.idea diff --git a/app.go b/app.go index a198df9..48e19e9 100644 --- a/app.go +++ b/app.go @@ -1,12 +1,13 @@ package main import ( - "github.com/pkg/errors" "os" "os/signal" "syscall" "time" + "github.com/pkg/errors" + "k8s.io/utils/clock" ) diff --git a/app_test.go b/app_test.go index 6937815..abcc8f9 100644 --- a/app_test.go +++ b/app_test.go @@ -3,13 +3,15 @@ package main import ( "context" "fmt" - "github.com/go-ldap/ldap/v3" - "go.ytsaurus.tech/library/go/ptr" "os" "strings" "testing" "time" + "github.com/go-ldap/ldap/v3" + + "go.ytsaurus.tech/library/go/ptr" + "github.com/google/go-cmp/cmp" "github.com/google/go-cmp/cmp/cmpopts" "github.com/stretchr/testify/require" @@ -41,7 +43,7 @@ type testCase struct { ytGroupsExpected []YtsaurusGroupWithMembers } -func getUserId(name string) string { +func getUserID(name string) string { switch name { case aliceName: return "1" @@ -59,7 +61,7 @@ func getSourceUser(name string, sourceType SourceType) SourceUser { return LdapUser{ BasicSourceUser: BasicSourceUser{SourceType: LdapSourceType}, Username: fmt.Sprintf("%v@acme.com", name), - Uid: getUserId(name), + UID: getUserID(name), FirstName: fmt.Sprintf("%v@acme.com-firstname", name), } case AzureSourceType: @@ -84,7 +86,7 @@ func getUpdatedSourceUser(name string, sourceType SourceType) SourceUser { return LdapUser{ BasicSourceUser: ldapSourceUser.BasicSourceUser, Username: ldapSourceUser.Username, - Uid: ldapSourceUser.Uid, + UID: ldapSourceUser.UID, FirstName: ldapSourceUser.FirstName + "-updated", } case AzureSourceType: @@ -331,11 +333,11 @@ func getTestCases(sourceType SourceType) []testCase { sourceGroupsSetUp: []SourceGroupWithMembers{ { SourceGroup: getSourceGroup("devs", sourceType), - Members: NewStringSetFromItems(getSourceUser(aliceName, sourceType).GetId()), + Members: NewStringSetFromItems(getSourceUser(aliceName, sourceType).GetID()), }, { SourceGroup: getSourceGroup("hq", sourceType), - Members: NewStringSetFromItems(getSourceUser(carolName, sourceType).GetId()), + Members: NewStringSetFromItems(getSourceUser(carolName, sourceType).GetID()), }, }, ytGroupsExpected: []YtsaurusGroupWithMembers{ @@ -380,8 +382,8 @@ func getTestCases(sourceType SourceType) []testCase { { SourceGroup: getSourceGroup("devs", sourceType), Members: NewStringSetFromItems( - getSourceUser(aliceName, sourceType).GetId(), - getSourceUser(carolName, sourceType).GetId(), + getSourceUser(aliceName, sourceType).GetID(), + getSourceUser(carolName, sourceType).GetID(), ), }, }, @@ -438,8 +440,8 @@ func getTestCases(sourceType SourceType) []testCase { SourceGroup: getUpdatedSourceGroup("devs", sourceType), // Members list are also updated. Members: NewStringSetFromItems( - getSourceUser(aliceName, sourceType).GetId(), - getSourceUser(carolName, sourceType).GetId(), + getSourceUser(aliceName, sourceType).GetID(), + getSourceUser(carolName, sourceType).GetID(), ), }, { @@ -447,8 +449,8 @@ func getTestCases(sourceType SourceType) []testCase { SourceGroup: getChangedBackwardCompatibleSourceGroup("hq", sourceType), // members also changed Members: NewStringSetFromItems( - getSourceUser(aliceName, sourceType).GetId(), - getSourceUser(carolName, sourceType).GetId(), + getSourceUser(aliceName, sourceType).GetID(), + getSourceUser(carolName, sourceType).GetID(), ), }, }, @@ -692,7 +694,7 @@ func setupLdapObjects(t *testing.T, conn *ldap.Conn, users []SourceUser, groups for _, user := range users { ldapUser := user.(LdapUser) addRequest := ldap.AddRequest{ - DN: fmt.Sprintf("uid=%s,ou=People,dc=example,dc=org", user.GetId()), + DN: fmt.Sprintf("uid=%s,ou=People,dc=example,dc=org", user.GetID()), Attributes: []ldap.Attribute{ { Type: "objectClass", @@ -708,15 +710,15 @@ func setupLdapObjects(t *testing.T, conn *ldap.Conn, users []SourceUser, groups }, { Type: "uid", - Vals: []string{user.GetId()}, + Vals: []string{user.GetID()}, }, { Type: "uidNumber", - Vals: []string{user.GetId()}, + Vals: []string{user.GetID()}, }, { Type: "gidNumber", - Vals: []string{user.GetId()}, + Vals: []string{user.GetID()}, }, { Type: "givenName", @@ -724,7 +726,7 @@ func setupLdapObjects(t *testing.T, conn *ldap.Conn, users []SourceUser, groups }, { Type: "homeDirectory", - Vals: []string{ldapUser.GetId()}, + Vals: []string{ldapUser.GetID()}, }, { Type: "sn", @@ -735,7 +737,7 @@ func setupLdapObjects(t *testing.T, conn *ldap.Conn, users []SourceUser, groups require.NoError(t, conn.Add(&addRequest)) } - for groupId, group := range groups { + for groupID, group := range groups { ldapGroup := group.SourceGroup.(LdapGroup) members := make([]string, 0) @@ -744,7 +746,7 @@ func setupLdapObjects(t *testing.T, conn *ldap.Conn, users []SourceUser, groups } addRequest := ldap.AddRequest{ - DN: fmt.Sprintf("cn=%s,ou=Group,dc=example,dc=org", ldapGroup.GetId()), + DN: fmt.Sprintf("cn=%s,ou=Group,dc=example,dc=org", ldapGroup.GetID()), Attributes: []ldap.Attribute{ { Type: "objectClass", @@ -756,7 +758,7 @@ func setupLdapObjects(t *testing.T, conn *ldap.Conn, users []SourceUser, groups }, { Type: "gidNumber", - Vals: []string{fmt.Sprint(groupId)}, + Vals: []string{fmt.Sprint(groupID)}, }, { Type: "memberUid", diff --git a/config.go b/config.go index d3d7fad..a9cead0 100644 --- a/config.go +++ b/config.go @@ -70,7 +70,7 @@ type LdapUsersConfig struct { // For example, `cn`. UsernameAttributeType string `yaml:"username_attribute_type"` // For example, `uid`. - UidAttributeType string `yaml:"uid_attribute_type"` + UIDAttributeType string `yaml:"uid_attribute_type"` FirstNameAttributeType *string `yaml:"first_name_attribute_type"` // A list of usernames for which app will print more debug info in logs. DebugUsernames []string `yaml:"debug_usernames"` @@ -85,7 +85,7 @@ type LdapGroupsConfig struct { GroupnameAttributeType string `yaml:"groupname_attribute_type"` // An attribute type which will be used for getting group members. // For example, `memberUid`. - MemberUidAttributeType string `yaml:"member_uid_attribute_type"` + MemberUIDAttributeType string `yaml:"member_uid_attribute_type"` // A list of groupnames for which app will print more debug info in logs. DebugGroupnames []string `yaml:"debug_groupnames"` diff --git a/config_test.go b/config_test.go index 6d075a2..12e0269 100644 --- a/config_test.go +++ b/config_test.go @@ -2,10 +2,11 @@ package main import ( "embed" - "go.ytsaurus.tech/library/go/ptr" "testing" "time" + "go.ytsaurus.tech/library/go/ptr" + "github.com/stretchr/testify/require" ) diff --git a/diff.go b/diff.go index 771f662..6d401fc 100644 --- a/diff.go +++ b/diff.go @@ -54,10 +54,10 @@ func (a *App) syncUsers() (map[ObjectID]YtsaurusUser, error) { ytUsersMap := make(map[ObjectID]YtsaurusUser) for _, user := range sourceUsers { - sourceUsersMap[user.GetId()] = user + sourceUsersMap[user.GetID()] = user } for _, user := range ytUsers { - ytUsersMap[user.SourceUser.GetId()] = user + ytUsersMap[user.SourceUser.GetID()] = user } diff := a.diffUsers(sourceUsersMap, ytUsersMap) @@ -80,7 +80,7 @@ func (a *App) syncUsers() (map[ObjectID]YtsaurusUser, error) { removedCount++ } // Actualizing user map for group sync later. - delete(ytUsersMap, user.SourceUser.GetId()) + delete(ytUsersMap, user.SourceUser.GetID()) } for _, user := range diff.create { err = a.ytsaurus.CreateUser(user) @@ -89,7 +89,7 @@ func (a *App) syncUsers() (map[ObjectID]YtsaurusUser, error) { a.logger.Errorw("failed to create user", zap.Error(err), "user", user) } // Actualizing user map for group sync later. - ytUsersMap[user.SourceUser.GetId()] = user + ytUsersMap[user.SourceUser.GetID()] = user } for _, updatedUser := range diff.update { err = a.ytsaurus.UpdateUser(updatedUser.OldUsername, updatedUser.YtsaurusUser) @@ -98,7 +98,7 @@ func (a *App) syncUsers() (map[ObjectID]YtsaurusUser, error) { a.logger.Errorw("failed to update user", zap.Error(err), "user", updatedUser) } // Actualizing user map for group sync later. - ytUsersMap[updatedUser.SourceUser.GetId()] = updatedUser.YtsaurusUser + ytUsersMap[updatedUser.SourceUser.GetID()] = updatedUser.YtsaurusUser } a.logger.Infow("Finish syncing users", "created", len(diff.create)-createErrCount, @@ -207,10 +207,10 @@ func (a *App) diffGroups( ytGroupsWithMembersMap := make(map[ObjectID]YtsaurusGroupWithMembers) for _, group := range sourceGroups { - sourceGroupsWithMembersMap[group.SourceGroup.GetId()] = group + sourceGroupsWithMembersMap[group.SourceGroup.GetID()] = group } for _, group := range ytGroups { - ytGroupsWithMembersMap[group.SourceGroup.GetId()] = group + ytGroupsWithMembersMap[group.SourceGroup.GetID()] = group } // Collecting groups to create (the ones that exist in Source but not in YTsaurus). diff --git a/ldap.go b/ldap.go index d9db27b..f9984c9 100644 --- a/ldap.go +++ b/ldap.go @@ -1,9 +1,10 @@ package main import ( + "log" + "github.com/go-ldap/ldap/v3" "k8s.io/utils/env" - "log" ) type Ldap struct { @@ -15,12 +16,17 @@ func NewLdap(cfg *LdapConfig, logger appLoggerType) (*Ldap, error) { conn, err := ldap.DialURL(cfg.Address) if err != nil { log.Fatalf("Failed to connect: %s\n", err) + return nil, err } _, err = conn.SimpleBind(&ldap.SimpleBindRequest{ Username: cfg.BindDN, Password: env.GetString(cfg.BindPasswordEnvVar, "adminpassword"), }) + if err != nil { + log.Fatalf("Failed to bind: %s\n", err) + return nil, err + } return &Ldap{ Connection: conn, Config: cfg, @@ -41,7 +47,7 @@ func (source *Ldap) GetUsers() ([]SourceUser, error) { var users []SourceUser for _, entry := range res.Entries { username := entry.GetAttributeValue(source.Config.Users.UsernameAttributeType) - uid := entry.GetAttributeValue(source.Config.Users.UidAttributeType) + uid := entry.GetAttributeValue(source.Config.Users.UIDAttributeType) var firstName string if source.Config.Users.FirstNameAttributeType != nil { firstName = entry.GetAttributeValue(*source.Config.Users.FirstNameAttributeType) @@ -49,7 +55,7 @@ func (source *Ldap) GetUsers() ([]SourceUser, error) { users = append(users, LdapUser{ BasicSourceUser: BasicSourceUser{SourceType: LdapSourceType}, Username: username, - Uid: uid, + UID: uid, FirstName: firstName}) } return users, nil @@ -69,7 +75,7 @@ func (source *Ldap) GetGroupsWithMembers() ([]SourceGroupWithMembers, error) { var groups []SourceGroupWithMembers for _, entry := range res.Entries { groupname := entry.GetAttributeValue(source.Config.Groups.GroupnameAttributeType) - members := entry.GetAttributeValues(source.Config.Groups.MemberUidAttributeType) + members := entry.GetAttributeValues(source.Config.Groups.MemberUIDAttributeType) groups = append(groups, SourceGroupWithMembers{ SourceGroup: LdapGroup{ BasicSourceGroup: BasicSourceGroup{SourceType: LdapSourceType}, diff --git a/main.go b/main.go index 0b36f24..f0330da 100644 --- a/main.go +++ b/main.go @@ -2,6 +2,7 @@ package main import ( "fmt" + "github.com/jessevdk/go-flags" "github.com/pkg/errors" "go.uber.org/zap" diff --git a/source_models.go b/source_models.go index 44c6929..e957fd3 100644 --- a/source_models.go +++ b/source_models.go @@ -2,7 +2,7 @@ package main import ( "fmt" - "github.com/pkg/errors" + "go.ytsaurus.tech/yt/go/yson" ) @@ -15,7 +15,7 @@ const ( ) type SourceUser interface { - GetId() ObjectID + GetID() ObjectID GetName() string GetSourceType() SourceType } @@ -41,9 +41,8 @@ func NewSourceUser(attributes map[string]any) (SourceUser, error) { return nil, err } return azureUser, nil - } else { - return nil, errors.New(fmt.Sprintf("Unknown source type: %v", sourceType)) } + return nil, fmt.Errorf("unknown source type: %v", sourceType) } type BasicSourceUser struct { @@ -63,7 +62,7 @@ type AzureUser struct { DisplayName string `yson:"display_name"` } -func (user AzureUser) GetId() ObjectID { +func (user AzureUser) GetID() ObjectID { return user.AzureID } @@ -78,13 +77,13 @@ func (user AzureUser) GetSourceType() SourceType { type LdapUser struct { BasicSourceUser Username string `yson:"username"` - Uid string `yson:"uid"` + UID string `yson:"uid"` FirstName string `yson:"first_name"` // TODO(nadya73): Add more fields. } -func (user LdapUser) GetId() ObjectID { - return user.Uid +func (user LdapUser) GetID() ObjectID { + return user.UID } func (user LdapUser) GetName() string { @@ -96,7 +95,7 @@ func (user LdapUser) GetSourceType() SourceType { } type SourceGroup interface { - GetId() ObjectID + GetID() ObjectID GetName() string GetSourceType() SourceType } @@ -126,9 +125,8 @@ func NewSourceGroup(attributes map[string]any) (SourceGroup, error) { return nil, err } return azureGroup, nil - } else { - return nil, errors.New(fmt.Sprintf("Unknown source type: %v", sourceType)) } + return nil, fmt.Errorf("unknown source type: %v", sourceType) } type AzureGroup struct { @@ -141,7 +139,7 @@ type AzureGroup struct { DisplayName string `yson:"display_name"` } -func (ag AzureGroup) GetId() ObjectID { +func (ag AzureGroup) GetID() ObjectID { return ag.AzureID } @@ -158,7 +156,7 @@ type LdapGroup struct { Groupname string `yson:"groupname"` } -func (lg LdapGroup) GetId() ObjectID { +func (lg LdapGroup) GetID() ObjectID { return lg.Groupname } diff --git a/testcontainer_openldap.go b/testcontainer_openldap.go index 0af388a..c114677 100644 --- a/testcontainer_openldap.go +++ b/testcontainer_openldap.go @@ -3,8 +3,10 @@ package main import ( "context" "errors" + "github.com/testcontainers/testcontainers-go" "github.com/testcontainers/testcontainers-go/modules/openldap" + "go.ytsaurus.tech/library/go/ptr" ) @@ -39,13 +41,13 @@ func (y *OpenLdapLocal) GetConfig() (*LdapConfig, error) { Users: LdapUsersConfig{ Filter: "(&(objectClass=posixAccount)(ou=People))", UsernameAttributeType: "cn", - UidAttributeType: "uid", + UIDAttributeType: "uid", FirstNameAttributeType: ptr.String("givenName"), }, Groups: LdapGroupsConfig{ Filter: "(objectClass=posixGroup)", GroupnameAttributeType: "cn", - MemberUidAttributeType: "memberUid", + MemberUIDAttributeType: "memberUid", }, }, nil } diff --git a/ytsaurus.go b/ytsaurus.go index 87ef20e..288f041 100644 --- a/ytsaurus.go +++ b/ytsaurus.go @@ -338,12 +338,18 @@ func (y *Ytsaurus) isUserManaged(username string) (bool, error) { ypath.Path("//sys/users/"+username+"/@azure"), nil, ) + if err != nil { + return false, err + } attrSourceExists, err := y.client.NodeExists( ctx, ypath.Path("//sys/users/"+username+"/@source"), nil, ) - return attrAzureExists || attrSourceExists, err + if err != nil { + return false, err + } + return attrAzureExists || attrSourceExists, nil } func (y *Ytsaurus) ensureUserManaged(username string) error { @@ -366,13 +372,19 @@ func (y *Ytsaurus) isGroupManaged(name string) (bool, error) { ypath.Path("//sys/groups/"+name+"/@azure"), nil, ) + if err != nil { + return false, err + } attrSourceExists, err := y.client.NodeExists( ctx, ypath.Path("//sys/groups/"+name+"/@source"), nil, ) + if err != nil { + return false, err + } - return attrAzureExists || attrSourceExists, err + return attrAzureExists || attrSourceExists, nil } func (y *Ytsaurus) ensureGroupManaged(groupname string) error { diff --git a/ytsaurus_models.go b/ytsaurus_models.go index ace5478..e6c6853 100644 --- a/ytsaurus_models.go +++ b/ytsaurus_models.go @@ -53,6 +53,11 @@ func (g YtsaurusGroup) GetSourceAttributeName() string { return "source" } +// IsManuallyManaged true if group doesn't have @azure attribute (system or manually created group). +func (g YtsaurusGroup) IsManuallyManaged() bool { + return g.SourceGroup == nil +} + type YtsaurusGroupWithMembers struct { YtsaurusGroup // Members is a set of group members' @name attribute. @@ -67,8 +72,3 @@ type YtsaurusMembership struct { GroupName string Username string } - -// IsManuallyManaged true if group doesn't have @azure attribute (system or manually created group). -func (u YtsaurusGroup) IsManuallyManaged() bool { - return u.SourceGroup == nil -} diff --git a/ytsaurus_test.go b/ytsaurus_test.go index 76090c6..b1ce4a3 100644 --- a/ytsaurus_test.go +++ b/ytsaurus_test.go @@ -41,12 +41,12 @@ func TestUpdateUserFirstName(t *testing.T) { defer func() { require.NoError(t, ytLocal.Stop()) }() yt := getYtsaurus(t, ytLocal) - const azureId = "fake-az-id-old" + const azureID = "fake-az-id-old" managedOleg := YtsaurusUser{ Username: "oleg", SourceUser: AzureUser{ - AzureID: azureId, + AzureID: azureID, FirstName: "Lego", }, } @@ -54,7 +54,7 @@ func TestUpdateUserFirstName(t *testing.T) { require.NoError(t, err) updateSourceUser := AzureUser{ - AzureID: azureId, + AzureID: azureID, FirstName: "Oleg", } managedOleg.SourceUser = updateSourceUser From 38e9184a47bf900c6f3b92fad06289bfa55cb3ff Mon Sep 17 00:00:00 2001 From: Nadezhda Savchenko Date: Thu, 22 Feb 2024 14:16:24 +0300 Subject: [PATCH 04/21] Add LdapConfigTest --- app_integration_test.go | 2 +- ....example.yaml => azure_config.example.yaml | 0 config.go | 6 +- config_test.go | 56 ++++++++++++++++++- ldap_config.example.yaml | 1 - testcontainer_openldap.go | 2 +- 6 files changed, 58 insertions(+), 9 deletions(-) rename config.example.yaml => azure_config.example.yaml (100%) diff --git a/app_integration_test.go b/app_integration_test.go index fc42d2c..63a3835 100644 --- a/app_integration_test.go +++ b/app_integration_test.go @@ -16,7 +16,7 @@ const ( runLocalYtsaurus = false ) -// TestAppIntegration checks sync with real Source API and local yt +// TestAppIntegration checks sync with real Azure API and local yt // It requires AZURE_CLIENT_SECRET to be set. func TestAppIntegration(t *testing.T) { require.NoError(t, os.Setenv(defaultYtsaurusSecretEnvVar, ytDevToken)) diff --git a/config.example.yaml b/azure_config.example.yaml similarity index 100% rename from config.example.yaml rename to azure_config.example.yaml diff --git a/config.go b/config.go index a9cead0..f1a42be 100644 --- a/config.go +++ b/config.go @@ -50,11 +50,11 @@ type AzureConfig struct { UsersFilter string `yaml:"users_filter"` // GroupsFilter is MS Graph $filter value used for group fetching requests. // See https://learn.microsoft.com/en-us/graph/api/group-list - GroupsFilter string `yaml:"groups_filter"` - // GroupsDisplayNameSuffixPostFilter applied to the fetched groups display names. - Timeout time.Duration `yaml:"timeout"` + GroupsFilter string `yaml:"groups_filter"` + Timeout time.Duration `yaml:"timeout"` // TODO(nadya73): support for ldap also, but with other name. + // GroupsDisplayNameSuffixPostFilter applied to the fetched groups display names. GroupsDisplayNameSuffixPostFilter string `yaml:"groups_display_name_suffix_post_filter"` // TODO(nadya73): support for ldap also, but with other name. diff --git a/config_test.go b/config_test.go index 12e0269..b1dbd53 100644 --- a/config_test.go +++ b/config_test.go @@ -10,11 +10,11 @@ import ( "github.com/stretchr/testify/require" ) -//go:embed config.example.yaml +//go:embed azure_config.example.yaml ldap_config.example.yaml var _ embed.FS -func TestConfig(t *testing.T) { - configPath := "config.example.yaml" +func TestAzureConfig(t *testing.T) { + configPath := "azure_config.example.yaml" cfg, err := loadConfig(configPath) require.NoError(t, err) @@ -30,6 +30,8 @@ func TestConfig(t *testing.T) { require.Equal(t, ptr.Int(10), cfg.App.RemoveLimit) require.Equal(t, ptr.Duration(7*24*time.Hour), cfg.App.BanBeforeRemoveDuration) + require.True(t, cfg.Ldap == nil) + require.Equal(t, "acme.onmicrosoft.com", cfg.Azure.Tenant) require.Equal(t, "abcdefgh-a000-b111-c222-abcdef123456", cfg.Azure.ClientID) require.Equal(t, 1*time.Second, cfg.Azure.Timeout) @@ -51,3 +53,51 @@ func TestConfig(t *testing.T) { require.NoError(t, err) logger.Debugw("test logging message", "key", "val") } + +func TestLdapConfig(t *testing.T) { + configPath := "ldap_config.example.yaml" + + cfg, err := loadConfig(configPath) + require.NoError(t, err) + + require.Equal(t, ptr.Duration(5*time.Minute), cfg.App.SyncInterval) + require.Equal(t, []ReplacementPair{ + {From: "@acme.com", To: ""}, + {From: "@", To: ":"}, + }, cfg.App.UsernameReplacements) + require.Equal(t, []ReplacementPair{ + {From: "|all", To: ""}, + }, cfg.App.GroupnameReplacements) + require.Equal(t, ptr.Int(10), cfg.App.RemoveLimit) + require.Equal(t, ptr.Duration(7*24*time.Hour), cfg.App.BanBeforeRemoveDuration) + + require.True(t, cfg.Azure == nil) + + require.Equal(t, "dc=example,dc=org", cfg.Ldap.BaseDN) + require.Equal(t, "cn=admin,dc=example,dc=org", cfg.Ldap.BindDN) + require.Equal(t, "localhost:10210", cfg.Ldap.Address) + require.Equal(t, "LDAP_PASSWORD", cfg.Ldap.BindPasswordEnvVar) + + require.Equal(t, "(&(objectClass=posixAccount)(ou=People))", cfg.Ldap.Users.Filter) + require.Equal(t, "cn", cfg.Ldap.Users.UsernameAttributeType) + require.Equal(t, "uid", cfg.Ldap.Users.UIDAttributeType) + require.Equal(t, ptr.String("givenName"), cfg.Ldap.Users.FirstNameAttributeType) + + require.Equal(t, "(objectClass=posixGroup)", cfg.Ldap.Groups.Filter) + require.Equal(t, "cn", cfg.Ldap.Groups.GroupnameAttributeType) + require.Equal(t, "memberUid", cfg.Ldap.Groups.MemberUIDAttributeType) + + require.Equal(t, "localhost:10110", cfg.Ytsaurus.Proxy) + require.Equal(t, true, cfg.Ytsaurus.ApplyUserChanges) + require.Equal(t, true, cfg.Ytsaurus.ApplyGroupChanges) + require.Equal(t, true, cfg.Ytsaurus.ApplyMemberChanges) + require.Equal(t, 1*time.Second, cfg.Ytsaurus.Timeout) + require.Equal(t, "DEBUG", cfg.Ytsaurus.LogLevel) + + require.Equal(t, "WARN", cfg.Logging.Level) + require.Equal(t, true, cfg.Logging.IsProduction) + + logger, err := configureLogger(&cfg.Logging) + require.NoError(t, err) + logger.Debugw("test logging message", "key", "val") +} diff --git a/ldap_config.example.yaml b/ldap_config.example.yaml index 3b5ca10..06fd81f 100644 --- a/ldap_config.example.yaml +++ b/ldap_config.example.yaml @@ -16,7 +16,6 @@ ldap: bind_dn: "cn=admin,dc=example,dc=org" bind_password_env_var: "LDAP_PASSWORD" base_dn: "dc=example,dc=org" - timeout: 1s users: filter: "(&(objectClass=posixAccount)(ou=People))" username_attribute_type: "cn" diff --git a/testcontainer_openldap.go b/testcontainer_openldap.go index c114677..ab74f93 100644 --- a/testcontainer_openldap.go +++ b/testcontainer_openldap.go @@ -34,7 +34,7 @@ func (y *OpenLdapLocal) GetConfig() (*LdapConfig, error) { return nil, err } return &LdapConfig{ - Address: connectionString, //"ldap://localhost:1389", + Address: connectionString, BaseDN: "dc=example,dc=org", BindDN: "cn=admin,dc=example,dc=org", BindPasswordEnvVar: "LDAP_PASSWORD", From 11fc26fb752482d37c2ce6f897603e7a6a7ad7a9 Mon Sep 17 00:00:00 2001 From: Nadezhda Savchenko Date: Thu, 22 Feb 2024 14:21:45 +0300 Subject: [PATCH 05/21] Ignore users from different source --- ytsaurus.go | 8 ++++---- ytsaurus_models.go | 8 ++++---- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/ytsaurus.go b/ytsaurus.go index 288f041..6cee4cd 100644 --- a/ytsaurus.go +++ b/ytsaurus.go @@ -75,7 +75,7 @@ func NewYtsaurus(cfg *YtsaurusConfig, logger appLoggerType, clock clock.PassiveC }, nil } -func (y *Ytsaurus) GetUsers() ([]YtsaurusUser, error) { +func (y *Ytsaurus) GetUsers(sourceType SourceType) ([]YtsaurusUser, error) { ctx, cancel := context.WithTimeout(context.Background(), y.timeout) defer cancel() @@ -86,7 +86,7 @@ func (y *Ytsaurus) GetUsers() ([]YtsaurusUser, error) { var managedUsers []YtsaurusUser for _, user := range users { y.maybePrintExtraLogs(user.Username, "get_user", "user", user) - if user.IsManuallyManaged() { + if user.IsManuallyManaged(sourceType) { continue } managedUsers = append(managedUsers, user) @@ -195,7 +195,7 @@ func (y *Ytsaurus) BanUser(username string) error { ) } -func (y *Ytsaurus) GetGroupsWithMembers() ([]YtsaurusGroupWithMembers, error) { +func (y *Ytsaurus) GetGroupsWithMembers(sourceType SourceType) ([]YtsaurusGroupWithMembers, error) { ctx, cancel := context.WithTimeout(context.Background(), y.timeout) defer cancel() @@ -206,7 +206,7 @@ func (y *Ytsaurus) GetGroupsWithMembers() ([]YtsaurusGroupWithMembers, error) { var managedGroups []YtsaurusGroupWithMembers for _, group := range groups { y.maybePrintExtraLogs(group.Name, "get_group", "group", group) - if group.IsManuallyManaged() { + if group.IsManuallyManaged(sourceType) { continue } managedGroups = append(managedGroups, group) diff --git a/ytsaurus_models.go b/ytsaurus_models.go index e6c6853..43dd96a 100644 --- a/ytsaurus_models.go +++ b/ytsaurus_models.go @@ -22,8 +22,8 @@ func (u YtsaurusUser) GetSourceAttributeName() string { } // IsManuallyManaged true if user doesn't have @azure attribute (system or manually created user). -func (u YtsaurusUser) IsManuallyManaged() bool { - return u.SourceUser == nil +func (u YtsaurusUser) IsManuallyManaged(sourceType SourceType) bool { + return u.SourceUser == nil || u.SourceUser.GetSourceType() != sourceType } func (u YtsaurusUser) IsBanned() bool { @@ -54,8 +54,8 @@ func (g YtsaurusGroup) GetSourceAttributeName() string { } // IsManuallyManaged true if group doesn't have @azure attribute (system or manually created group). -func (g YtsaurusGroup) IsManuallyManaged() bool { - return g.SourceGroup == nil +func (g YtsaurusGroup) IsManuallyManaged(sourceType SourceType) bool { + return g.SourceGroup == nil || g.SourceGroup.GetSourceType() != sourceType } type YtsaurusGroupWithMembers struct { From 8e056a825ecc32f582bafd1afa0cfe0535b68722 Mon Sep 17 00:00:00 2001 From: Nadezhda Savchenko Date: Thu, 22 Feb 2024 14:26:27 +0300 Subject: [PATCH 06/21] Fix build --- app.go | 5 ----- azure_fake.go | 4 ++++ azure_real.go | 4 ++++ diff.go | 4 ++-- ldap.go | 4 ++++ source_models.go => source.go | 6 ++++++ 6 files changed, 20 insertions(+), 7 deletions(-) rename source_models.go => source.go (96%) diff --git a/app.go b/app.go index 48e19e9..f1dbc37 100644 --- a/app.go +++ b/app.go @@ -11,11 +11,6 @@ import ( "k8s.io/utils/clock" ) -type Source interface { - GetUsers() ([]SourceUser, error) - GetGroupsWithMembers() ([]SourceGroupWithMembers, error) -} - type App struct { syncInterval *time.Duration usernameReplaces []ReplacementPair diff --git a/azure_fake.go b/azure_fake.go index 937261c..9986bde 100644 --- a/azure_fake.go +++ b/azure_fake.go @@ -17,6 +17,10 @@ func (a *AzureFake) setGroups(groups []SourceGroupWithMembers) { a.groups = groups } +func (a *AzureFake) GetSourceType() SourceType { + return AzureSourceType +} + func (a *AzureFake) GetUsers() ([]SourceUser, error) { return a.users, nil } diff --git a/azure_real.go b/azure_real.go index 5ecf5a4..d25976d 100644 --- a/azure_real.go +++ b/azure_real.go @@ -98,6 +98,10 @@ func handleNil[T any](s *T) T { return result } +func (a *AzureReal) GetSourceType() SourceType { + return AzureSourceType +} + func (a *AzureReal) GetUsers() ([]SourceUser, error) { ctx, cancel := context.WithTimeout(context.Background(), a.timeout) defer cancel() diff --git a/diff.go b/diff.go index 6d401fc..e39b87f 100644 --- a/diff.go +++ b/diff.go @@ -45,7 +45,7 @@ func (a *App) syncUsers() (map[ObjectID]YtsaurusUser, error) { } } - ytUsers, err := a.ytsaurus.GetUsers() + ytUsers, err := a.ytsaurus.GetUsers(a.source.GetSourceType()) if err != nil { return nil, errors.Wrap(err, "failed to get YTsaurus users") } @@ -118,7 +118,7 @@ func (a *App) syncGroups(usersMap map[ObjectID]YtsaurusUser) error { if err != nil { return errors.Wrap(err, "failed to get Source groups") } - ytGroups, err := a.ytsaurus.GetGroupsWithMembers() + ytGroups, err := a.ytsaurus.GetGroupsWithMembers(a.source.GetSourceType()) if err != nil { return errors.Wrap(err, "failed to get YTsaurus groups") } diff --git a/ldap.go b/ldap.go index f9984c9..a715eeb 100644 --- a/ldap.go +++ b/ldap.go @@ -33,6 +33,10 @@ func NewLdap(cfg *LdapConfig, logger appLoggerType) (*Ldap, error) { }, nil } +func (source *Ldap) GetSourceType() SourceType { + return LdapSourceType +} + func (source *Ldap) GetUsers() ([]SourceUser, error) { res, err := source.Connection.Search(&ldap.SearchRequest{ BaseDN: source.Config.BaseDN, diff --git a/source_models.go b/source.go similarity index 96% rename from source_models.go rename to source.go index e957fd3..c50ffb1 100644 --- a/source_models.go +++ b/source.go @@ -14,6 +14,12 @@ const ( AzureSourceType SourceType = "azure" ) +type Source interface { + GetUsers() ([]SourceUser, error) + GetGroupsWithMembers() ([]SourceGroupWithMembers, error) + GetSourceType() SourceType +} + type SourceUser interface { GetID() ObjectID GetName() string From 1c40a387afd420743e3e7fb066717ef2b375bb7e Mon Sep 17 00:00:00 2001 From: Nadezhda Savchenko Date: Thu, 22 Feb 2024 14:29:46 +0300 Subject: [PATCH 07/21] Fix test --- ytsaurus_test.go | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/ytsaurus_test.go b/ytsaurus_test.go index b1ce4a3..32eb425 100644 --- a/ytsaurus_test.go +++ b/ytsaurus_test.go @@ -82,7 +82,7 @@ func TestGroups(t *testing.T) { defer func() { require.NoError(t, ytLocal.Stop()) }() yt := getYtsaurus(t, ytLocal) - groupsInitial, err := yt.GetGroupsWithMembers() + groupsInitial, err := yt.GetGroupsWithMembers(AzureSourceType) require.NoError(t, err) require.Empty(t, groupsInitial) @@ -108,7 +108,7 @@ func TestGroups(t *testing.T) { err = yt.AddMember(managedOleg.Username, managedOlegsGroup.Name) require.NoError(t, err) - groupsAfterCreate, err := yt.GetGroupsWithMembers() + groupsAfterCreate, err := yt.GetGroupsWithMembers(AzureSourceType) require.NoError(t, err) members := NewStringSet() members.Add(managedOleg.Username) @@ -128,7 +128,7 @@ func TestGroups(t *testing.T) { err = yt.RemoveMember(managedOleg.Username, managedOlegsGroup.Name) require.NoError(t, err) - groupsAfterRemoveMember, err := yt.GetGroupsWithMembers() + groupsAfterRemoveMember, err := yt.GetGroupsWithMembers(AzureSourceType) require.NoError(t, err) require.Equal(t, []YtsaurusGroupWithMembers{ { @@ -146,7 +146,7 @@ func TestGroups(t *testing.T) { err = yt.RemoveGroup(managedOlegsGroup.Name) require.NoError(t, err) - groupsAfterRemove, err := yt.GetGroupsWithMembers() + groupsAfterRemove, err := yt.GetGroupsWithMembers(AzureSourceType) require.NoError(t, err) require.Empty(t, groupsAfterRemove) From 4d94268b29080fe9fca3e27cc081777b53865b13 Mon Sep 17 00:00:00 2001 From: Nadezhda Savchenko Date: Mon, 26 Feb 2024 16:33:40 +0300 Subject: [PATCH 08/21] Add source_type attribute --- app_test.go | 67 ++++++++++++++++++++------------------------- ldap.go | 10 +++---- source.go | 28 ++++++------------- ytsaurus.go | 2 ++ ytsaurus_helpers.go | 28 +++++++++++++------ 5 files changed, 62 insertions(+), 73 deletions(-) diff --git a/app_test.go b/app_test.go index abcc8f9..a8326b7 100644 --- a/app_test.go +++ b/app_test.go @@ -59,20 +59,18 @@ func getSourceUser(name string, sourceType SourceType) SourceUser { switch sourceType { case LdapSourceType: return LdapUser{ - BasicSourceUser: BasicSourceUser{SourceType: LdapSourceType}, - Username: fmt.Sprintf("%v@acme.com", name), - UID: getUserID(name), - FirstName: fmt.Sprintf("%v@acme.com-firstname", name), + Username: fmt.Sprintf("%v@acme.com", name), + UID: getUserID(name), + FirstName: fmt.Sprintf("%v@acme.com-firstname", name), } case AzureSourceType: return AzureUser{ - BasicSourceUser: BasicSourceUser{SourceType: AzureSourceType}, - PrincipalName: fmt.Sprintf("%v@acme.com", name), - AzureID: fmt.Sprintf("fake-az-id-%v", name), - Email: fmt.Sprintf("%v@acme.com", name), - FirstName: fmt.Sprintf("%v@acme.com-firstname", name), - LastName: fmt.Sprintf("%v-lastname", name), - DisplayName: fmt.Sprintf("Henderson, %v (ACME)", name), + PrincipalName: fmt.Sprintf("%v@acme.com", name), + AzureID: fmt.Sprintf("fake-az-id-%v", name), + Email: fmt.Sprintf("%v@acme.com", name), + FirstName: fmt.Sprintf("%v@acme.com-firstname", name), + LastName: fmt.Sprintf("%v-lastname", name), + DisplayName: fmt.Sprintf("Henderson, %v (ACME)", name), } } return nil @@ -84,21 +82,19 @@ func getUpdatedSourceUser(name string, sourceType SourceType) SourceUser { case LdapSourceType: ldapSourceUser := sourceUser.(LdapUser) return LdapUser{ - BasicSourceUser: ldapSourceUser.BasicSourceUser, - Username: ldapSourceUser.Username, - UID: ldapSourceUser.UID, - FirstName: ldapSourceUser.FirstName + "-updated", + Username: ldapSourceUser.Username, + UID: ldapSourceUser.UID, + FirstName: ldapSourceUser.FirstName + "-updated", } case AzureSourceType: azureSourceUser := sourceUser.(AzureUser) return AzureUser{ - BasicSourceUser: azureSourceUser.BasicSourceUser, - PrincipalName: azureSourceUser.PrincipalName, - AzureID: azureSourceUser.AzureID, - Email: azureSourceUser.Email + "-updated", - FirstName: azureSourceUser.FirstName, - LastName: azureSourceUser.LastName, - DisplayName: azureSourceUser.DisplayName, + PrincipalName: azureSourceUser.PrincipalName, + AzureID: azureSourceUser.AzureID, + Email: azureSourceUser.Email + "-updated", + FirstName: azureSourceUser.FirstName, + LastName: azureSourceUser.LastName, + DisplayName: azureSourceUser.DisplayName, } } return nil @@ -120,15 +116,13 @@ func getSourceGroup(name string, sourceType SourceType) SourceGroup { switch sourceType { case AzureSourceType: return AzureGroup{ - BasicSourceGroup: BasicSourceGroup{SourceType: AzureSourceType}, - Identity: fmt.Sprintf("acme.%v|all", name), - AzureID: fmt.Sprintf("fake-az-acme.%v", name), - DisplayName: fmt.Sprintf("acme.%v|all", name), + Identity: fmt.Sprintf("acme.%v|all", name), + AzureID: fmt.Sprintf("fake-az-acme.%v", name), + DisplayName: fmt.Sprintf("acme.%v|all", name), } case LdapSourceType: return LdapGroup{ - BasicSourceGroup: BasicSourceGroup{SourceType: LdapSourceType}, - Groupname: fmt.Sprintf("acme.%v|all", name), + Groupname: fmt.Sprintf("acme.%v|all", name), } } return nil @@ -141,16 +135,14 @@ func getUpdatedSourceGroup(name string, sourceType SourceType) SourceGroup { // TODO(nadya73): add more fields. ldapSourceGroup := sourceGroup.(LdapGroup) return LdapGroup{ - BasicSourceGroup: ldapSourceGroup.BasicSourceGroup, - Groupname: ldapSourceGroup.Groupname, + Groupname: ldapSourceGroup.Groupname, } case AzureSourceType: azureSourceGroup := sourceGroup.(AzureGroup) return AzureGroup{ - BasicSourceGroup: azureSourceGroup.BasicSourceGroup, - Identity: azureSourceGroup.Identity, - AzureID: azureSourceGroup.AzureID, - DisplayName: azureSourceGroup.DisplayName + "-updated", + Identity: azureSourceGroup.Identity, + AzureID: azureSourceGroup.AzureID, + DisplayName: azureSourceGroup.DisplayName + "-updated", } } return nil @@ -163,10 +155,9 @@ func getChangedBackwardCompatibleSourceGroup(name string, sourceType SourceType) sourceGroup := getSourceGroup(name, sourceType) azureSourceGroup := sourceGroup.(AzureGroup) return AzureGroup{ - BasicSourceGroup: azureSourceGroup.BasicSourceGroup, - Identity: azureSourceGroup.Identity + "-changed", - AzureID: azureSourceGroup.AzureID, - DisplayName: azureSourceGroup.DisplayName + "-updated", + Identity: azureSourceGroup.Identity + "-changed", + AzureID: azureSourceGroup.AzureID, + DisplayName: azureSourceGroup.DisplayName + "-updated", } } diff --git a/ldap.go b/ldap.go index a715eeb..432d3c5 100644 --- a/ldap.go +++ b/ldap.go @@ -57,10 +57,9 @@ func (source *Ldap) GetUsers() ([]SourceUser, error) { firstName = entry.GetAttributeValue(*source.Config.Users.FirstNameAttributeType) } users = append(users, LdapUser{ - BasicSourceUser: BasicSourceUser{SourceType: LdapSourceType}, - Username: username, - UID: uid, - FirstName: firstName}) + Username: username, + UID: uid, + FirstName: firstName}) } return users, nil } @@ -82,8 +81,7 @@ func (source *Ldap) GetGroupsWithMembers() ([]SourceGroupWithMembers, error) { members := entry.GetAttributeValues(source.Config.Groups.MemberUIDAttributeType) groups = append(groups, SourceGroupWithMembers{ SourceGroup: LdapGroup{ - BasicSourceGroup: BasicSourceGroup{SourceType: LdapSourceType}, - Groupname: groupname, + Groupname: groupname, }, Members: NewStringSetFromItems(members...), }) diff --git a/source.go b/source.go index c50ffb1..753fd71 100644 --- a/source.go +++ b/source.go @@ -26,21 +26,21 @@ type SourceUser interface { GetSourceType() SourceType } -func NewSourceUser(attributes map[string]any) (SourceUser, error) { +func NewSourceUser(sourceType SourceType, attributes map[string]any) (SourceUser, error) { bytes, err := yson.Marshal(attributes) if err != nil { return nil, err } - sourceType := attributes["source_type"] - if sourceType == string(LdapSourceType) { + switch sourceType { + case LdapSourceType: var ldapUser LdapUser err = yson.Unmarshal(bytes, &ldapUser) if err != nil { return nil, err } return ldapUser, nil - } else if sourceType == string(AzureSourceType) { + case AzureSourceType: var azureUser AzureUser err = yson.Unmarshal(bytes, &azureUser) if err != nil { @@ -51,12 +51,7 @@ func NewSourceUser(attributes map[string]any) (SourceUser, error) { return nil, fmt.Errorf("unknown source type: %v", sourceType) } -type BasicSourceUser struct { - SourceType SourceType `yson:"source_type"` -} - type AzureUser struct { - BasicSourceUser // PrincipalName is unique human-readable Azure user field, used (possibly with changes) // for the corresponding YTsaurus user's `name` attribute. PrincipalName string `yson:"principal_name"` @@ -81,7 +76,6 @@ func (user AzureUser) GetSourceType() SourceType { } type LdapUser struct { - BasicSourceUser Username string `yson:"username"` UID string `yson:"uid"` FirstName string `yson:"first_name"` @@ -106,25 +100,21 @@ type SourceGroup interface { GetSourceType() SourceType } -type BasicSourceGroup struct { - SourceType SourceType `yson:"source_type"` -} - -func NewSourceGroup(attributes map[string]any) (SourceGroup, error) { +func NewSourceGroup(sourceType SourceType, attributes map[string]any) (SourceGroup, error) { bytes, err := yson.Marshal(attributes) if err != nil { return nil, err } - sourceType := attributes["source_type"] - if sourceType == string(LdapSourceType) { + switch sourceType { + case LdapSourceType: var ldapGroup LdapGroup err = yson.Unmarshal(bytes, &ldapGroup) if err != nil { return nil, err } return ldapGroup, nil - } else if sourceType == string(AzureSourceType) { + case AzureSourceType: var azureGroup AzureGroup err = yson.Unmarshal(bytes, &azureGroup) if err != nil { @@ -136,7 +126,6 @@ func NewSourceGroup(attributes map[string]any) (SourceGroup, error) { } type AzureGroup struct { - BasicSourceGroup // Identity is unique human-readable Source user field, used (possibly with changes) // for the corresponding YTsaurus user's `name` attribute. Identity string `yson:"identity"` @@ -158,7 +147,6 @@ func (ag AzureGroup) GetSourceType() SourceType { } type LdapGroup struct { - BasicSourceGroup Groupname string `yson:"groupname"` } diff --git a/ytsaurus.go b/ytsaurus.go index 6cee4cd..740bcef 100644 --- a/ytsaurus.go +++ b/ytsaurus.go @@ -116,6 +116,7 @@ func (y *Ytsaurus) CreateUser(user YtsaurusUser) error { user.Username, map[string]any{ user.GetSourceAttributeName(): user.SourceUser, + "source_type": user.SourceUser.GetSourceType(), }, ) } @@ -234,6 +235,7 @@ func (y *Ytsaurus) CreateGroup(group YtsaurusGroup) error { y.client, group.Name, map[string]any{ + "source_type": group.SourceGroup.GetSourceType(), group.GetSourceAttributeName(): group.SourceGroup, }, ) diff --git a/ytsaurus_helpers.go b/ytsaurus_helpers.go index 39d565f..3b92b83 100644 --- a/ytsaurus_helpers.go +++ b/ytsaurus_helpers.go @@ -16,6 +16,7 @@ func doGetAllYtsaurusUsers(ctx context.Context, client yt.Client) ([]YtsaurusUse type YtsaurusUserResponse struct { Name string `yson:",value"` Azure *AzureUser `yson:"azure,attr"` + SourceType *SourceType `yson:"source_type,attr"` Source map[string]any `yson:"source,attr"` Banned bool `yson:"banned,attr"` BannedSince string `yson:"banned_since,attr"` @@ -31,6 +32,7 @@ func doGetAllYtsaurusUsers(ctx context.Context, client yt.Client) ([]YtsaurusUse "azure", "banned", "banned_since", + "source_type", "source", }, }, @@ -51,8 +53,8 @@ func doGetAllYtsaurusUsers(ctx context.Context, client yt.Client) ([]YtsaurusUse var sourceUser SourceUser if ytUser.Azure != nil { sourceUser = *ytUser.Azure - } else if ytUser.Source != nil { - sourceUser, err = NewSourceUser(ytUser.Source) + } else if ytUser.SourceType != nil && ytUser.Source != nil { + sourceUser, err = NewSourceUser(*ytUser.SourceType, ytUser.Source) if err != nil { return nil, errors.Wrapf(err, "failed to create source user. %v", ytUser) } @@ -69,10 +71,11 @@ func doGetAllYtsaurusUsers(ctx context.Context, client yt.Client) ([]YtsaurusUse func doGetAllYtsaurusGroupsWithMembers(ctx context.Context, client yt.Client) ([]YtsaurusGroupWithMembers, error) { type YtsaurusGroupReponse struct { - Name string `yson:",value"` - Azure *AzureGroup `yson:"azure,attr"` - Source map[string]any `yson:"source,attr"` - Members []string `yson:"members,attr"` + Name string `yson:",value"` + SourceType *SourceType `yson:"source_type,attr"` + Azure *AzureGroup `yson:"azure,attr"` + Source map[string]any `yson:"source,attr"` + Members []string `yson:"members,attr"` } var response []YtsaurusGroupReponse @@ -81,7 +84,12 @@ func doGetAllYtsaurusGroupsWithMembers(ctx context.Context, client yt.Client) ([ ypath.Path("//sys/groups"), &response, &yt.ListNodeOptions{ - Attributes: []string{"members", "azure", "source"}, + Attributes: []string{ + "members", + "azure", + "source", + "source_type", + }, }, ) if err != nil { @@ -98,8 +106,8 @@ func doGetAllYtsaurusGroupsWithMembers(ctx context.Context, client yt.Client) ([ var sourceGroup SourceGroup if ytGroup.Azure != nil { sourceGroup = *ytGroup.Azure - } else if ytGroup.Source != nil { - sourceGroup, err = NewSourceGroup(ytGroup.Source) + } else if ytGroup.SourceType != nil && ytGroup.Source != nil { + sourceGroup, err = NewSourceGroup(*ytGroup.SourceType, ytGroup.Source) if err != nil { return nil, errors.Wrapf(err, "failed to create source group. %v", ytGroup) } @@ -166,6 +174,7 @@ func buildUserAttributes(user YtsaurusUser) map[string]any { "banned_since": user.BannedSinceString(), "banned": user.IsBanned(), user.GetSourceAttributeName(): user.SourceUser, + "source_type": user.SourceUser.GetSourceType(), } } @@ -173,6 +182,7 @@ func buildGroupAttributes(group YtsaurusGroup) map[string]any { return map[string]any{ group.GetSourceAttributeName(): group.SourceGroup, "name": group.Name, + "source_type": group.SourceGroup.GetSourceType(), } } From e728afd11a7fc54f9b6b92630bc854bc05836afc Mon Sep 17 00:00:00 2001 From: Nadezhda Savchenko Date: Wed, 28 Feb 2024 12:49:11 +0300 Subject: [PATCH 09/21] Review fixes --- app.go | 14 +++++++------- app_test.go | 14 ++++++-------- azure_real.go | 12 ++++++------ config.go | 6 +++--- config_test.go | 6 +++--- diff.go | 20 +++++++++----------- ldap.go | 30 +++++++++++++++--------------- 7 files changed, 49 insertions(+), 53 deletions(-) diff --git a/app.go b/app.go index f1dbc37..6a0f6bf 100644 --- a/app.go +++ b/app.go @@ -12,11 +12,11 @@ import ( ) type App struct { - syncInterval *time.Duration + syncInterval time.Duration usernameReplaces []ReplacementPair groupnameReplaces []ReplacementPair - removeLimit *int - banDuration *time.Duration + removeLimit int + banDuration time.Duration ytsaurus *Ytsaurus source Source @@ -27,8 +27,8 @@ type App struct { } func NewApp(cfg *Config, logger appLoggerType) (*App, error) { - if cfg.Azure == nil && cfg.Ldap == nil { - return nil, errors.New("no source (source or ldap) is specified") + if (cfg.Azure == nil) == (cfg.Ldap == nil) { + return nil, errors.New("one and only one source should be specified") } var err error @@ -78,8 +78,8 @@ func NewAppCustomized(cfg *Config, logger appLoggerType, source Source, clock cl func (a *App) Start() { a.logger.Info("Starting the application") - if a.syncInterval != nil && *a.syncInterval > 0 { - ticker := time.NewTicker(*a.syncInterval) + if a.syncInterval > 0 { + ticker := time.NewTicker(a.syncInterval) for { select { case <-a.stopCh: diff --git a/app_test.go b/app_test.go index a8326b7..c8b7b41 100644 --- a/app_test.go +++ b/app_test.go @@ -10,8 +10,6 @@ import ( "github.com/go-ldap/ldap/v3" - "go.ytsaurus.tech/library/go/ptr" - "github.com/google/go-cmp/cmp" "github.com/google/go-cmp/cmp/cmpopts" "github.com/stretchr/testify/require" @@ -195,7 +193,7 @@ func getTestCases(sourceType SourceType) []testCase { appConfig: &AppConfig{ UsernameReplacements: defaultUsernameReplacements, GroupnameReplacements: defaultGroupnameReplacements, - BanBeforeRemoveDuration: ptr.Duration(24 * time.Hour), + BanBeforeRemoveDuration: 24 * time.Hour, }, sourceUsersSetUp: []SourceUser{ getSourceUser(aliceName, sourceType), @@ -219,7 +217,7 @@ func getTestCases(sourceType SourceType) []testCase { appConfig: &AppConfig{ UsernameReplacements: defaultUsernameReplacements, GroupnameReplacements: defaultGroupnameReplacements, - BanBeforeRemoveDuration: ptr.Duration(24 * time.Hour), + BanBeforeRemoveDuration: 24 * time.Hour, }, sourceUsersSetUp: []SourceUser{ getSourceUser(aliceName, sourceType), @@ -241,7 +239,7 @@ func getTestCases(sourceType SourceType) []testCase { appConfig: &AppConfig{ UsernameReplacements: defaultUsernameReplacements, GroupnameReplacements: defaultGroupnameReplacements, - RemoveLimit: ptr.Int(3), + RemoveLimit: 3, }, sourceUsersSetUp: []SourceUser{}, ytUsersSetUp: []YtsaurusUser{ @@ -262,7 +260,7 @@ func getTestCases(sourceType SourceType) []testCase { appConfig: &AppConfig{ UsernameReplacements: defaultUsernameReplacements, GroupnameReplacements: defaultGroupnameReplacements, - RemoveLimit: ptr.Int(3), + RemoveLimit: 3, }, sourceGroupsSetUp: []SourceGroupWithMembers{}, ytGroupsSetUp: []YtsaurusGroupWithMembers{ @@ -483,8 +481,8 @@ var ( } ) -// TestAppSync uses local YTsaurus container and fake Source to test all the cases: -// [x] If Source user not in YTsaurus -> created; +// TestAppSync uses local YTsaurus container and some source (Azure or Ldap) to test all the cases: +// [x] If azur user not in YTsaurus -> created; // [x] If Source user already in YTsaurus no changes -> skipped; // [x] If Source user already in YTsaurus with changes -> updated; // [x] If user in YTsaurus but not in Source (and ban_before_remove_duration=0) -> removed; diff --git a/azure_real.go b/azure_real.go index d25976d..28fd89e 100644 --- a/azure_real.go +++ b/azure_real.go @@ -67,7 +67,7 @@ func NewAzureReal(cfg *AzureConfig, logger appLoggerType) (*AzureReal, error) { nil, ) if err != nil { - return nil, errors.Wrap(err, "failed to create Source secret credentials") + return nil, errors.Wrap(err, "failed to create Azure secret credentials") } graphClient, err := msgraphsdk.NewGraphServiceClientWithCredentials(cred, []string{scope}) @@ -146,7 +146,7 @@ func (a *AzureReal) GetUsers() ([]SourceUser, error) { } } - a.logger.Infow("Fetched users from Source AD", "got", len(usersRaw), "skipped", usersSkipped) + a.logger.Infow("Fetched users from Azure AD", "got", len(usersRaw), "skipped", usersSkipped) return users, nil } @@ -208,7 +208,7 @@ func (a *AzureReal) GetGroupsWithMembers() ([]SourceGroupWithMembers, error) { }) } - a.logger.Infow("Fetched groups from Source AD", "got", len(groupsRaw), "skipped", groupsSkipped) + a.logger.Infow("Fetched groups from Azure AD", "got", len(groupsRaw), "skipped", groupsSkipped) return groups, nil } @@ -258,7 +258,7 @@ func (a *AzureReal) getUsersRaw(ctx context.Context, fieldsToSelect []string, fi return true }) if err != nil { - return nil, errors.Wrap(err, "failed to iterate over Source users") + return nil, errors.Wrap(err, "failed to iterate over Azure users") } return rawUsers, nil } @@ -299,7 +299,7 @@ func (a *AzureReal) getGroupsWithMembersRaw(ctx context.Context, fieldsToSelect return true }) if err != nil { - return nil, errors.Wrap(err, "failed to iterate over Source groups") + return nil, errors.Wrap(err, "failed to iterate over Azure groups") } return rawGroups, nil } @@ -337,7 +337,7 @@ func (a *AzureReal) getGroupMembers(ctx context.Context, groupID string) ([]mode return true }) if err != nil { - return nil, errors.Wrap(err, "failed to iterate over Source group members") + return nil, errors.Wrap(err, "failed to iterate over Azure group members") } return rawMembers, nil diff --git a/config.go b/config.go index f1a42be..8d5baac 100644 --- a/config.go +++ b/config.go @@ -17,7 +17,7 @@ type Config struct { type AppConfig struct { // SyncInterval is the interval between full synchronizations. // If it is not speciied or value is zero than auto-sync disabled (sync can be invoked only manually). - SyncInterval *time.Duration `yaml:"sync_interval"` + SyncInterval time.Duration `yaml:"sync_interval"` // UsernameReplacements is a list of replaces which will be applied to a username for source (Source or Ldap). // For example, you may use it to strip off characters like @ which are not recommended for use @@ -28,11 +28,11 @@ type AppConfig struct { // If count users or groups for planned delete in on sync cycle reaches RemoveLimit // app will fail that sync cycle. // No limit if it is not specified. - RemoveLimit *int `yaml:"remove_limit,omitempty"` + RemoveLimit int `yaml:"remove_limit,omitempty"` // BanBeforeRemoveDuration is a duration of a graceful ban before finally removing the user from YTsaurus. // If it is not specified, user will be removed straight after user was found to be missing from source (Source or Ldap). - BanBeforeRemoveDuration *time.Duration `yaml:"ban_before_remove_duration"` + BanBeforeRemoveDuration time.Duration `yaml:"ban_before_remove_duration"` } type ReplacementPair struct { diff --git a/config_test.go b/config_test.go index b1dbd53..7c01e8e 100644 --- a/config_test.go +++ b/config_test.go @@ -19,7 +19,7 @@ func TestAzureConfig(t *testing.T) { cfg, err := loadConfig(configPath) require.NoError(t, err) - require.Equal(t, ptr.Duration(5*time.Minute), cfg.App.SyncInterval) + require.Equal(t, 5*time.Minute, cfg.App.SyncInterval) require.Equal(t, []ReplacementPair{ {From: "@acme.com", To: ""}, {From: "@", To: ":"}, @@ -27,8 +27,8 @@ func TestAzureConfig(t *testing.T) { require.Equal(t, []ReplacementPair{ {From: "|all", To: ""}, }, cfg.App.GroupnameReplacements) - require.Equal(t, ptr.Int(10), cfg.App.RemoveLimit) - require.Equal(t, ptr.Duration(7*24*time.Hour), cfg.App.BanBeforeRemoveDuration) + require.Equal(t, 10, cfg.App.RemoveLimit) + require.Equal(t, 7*24*time.Hour, cfg.App.BanBeforeRemoveDuration) require.True(t, cfg.Ldap == nil) diff --git a/diff.go b/diff.go index e39b87f..84a7dfa 100644 --- a/diff.go +++ b/diff.go @@ -25,10 +25,10 @@ func (a *App) syncOnce() { } func (a *App) isRemoveLimitReached(objectsCount int) bool { - if a.removeLimit == nil || *a.removeLimit <= 0 { + if a.removeLimit <= 0 { return false } - return objectsCount >= *a.removeLimit + return objectsCount >= a.removeLimit } // syncUsers syncs AD users with YTsaurus cluster and returns /actual/ map[ObjectID]YtsaurusUser @@ -38,11 +38,9 @@ func (a *App) syncUsers() (map[ObjectID]YtsaurusUser, error) { var err error var sourceUsers []SourceUser - if a.source != nil { - sourceUsers, err = a.source.GetUsers() - if err != nil { - return nil, errors.Wrap(err, "failed to get Source users") - } + sourceUsers, err = a.source.GetUsers() + if err != nil { + return nil, errors.Wrap(err, "failed to get Source users") } ytUsers, err := a.ytsaurus.GetUsers(a.source.GetSourceType()) @@ -228,7 +226,7 @@ func (a *App) diffGroups( } for objectID, ytGroupWithMembers := range ytGroupsWithMembersMap { - // Collecting groups to remove (the ones that exist in YTsaurus and not in Source). + // Collecting groups to remove (the ones that exist in YTsaurus and not in Azure). sourceGroupWithMembers, ok := sourceGroupsWithMembersMap[objectID] if !ok { groupsToRemove = append(groupsToRemove, ytGroupWithMembers.YtsaurusGroup) @@ -406,15 +404,15 @@ func (a *App) isGroupMembersChanged(sourceGroup SourceGroupWithMembers, ytGroup func (a *App) banOrRemoveUser(user YtsaurusUser) (wasBanned, wasRemoved bool, err error) { // Ban settings is disabled. - if a.banDuration == nil { + if a.banDuration == 0 { return false, true, a.ytsaurus.RemoveUser(user.Username) } // If user is not already banned we should do it. - if !user.IsBanned() && *a.banDuration != 0 { + if !user.IsBanned() { return true, false, a.ytsaurus.BanUser(user.Username) } // If user was banned longer than setting permits, we remove it. - if user.IsBanned() && time.Since(user.BannedSince) > *a.banDuration { + if user.IsBanned() && time.Since(user.BannedSince) > a.banDuration { return false, true, a.ytsaurus.RemoveUser(user.Username) } a.logger.Debugw("user is banned, but not yet removed", "user", user.Username, "since", user.BannedSince) diff --git a/ldap.go b/ldap.go index 432d3c5..0c9f736 100644 --- a/ldap.go +++ b/ldap.go @@ -33,14 +33,14 @@ func NewLdap(cfg *LdapConfig, logger appLoggerType) (*Ldap, error) { }, nil } -func (source *Ldap) GetSourceType() SourceType { +func (l *Ldap) GetSourceType() SourceType { return LdapSourceType } -func (source *Ldap) GetUsers() ([]SourceUser, error) { - res, err := source.Connection.Search(&ldap.SearchRequest{ - BaseDN: source.Config.BaseDN, - Filter: source.Config.Users.Filter, +func (l *Ldap) GetUsers() ([]SourceUser, error) { + res, err := l.Connection.Search(&ldap.SearchRequest{ + BaseDN: l.Config.BaseDN, + Filter: l.Config.Users.Filter, Attributes: []string{"*"}, Scope: ldap.ScopeWholeSubtree, }) @@ -50,11 +50,11 @@ func (source *Ldap) GetUsers() ([]SourceUser, error) { var users []SourceUser for _, entry := range res.Entries { - username := entry.GetAttributeValue(source.Config.Users.UsernameAttributeType) - uid := entry.GetAttributeValue(source.Config.Users.UIDAttributeType) + username := entry.GetAttributeValue(l.Config.Users.UsernameAttributeType) + uid := entry.GetAttributeValue(l.Config.Users.UIDAttributeType) var firstName string - if source.Config.Users.FirstNameAttributeType != nil { - firstName = entry.GetAttributeValue(*source.Config.Users.FirstNameAttributeType) + if l.Config.Users.FirstNameAttributeType != nil { + firstName = entry.GetAttributeValue(*l.Config.Users.FirstNameAttributeType) } users = append(users, LdapUser{ Username: username, @@ -64,10 +64,10 @@ func (source *Ldap) GetUsers() ([]SourceUser, error) { return users, nil } -func (source *Ldap) GetGroupsWithMembers() ([]SourceGroupWithMembers, error) { - res, err := source.Connection.Search(&ldap.SearchRequest{ - BaseDN: source.Config.BaseDN, - Filter: source.Config.Groups.Filter, +func (l *Ldap) GetGroupsWithMembers() ([]SourceGroupWithMembers, error) { + res, err := l.Connection.Search(&ldap.SearchRequest{ + BaseDN: l.Config.BaseDN, + Filter: l.Config.Groups.Filter, Attributes: []string{"*"}, Scope: ldap.ScopeWholeSubtree, }) @@ -77,8 +77,8 @@ func (source *Ldap) GetGroupsWithMembers() ([]SourceGroupWithMembers, error) { var groups []SourceGroupWithMembers for _, entry := range res.Entries { - groupname := entry.GetAttributeValue(source.Config.Groups.GroupnameAttributeType) - members := entry.GetAttributeValues(source.Config.Groups.MemberUIDAttributeType) + groupname := entry.GetAttributeValue(l.Config.Groups.GroupnameAttributeType) + members := entry.GetAttributeValues(l.Config.Groups.MemberUIDAttributeType) groups = append(groups, SourceGroupWithMembers{ SourceGroup: LdapGroup{ Groupname: groupname, From 6fbdcaf9bb32363f366c0222aa0ebac3e6779434 Mon Sep 17 00:00:00 2001 From: Nadezhda Savchenko Date: Wed, 28 Feb 2024 17:23:49 +0300 Subject: [PATCH 10/21] Refactoring only, without ldap related changes --- app.go | 19 +- app_test.go | 912 +++++++++++++++++--------------------- config.go | 48 +- config_test.go | 54 +-- go.mod | 6 +- go.sum | 39 -- ldap.go | 90 ---- ldap_config.example.yaml | 39 -- source.go | 50 --- testcontainer_openldap.go | 66 --- ytsaurus.go | 9 +- ytsaurus_helpers.go | 42 +- ytsaurus_models.go | 4 - 13 files changed, 451 insertions(+), 927 deletions(-) delete mode 100644 ldap.go delete mode 100644 ldap_config.example.yaml delete mode 100644 testcontainer_openldap.go diff --git a/app.go b/app.go index 6a0f6bf..7736082 100644 --- a/app.go +++ b/app.go @@ -27,24 +27,13 @@ type App struct { } func NewApp(cfg *Config, logger appLoggerType) (*App, error) { - if (cfg.Azure == nil) == (cfg.Ldap == nil) { + if cfg.Azure == nil { return nil, errors.New("one and only one source should be specified") } - var err error - var source Source - if cfg.Azure != nil { - source, err = NewAzureReal(cfg.Azure, logger) - if err != nil { - return nil, err - } - } - - if cfg.Ldap != nil { - source, err = NewLdap(cfg.Ldap, logger) - if err != nil { - return nil, err - } + source, err := NewAzureReal(cfg.Azure, logger) + if err != nil { + return nil, err } return NewAppCustomized(cfg, logger, source, clock.RealClock{}) diff --git a/app_test.go b/app_test.go index c8b7b41..bcbebd0 100644 --- a/app_test.go +++ b/app_test.go @@ -4,12 +4,9 @@ import ( "context" "fmt" "os" - "strings" "testing" "time" - "github.com/go-ldap/ldap/v3" - "github.com/google/go-cmp/cmp" "github.com/google/go-cmp/cmp/cmpopts" "github.com/stretchr/testify/require" @@ -20,9 +17,6 @@ import ( const ( ytDevToken = "password" - aliceName = "alice" - bobName = "bob" - carolName = "carol" ) type testCase struct { @@ -30,571 +24,586 @@ type testCase struct { appConfig *AppConfig testTime time.Time - sourceType SourceType - - sourceUsersSetUp []SourceUser - ytUsersSetUp []YtsaurusUser - ytUsersExpected []YtsaurusUser + azureUsersSetUp []SourceUser + ytUsersSetUp []YtsaurusUser + ytUsersExpected []YtsaurusUser - sourceGroupsSetUp []SourceGroupWithMembers - ytGroupsSetUp []YtsaurusGroupWithMembers - ytGroupsExpected []YtsaurusGroupWithMembers + azureGroupsSetUp []SourceGroupWithMembers + ytGroupsSetUp []YtsaurusGroupWithMembers + ytGroupsExpected []YtsaurusGroupWithMembers } -func getUserID(name string) string { - switch name { - case aliceName: - return "1" - case bobName: - return "2" - case carolName: - return "3" - } - return "4" -} +var ( + testTimeStr = "2023-10-20T12:00:00Z" + initialTestTime = parseAppTime(testTimeStr) -func getSourceUser(name string, sourceType SourceType) SourceUser { - switch sourceType { - case LdapSourceType: - return LdapUser{ - Username: fmt.Sprintf("%v@acme.com", name), - UID: getUserID(name), - FirstName: fmt.Sprintf("%v@acme.com-firstname", name), - } - case AzureSourceType: - return AzureUser{ - PrincipalName: fmt.Sprintf("%v@acme.com", name), - AzureID: fmt.Sprintf("fake-az-id-%v", name), - Email: fmt.Sprintf("%v@acme.com", name), - FirstName: fmt.Sprintf("%v@acme.com-firstname", name), - LastName: fmt.Sprintf("%v-lastname", name), - DisplayName: fmt.Sprintf("Henderson, %v (ACME)", name), - } + aliceAzure = AzureUser{ + PrincipalName: "alice@acme.com", + AzureID: "fake-az-id-alice", + Email: "alice@acme.com", + FirstName: "Alice", + LastName: "Henderson", + DisplayName: "Henderson, Alice (ACME)", } - return nil -} - -func getUpdatedSourceUser(name string, sourceType SourceType) SourceUser { - sourceUser := getSourceUser(name, sourceType) - switch sourceType { - case LdapSourceType: - ldapSourceUser := sourceUser.(LdapUser) - return LdapUser{ - Username: ldapSourceUser.Username, - UID: ldapSourceUser.UID, - FirstName: ldapSourceUser.FirstName + "-updated", - } - case AzureSourceType: - azureSourceUser := sourceUser.(AzureUser) - return AzureUser{ - PrincipalName: azureSourceUser.PrincipalName, - AzureID: azureSourceUser.AzureID, - Email: azureSourceUser.Email + "-updated", - FirstName: azureSourceUser.FirstName, - LastName: azureSourceUser.LastName, - DisplayName: azureSourceUser.DisplayName, - } + bobAzure = AzureUser{ + PrincipalName: "Bob@acme.com", + AzureID: "fake-az-id-bob", + Email: "Bob@acme.com", + FirstName: "Bob", + LastName: "Sanders", + DisplayName: "Sanders, Bob (ACME)", } - return nil -} - -func getYtsaurusUser(sourceUser SourceUser) YtsaurusUser { - name := sourceUser.GetName() - for _, replacement := range defaultUsernameReplacements { - name = strings.Replace(name, replacement.From, replacement.To, -1) + carolAzure = AzureUser{ + PrincipalName: "carol@acme.com", + AzureID: "fake-az-id-carol", + Email: "carol@acme.com", + FirstName: "Carol", + LastName: "Sanders", + DisplayName: "Sanders, Carol (ACME)", } - return YtsaurusUser{Username: name, SourceUser: sourceUser} -} - -func bannedYtsaurusUser(ytUser YtsaurusUser, bannedSince time.Time) YtsaurusUser { - return YtsaurusUser{Username: ytUser.Username, SourceUser: ytUser.SourceUser, BannedSince: bannedSince} -} - -func getSourceGroup(name string, sourceType SourceType) SourceGroup { - switch sourceType { - case AzureSourceType: - return AzureGroup{ - Identity: fmt.Sprintf("acme.%v|all", name), - AzureID: fmt.Sprintf("fake-az-acme.%v", name), - DisplayName: fmt.Sprintf("acme.%v|all", name), - } - case LdapSourceType: - return LdapGroup{ - Groupname: fmt.Sprintf("acme.%v|all", name), - } + aliceAzureChangedLastName = AzureUser{ + PrincipalName: aliceAzure.PrincipalName, + AzureID: aliceAzure.AzureID, + Email: aliceAzure.Email, + FirstName: aliceAzure.FirstName, + LastName: "Smith", + DisplayName: aliceAzure.DisplayName, } - return nil -} - -func getUpdatedSourceGroup(name string, sourceType SourceType) SourceGroup { - sourceGroup := getSourceGroup(name, sourceType) - switch sourceType { - case LdapSourceType: - // TODO(nadya73): add more fields. - ldapSourceGroup := sourceGroup.(LdapGroup) - return LdapGroup{ - Groupname: ldapSourceGroup.Groupname, - } - case AzureSourceType: - azureSourceGroup := sourceGroup.(AzureGroup) - return AzureGroup{ - Identity: azureSourceGroup.Identity, - AzureID: azureSourceGroup.AzureID, - DisplayName: azureSourceGroup.DisplayName + "-updated", - } + bobAzureChangedEmail = AzureUser{ + PrincipalName: "bobby@example.com", + AzureID: bobAzure.AzureID, + Email: "bobby@example.com", + FirstName: bobAzure.FirstName, + LastName: bobAzure.LastName, + DisplayName: bobAzure.DisplayName, + } + devsAzureGroup = AzureGroup{ + Identity: "acme.devs|all", + AzureID: "fake-az-acme.devs", + DisplayName: "acme.devs|all", + } + hqAzureGroup = AzureGroup{ + Identity: "acme.hq", + AzureID: "fake-az-acme.hq", + DisplayName: "acme.hq", + } + devsAzureGroupChangedDisplayName = AzureGroup{ + Identity: "acme.developers|all", + AzureID: devsAzureGroup.AzureID, + DisplayName: "acme.developers|all", + } + hqAzureGroupChangedBackwardCompatible = AzureGroup{ + Identity: "acme.hq|all", + AzureID: hqAzureGroup.AzureID, + DisplayName: "acme.hq|all", } - return nil -} -func getChangedBackwardCompatibleSourceGroup(name string, sourceType SourceType) SourceGroup { - if sourceType != AzureSourceType { - return nil + aliceYtsaurus = YtsaurusUser{ + Username: "alice", + SourceUser: AzureUser{ + AzureID: aliceAzure.AzureID, + PrincipalName: aliceAzure.PrincipalName, + Email: aliceAzure.Email, + FirstName: aliceAzure.FirstName, + LastName: aliceAzure.LastName, + DisplayName: aliceAzure.DisplayName, + }, } - sourceGroup := getSourceGroup(name, sourceType) - azureSourceGroup := sourceGroup.(AzureGroup) - return AzureGroup{ - Identity: azureSourceGroup.Identity + "-changed", - AzureID: azureSourceGroup.AzureID, - DisplayName: azureSourceGroup.DisplayName + "-updated", + bobYtsaurus = YtsaurusUser{ + Username: "bob", + SourceUser: AzureUser{ + AzureID: bobAzure.AzureID, + PrincipalName: bobAzure.PrincipalName, + Email: bobAzure.Email, + FirstName: bobAzure.FirstName, + LastName: bobAzure.LastName, + DisplayName: bobAzure.DisplayName, + }, + } + carolYtsaurus = YtsaurusUser{ + Username: "carol", + SourceUser: AzureUser{ + AzureID: carolAzure.AzureID, + PrincipalName: carolAzure.PrincipalName, + Email: carolAzure.Email, + FirstName: carolAzure.FirstName, + LastName: carolAzure.LastName, + DisplayName: carolAzure.DisplayName, + }, + } + aliceYtsaurusChangedLastName = YtsaurusUser{ + Username: aliceYtsaurus.Username, + SourceUser: AzureUser{ + AzureID: aliceYtsaurus.SourceUser.(AzureUser).AzureID, + PrincipalName: aliceYtsaurus.SourceUser.(AzureUser).PrincipalName, + Email: aliceYtsaurus.SourceUser.(AzureUser).Email, + FirstName: aliceYtsaurus.SourceUser.(AzureUser).FirstName, + LastName: aliceAzureChangedLastName.LastName, + DisplayName: aliceYtsaurus.SourceUser.(AzureUser).DisplayName, + }, + } + bobYtsaurusChangedEmail = YtsaurusUser{ + Username: "bobby:example.com", + SourceUser: AzureUser{ + AzureID: bobYtsaurus.SourceUser.(AzureUser).AzureID, + PrincipalName: bobAzureChangedEmail.PrincipalName, + Email: bobAzureChangedEmail.Email, + FirstName: bobYtsaurus.SourceUser.(AzureUser).FirstName, + LastName: bobYtsaurus.SourceUser.(AzureUser).LastName, + DisplayName: bobYtsaurus.SourceUser.(AzureUser).DisplayName, + }, + } + bobYtsaurusBanned = YtsaurusUser{ + Username: bobYtsaurus.Username, + SourceUser: AzureUser{ + AzureID: bobYtsaurus.SourceUser.(AzureUser).AzureID, + PrincipalName: bobYtsaurus.SourceUser.(AzureUser).PrincipalName, + Email: bobYtsaurus.SourceUser.(AzureUser).Email, + FirstName: bobYtsaurus.SourceUser.(AzureUser).FirstName, + LastName: bobYtsaurus.SourceUser.(AzureUser).LastName, + DisplayName: bobYtsaurus.SourceUser.(AzureUser).DisplayName, + }, + BannedSince: initialTestTime, + } + carolYtsaurusBanned = YtsaurusUser{ + Username: carolYtsaurus.Username, + SourceUser: AzureUser{ + AzureID: carolYtsaurus.SourceUser.(AzureUser).AzureID, + PrincipalName: carolYtsaurus.SourceUser.(AzureUser).PrincipalName, + Email: carolYtsaurus.SourceUser.(AzureUser).Email, + FirstName: carolYtsaurus.SourceUser.(AzureUser).FirstName, + LastName: carolYtsaurus.SourceUser.(AzureUser).LastName, + DisplayName: carolYtsaurus.SourceUser.(AzureUser).DisplayName, + }, + BannedSince: initialTestTime.Add(40 * time.Hour), + } + devsYtsaurusGroup = YtsaurusGroup{ + Name: "acme.devs", + SourceGroup: AzureGroup{ + AzureID: devsAzureGroup.AzureID, + DisplayName: "acme.devs|all", + Identity: "acme.devs|all", + }, + } + qaYtsaurusGroup = YtsaurusGroup{ + Name: "acme.qa", + SourceGroup: AzureGroup{ + AzureID: "fake-az-acme.qa", + DisplayName: "acme.qa|all", + Identity: "acme.qa", + }, + } + hqYtsaurusGroup = YtsaurusGroup{ + Name: "acme.hq", + SourceGroup: AzureGroup{ + AzureID: hqAzureGroup.AzureID, + DisplayName: "acme.hq", + Identity: "acme.hq", + }, + } + devsYtsaurusGroupChangedDisplayName = YtsaurusGroup{ + Name: "acme.developers", + SourceGroup: AzureGroup{ + AzureID: devsAzureGroup.AzureID, + DisplayName: "acme.developers|all", + Identity: "acme.developers|all", + }, + } + hqYtsaurusGroupChangedBackwardCompatible = YtsaurusGroup{ + Name: "acme.hq", + SourceGroup: AzureGroup{ + AzureID: hqAzureGroup.AzureID, + DisplayName: "acme.hq|all", + Identity: "acme.hq|all", + }, } -} -func getYtsaurusGroup(sourceGroup SourceGroup) YtsaurusGroup { - name := sourceGroup.GetName() - for _, replacement := range defaultGroupnameReplacements { - name = strings.Replace(name, replacement.From, replacement.To, -1) + defaultUsernameReplacements = []ReplacementPair{ + {"@acme.com", ""}, + {"@", ":"}, + } + defaultGroupnameReplacements = []ReplacementPair{ + {"|all", ""}, + } + defaultAppConfig = &AppConfig{ + UsernameReplacements: defaultUsernameReplacements, + GroupnameReplacements: defaultGroupnameReplacements, } - return YtsaurusGroup{Name: name, SourceGroup: sourceGroup} -} -// We test several things in each test case, because of long wait for local ytsaurus -// container start. -func getTestCases(sourceType SourceType) []testCase { - testCases := []testCase{ + // we test several things in each test case, because of long wait for local ytsaurus + // container start. + testCases = []testCase{ { - name: "a-skip-b-create-c-remove", - sourceType: sourceType, - sourceUsersSetUp: []SourceUser{ - getSourceUser(aliceName, sourceType), - getSourceUser(bobName, sourceType), + name: "a-skip-b-create-c-remove", + azureUsersSetUp: []SourceUser{ + aliceAzure, + bobAzure, }, ytUsersSetUp: []YtsaurusUser{ - getYtsaurusUser(getSourceUser(aliceName, sourceType)), - getYtsaurusUser(getSourceUser(carolName, sourceType)), + aliceYtsaurus, + carolYtsaurus, }, ytUsersExpected: []YtsaurusUser{ - getYtsaurusUser(getSourceUser(aliceName, sourceType)), - getYtsaurusUser(getSourceUser(bobName, sourceType)), + aliceYtsaurus, + bobYtsaurus, }, }, { - name: "bob-is-banned", - sourceType: sourceType, + name: "bob-is-banned", appConfig: &AppConfig{ UsernameReplacements: defaultUsernameReplacements, GroupnameReplacements: defaultGroupnameReplacements, BanBeforeRemoveDuration: 24 * time.Hour, }, - sourceUsersSetUp: []SourceUser{ - getSourceUser(aliceName, sourceType), - }, ytUsersSetUp: []YtsaurusUser{ - getYtsaurusUser(getSourceUser(aliceName, sourceType)), - getYtsaurusUser(getSourceUser(bobName, sourceType)), + aliceYtsaurus, + bobYtsaurus, + }, + azureUsersSetUp: []SourceUser{ + aliceAzure, }, ytUsersExpected: []YtsaurusUser{ - getYtsaurusUser(getSourceUser(aliceName, sourceType)), - bannedYtsaurusUser(getYtsaurusUser(getSourceUser(bobName, sourceType)), initialTestTime), + aliceYtsaurus, + bobYtsaurusBanned, }, }, { - name: "bob-was-banned-now-deleted-carol-was-banned-now-back", - sourceType: sourceType, + name: "bob-was-banned-now-deleted-carol-was-banned-now-back", // Bob was banned at initialTestTime, // 2 days have passed (more than setting allows) —> he should be removed. - // Carol was banned 8 hours ago and has been found in Source -> she should be restored. + // Carol was banned 8 hours ago and has been found in Azure -> she should be restored. testTime: initialTestTime.Add(48 * time.Hour), appConfig: &AppConfig{ UsernameReplacements: defaultUsernameReplacements, GroupnameReplacements: defaultGroupnameReplacements, BanBeforeRemoveDuration: 24 * time.Hour, }, - sourceUsersSetUp: []SourceUser{ - getSourceUser(aliceName, sourceType), - getSourceUser(carolName, sourceType), - }, ytUsersSetUp: []YtsaurusUser{ - getYtsaurusUser(getSourceUser(aliceName, sourceType)), - bannedYtsaurusUser(getYtsaurusUser(getSourceUser(bobName, sourceType)), initialTestTime), - bannedYtsaurusUser(getYtsaurusUser(getSourceUser(carolName, sourceType)), initialTestTime.Add(40*time.Hour)), + aliceYtsaurus, + bobYtsaurusBanned, + carolYtsaurusBanned, + }, + azureUsersSetUp: []SourceUser{ + aliceAzure, + carolAzure, }, ytUsersExpected: []YtsaurusUser{ - getYtsaurusUser(getSourceUser(aliceName, sourceType)), - getYtsaurusUser(getSourceUser(carolName, sourceType)), + aliceYtsaurus, + carolYtsaurus, }, }, { - name: "remove-limit-users-3", - sourceType: sourceType, + name: "remove-limit-users-3", appConfig: &AppConfig{ UsernameReplacements: defaultUsernameReplacements, GroupnameReplacements: defaultGroupnameReplacements, RemoveLimit: 3, }, - sourceUsersSetUp: []SourceUser{}, + azureUsersSetUp: []SourceUser{}, ytUsersSetUp: []YtsaurusUser{ - getYtsaurusUser(getSourceUser(aliceName, sourceType)), - getYtsaurusUser(getSourceUser(bobName, sourceType)), - getYtsaurusUser(getSourceUser(carolName, sourceType)), + aliceYtsaurus, + bobYtsaurus, + carolYtsaurus, }, // no one is deleted: limitation works ytUsersExpected: []YtsaurusUser{ - getYtsaurusUser(getSourceUser(aliceName, sourceType)), - getYtsaurusUser(getSourceUser(bobName, sourceType)), - getYtsaurusUser(getSourceUser(carolName, sourceType)), + aliceYtsaurus, + bobYtsaurus, + carolYtsaurus, }, }, { - name: "remove-limit-groups-3", - sourceType: sourceType, + name: "remove-limit-groups-3", appConfig: &AppConfig{ UsernameReplacements: defaultUsernameReplacements, GroupnameReplacements: defaultGroupnameReplacements, RemoveLimit: 3, }, - sourceGroupsSetUp: []SourceGroupWithMembers{}, + azureGroupsSetUp: []SourceGroupWithMembers{}, ytGroupsSetUp: []YtsaurusGroupWithMembers{ - NewEmptyYtsaurusGroupWithMembers(getYtsaurusGroup(getSourceGroup("dev", sourceType))), - NewEmptyYtsaurusGroupWithMembers(getYtsaurusGroup(getSourceGroup("qa", sourceType))), - NewEmptyYtsaurusGroupWithMembers(getYtsaurusGroup(getSourceGroup("hq", sourceType))), + NewEmptyYtsaurusGroupWithMembers(devsYtsaurusGroup), + NewEmptyYtsaurusGroupWithMembers(qaYtsaurusGroup), + NewEmptyYtsaurusGroupWithMembers(hqYtsaurusGroup), }, // no group is deleted: limitation works ytGroupsExpected: []YtsaurusGroupWithMembers{ - NewEmptyYtsaurusGroupWithMembers(getYtsaurusGroup(getSourceGroup("dev", sourceType))), - NewEmptyYtsaurusGroupWithMembers(getYtsaurusGroup(getSourceGroup("qa", sourceType))), - NewEmptyYtsaurusGroupWithMembers(getYtsaurusGroup(getSourceGroup("hq", sourceType))), + NewEmptyYtsaurusGroupWithMembers(devsYtsaurusGroup), + NewEmptyYtsaurusGroupWithMembers(qaYtsaurusGroup), + NewEmptyYtsaurusGroupWithMembers(hqYtsaurusGroup), }, }, { - name: "a-changed-name-b-changed-email", - sourceType: sourceType, - sourceUsersSetUp: []SourceUser{ - getUpdatedSourceUser(aliceName, sourceType), - getUpdatedSourceUser(bobName, sourceType), + name: "a-changed-name-b-changed-email", + azureUsersSetUp: []SourceUser{ + aliceAzureChangedLastName, + bobAzureChangedEmail, }, ytUsersSetUp: []YtsaurusUser{ - getYtsaurusUser(getSourceUser(aliceName, sourceType)), - getYtsaurusUser(getSourceUser(bobName, sourceType)), + aliceYtsaurus, + bobYtsaurus, }, ytUsersExpected: []YtsaurusUser{ - getYtsaurusUser(getUpdatedSourceUser(aliceName, sourceType)), - getYtsaurusUser(getUpdatedSourceUser(bobName, sourceType)), + aliceYtsaurusChangedLastName, + bobYtsaurusChangedEmail, }, }, { - name: "skip-create-remove-group-no-members-change-correct-name-replace", - sourceType: sourceType, - sourceUsersSetUp: []SourceUser{ - getSourceUser(aliceName, sourceType), - getSourceUser(bobName, sourceType), - getSourceUser(carolName, sourceType), + name: "skip-create-remove-group-no-members-change-correct-name-replace", + azureUsersSetUp: []SourceUser{ + aliceAzure, + bobAzure, + carolAzure, }, ytUsersSetUp: []YtsaurusUser{ - getYtsaurusUser(getSourceUser(aliceName, sourceType)), - getYtsaurusUser(getSourceUser(bobName, sourceType)), - getYtsaurusUser(getSourceUser(carolName, sourceType)), + aliceYtsaurus, + bobYtsaurus, + carolYtsaurus, }, ytUsersExpected: []YtsaurusUser{ - getYtsaurusUser(getSourceUser(aliceName, sourceType)), - getYtsaurusUser(getSourceUser(bobName, sourceType)), - getYtsaurusUser(getSourceUser(carolName, sourceType)), + aliceYtsaurus, + bobYtsaurus, + carolYtsaurus, }, ytGroupsSetUp: []YtsaurusGroupWithMembers{ { - YtsaurusGroup: getYtsaurusGroup(getSourceGroup("devs", sourceType)), - Members: NewStringSetFromItems(aliceName), + YtsaurusGroup: devsYtsaurusGroup, + Members: NewStringSetFromItems(aliceYtsaurus.Username), }, { - YtsaurusGroup: getYtsaurusGroup(getSourceGroup("qa", sourceType)), - Members: NewStringSetFromItems(bobName), + YtsaurusGroup: qaYtsaurusGroup, + Members: NewStringSetFromItems(bobYtsaurus.Username), }, }, - sourceGroupsSetUp: []SourceGroupWithMembers{ + azureGroupsSetUp: []SourceGroupWithMembers{ { - SourceGroup: getSourceGroup("devs", sourceType), - Members: NewStringSetFromItems(getSourceUser(aliceName, sourceType).GetID()), + SourceGroup: devsAzureGroup, + Members: NewStringSetFromItems(aliceAzure.AzureID), }, { - SourceGroup: getSourceGroup("hq", sourceType), - Members: NewStringSetFromItems(getSourceUser(carolName, sourceType).GetID()), + SourceGroup: hqAzureGroup, + Members: NewStringSetFromItems(carolAzure.AzureID), }, }, ytGroupsExpected: []YtsaurusGroupWithMembers{ { - YtsaurusGroup: getYtsaurusGroup(getSourceGroup("devs", sourceType)), - Members: NewStringSetFromItems(aliceName), + YtsaurusGroup: devsYtsaurusGroup, + Members: NewStringSetFromItems(aliceYtsaurus.Username), }, { - YtsaurusGroup: getYtsaurusGroup(getSourceGroup("hq", sourceType)), - Members: NewStringSetFromItems(carolName), + YtsaurusGroup: hqYtsaurusGroup, + Members: NewStringSetFromItems(carolYtsaurus.Username), }, }, }, { - name: "memberships-add-remove", - sourceType: sourceType, - sourceUsersSetUp: []SourceUser{ - getSourceUser(aliceName, sourceType), - getSourceUser(bobName, sourceType), - getSourceUser(carolName, sourceType), + name: "memberships-add-remove", + azureUsersSetUp: []SourceUser{ + aliceAzure, + bobAzure, + carolAzure, }, ytUsersSetUp: []YtsaurusUser{ - getYtsaurusUser(getSourceUser(aliceName, sourceType)), - getYtsaurusUser(getSourceUser(bobName, sourceType)), - getYtsaurusUser(getSourceUser(carolName, sourceType)), + aliceYtsaurus, + bobYtsaurus, + carolYtsaurus, }, ytUsersExpected: []YtsaurusUser{ - getYtsaurusUser(getSourceUser(aliceName, sourceType)), - getYtsaurusUser(getSourceUser(bobName, sourceType)), - getYtsaurusUser(getSourceUser(carolName, sourceType)), + aliceYtsaurus, + bobYtsaurus, + carolYtsaurus, }, ytGroupsSetUp: []YtsaurusGroupWithMembers{ { - YtsaurusGroup: getYtsaurusGroup(getSourceGroup("devs", sourceType)), + YtsaurusGroup: devsYtsaurusGroup, Members: NewStringSetFromItems( - aliceName, - bobName, + aliceYtsaurus.Username, + bobYtsaurus.Username, ), }, }, - sourceGroupsSetUp: []SourceGroupWithMembers{ + azureGroupsSetUp: []SourceGroupWithMembers{ { - SourceGroup: getSourceGroup("devs", sourceType), + SourceGroup: devsAzureGroup, Members: NewStringSetFromItems( - getSourceUser(aliceName, sourceType).GetID(), - getSourceUser(carolName, sourceType).GetID(), + aliceAzure.AzureID, + carolAzure.AzureID, ), }, }, ytGroupsExpected: []YtsaurusGroupWithMembers{ { - YtsaurusGroup: getYtsaurusGroup(getSourceGroup("devs", sourceType)), + YtsaurusGroup: devsYtsaurusGroup, Members: NewStringSetFromItems( - aliceName, - carolName, + aliceYtsaurus.Username, + carolYtsaurus.Username, ), }, }, }, - } - - if sourceType == AzureSourceType { - testCases = append(testCases, testCase{ - name: "display-name-changes", - sourceType: sourceType, - sourceUsersSetUp: []SourceUser{ - getSourceUser(aliceName, sourceType), - getSourceUser(bobName, sourceType), - getSourceUser(carolName, sourceType), + { + name: "display-name-changes", + azureUsersSetUp: []SourceUser{ + aliceAzure, + bobAzure, + carolAzure, }, ytUsersSetUp: []YtsaurusUser{ - getYtsaurusUser(getSourceUser(aliceName, sourceType)), - getYtsaurusUser(getSourceUser(bobName, sourceType)), - getYtsaurusUser(getSourceUser(carolName, sourceType)), + aliceYtsaurus, + bobYtsaurus, + carolYtsaurus, }, ytUsersExpected: []YtsaurusUser{ - getYtsaurusUser(getSourceUser(aliceName, sourceType)), - getYtsaurusUser(getSourceUser(bobName, sourceType)), - getYtsaurusUser(getSourceUser(carolName, sourceType)), + aliceYtsaurus, + bobYtsaurus, + carolYtsaurus, }, ytGroupsSetUp: []YtsaurusGroupWithMembers{ { - YtsaurusGroup: getYtsaurusGroup(getSourceGroup("devs", sourceType)), + YtsaurusGroup: devsYtsaurusGroup, Members: NewStringSetFromItems( - aliceName, - bobName, + aliceYtsaurus.Username, + bobYtsaurus.Username, ), }, { - YtsaurusGroup: getYtsaurusGroup(getSourceGroup("hq", sourceType)), + YtsaurusGroup: hqYtsaurusGroup, Members: NewStringSetFromItems( - aliceName, - bobName, + aliceYtsaurus.Username, + bobYtsaurus.Username, ), }, }, - sourceGroupsSetUp: []SourceGroupWithMembers{ + azureGroupsSetUp: []SourceGroupWithMembers{ { // This group should be updated. - SourceGroup: getUpdatedSourceGroup("devs", sourceType), + SourceGroup: devsAzureGroupChangedDisplayName, // Members list are also updated. Members: NewStringSetFromItems( - getSourceUser(aliceName, sourceType).GetID(), - getSourceUser(carolName, sourceType).GetID(), + aliceAzure.AzureID, + carolAzure.AzureID, ), }, { // for this group only displayName should be updated - SourceGroup: getChangedBackwardCompatibleSourceGroup("hq", sourceType), + SourceGroup: hqAzureGroupChangedBackwardCompatible, // members also changed Members: NewStringSetFromItems( - getSourceUser(aliceName, sourceType).GetID(), - getSourceUser(carolName, sourceType).GetID(), + aliceAzure.AzureID, + carolAzure.AzureID, ), }, }, ytGroupsExpected: []YtsaurusGroupWithMembers{ { - YtsaurusGroup: getYtsaurusGroup(getUpdatedSourceGroup("devs", sourceType)), + YtsaurusGroup: devsYtsaurusGroupChangedDisplayName, Members: NewStringSetFromItems( - aliceName, - carolName, + aliceYtsaurus.Username, + carolYtsaurus.Username, ), }, { - YtsaurusGroup: getYtsaurusGroup(getChangedBackwardCompatibleSourceGroup("hq", sourceType)), + YtsaurusGroup: hqYtsaurusGroupChangedBackwardCompatible, Members: NewStringSetFromItems( - aliceName, - carolName, + aliceYtsaurus.Username, + carolYtsaurus.Username, ), }, }, - }) - } - return testCases -} - -var ( - testTimeStr = "2023-10-20T12:00:00Z" - initialTestTime = parseAppTime(testTimeStr) - - defaultUsernameReplacements = []ReplacementPair{ - {"@acme.com", ""}, - {"@", ":"}, - } - defaultGroupnameReplacements = []ReplacementPair{ - {"|all", ""}, - } - defaultAppConfig = &AppConfig{ - UsernameReplacements: defaultUsernameReplacements, - GroupnameReplacements: defaultGroupnameReplacements, + }, } ) -// TestAppSync uses local YTsaurus container and some source (Azure or Ldap) to test all the cases: -// [x] If azur user not in YTsaurus -> created; -// [x] If Source user already in YTsaurus no changes -> skipped; -// [x] If Source user already in YTsaurus with changes -> updated; -// [x] If user in YTsaurus but not in Source (and ban_before_remove_duration=0) -> removed; -// [x] If user in YTsaurus but not in Source (and ban_before_remove_duration != 0) -> banned -> removed; +// TestAppSync uses local YTsaurus container and fake Azure to test all the cases: +// [x] If Azure user not in YTsaurus -> created; +// [x] If Azure user already in YTsaurus no changes -> skipped; +// [x] If Azure user already in YTsaurus with changes -> updated; +// [x] If user in YTsaurus but not in Azure (and ban_before_remove_duration=0) -> removed; +// [x] If user in YTsaurus but not in Azure (and ban_before_remove_duration != 0) -> banned -> removed; // [x] If Azure user without @azure attribute in YTsaurus —> ignored; -// [x] Source user field updates is reflected in YTsaurus user; +// [x] Azure user field updates is reflected in YTsaurus user; // [x] YTsaurus username is built according to config; // [x] YTsaurus username is in lowercase; -// [x] If Source group is not exist in YTsaurus -> creating YTsaurus with members; -// [x] If YTsaurus group is not exist in Source -> delete YTsaurus group; -// [x] If Source group membership changed -> update in YTsaurus group membership; -// [x] If Source group fields (though there are none extra fields) changed -> update YTsaurus group; -// [x] If Source group displayName changed -> recreate YTsaurus group; -// [x] If Source group displayName changed AND Source members changed -> recreate YTsaurus group with actual members set; +// [x] If Azure group is not exist in YTsaurus -> creating YTsaurus with members; +// [x] If YTsaurus group is not exist in Azure -> delete YTsaurus group; +// [x] If Azure group membership changed -> update in YTsaurus group membership; +// [x] If Azure group fields (though there are none extra fields) changed -> update YTsaurus group; +// [x] If Azure group displayName changed -> recreate YTsaurus group; +// [x] If Azure group displayName changed AND Azure members changed -> recreate YTsaurus group with actual members set; // [x] YTsaurus group name is built according to config; // [x] Remove limits config option works. func TestAppSync(t *testing.T) { require.NoError(t, os.Setenv(defaultYtsaurusSecretEnvVar, ytDevToken)) - for _, sourceType := range []SourceType{LdapSourceType, AzureSourceType} { - for _, tc := range getTestCases(sourceType) { - t.Run( - fmt.Sprintf("%v-%v", sourceType, tc.name), - func(tc testCase) func(t *testing.T) { - return func(t *testing.T) { - if tc.testTime.IsZero() { - tc.testTime = initialTestTime - } - clock := testclock.NewFakePassiveClock(initialTestTime) - - ytLocal := NewYtsaurusLocal() - defer func() { require.NoError(t, ytLocal.Stop()) }() - require.NoError(t, ytLocal.Start()) - - var source Source - - switch tc.sourceType { - case AzureSourceType: - azure := NewAzureFake() - azure.setUsers(tc.sourceUsersSetUp) - azure.setGroups(tc.sourceGroupsSetUp) - - source = azure - case LdapSourceType: - ldapLocal := NewOpenLdapLocal() - - defer func() { require.NoError(t, ldapLocal.Stop()) }() - require.NoError(t, ldapLocal.Start()) - - ldapConfig, err := ldapLocal.GetConfig() - require.NoError(t, err) - ldapSource, err := NewLdap(ldapConfig, getDevelopmentLogger()) - require.NoError(t, err) - - setupLdapObjects(t, ldapSource.Connection, tc.sourceUsersSetUp, tc.sourceGroupsSetUp) - source = ldapSource - } - - ytClient, err := ytLocal.GetClient() - require.NoError(t, err) - - initialYtUsers, initialYtGroups := getAllYtsaurusObjects(t, ytClient) - setupYtsaurusObjects(t, ytClient, tc.ytUsersSetUp, tc.ytGroupsSetUp) - - if tc.appConfig == nil { - tc.appConfig = defaultAppConfig - } - app, err := NewAppCustomized( - &Config{ - App: *tc.appConfig, - Azure: &AzureConfig{}, - Ldap: &LdapConfig{}, - Ytsaurus: YtsaurusConfig{ - Proxy: ytLocal.GetProxy(), - ApplyUserChanges: true, - ApplyGroupChanges: true, - ApplyMemberChanges: true, - LogLevel: "DEBUG", - }, - }, - getDevelopmentLogger(), - source, - clock, - ) - require.NoError(t, err) - - app.syncOnce() - - // We have eventually here, because user removal takes some time. - require.Eventually( - t, - func() bool { - udiff, gdiff := diffYtsaurusObjects(t, ytClient, tc.ytUsersExpected, initialYtUsers, tc.ytGroupsExpected, initialYtGroups) - actualUsers, actualGroups := getAllYtsaurusObjects(t, ytClient) - if udiff != "" { - t.Log("Users diff is not empty yet:", udiff) - t.Log("expected users", tc.ytUsersExpected) - t.Log("actual users", actualUsers) - } - if gdiff != "" { - t.Log("Groups diff is not empty yet:", gdiff) - t.Log("expected groups", tc.ytGroupsExpected) - t.Log("actual groups", actualGroups) - } - return udiff == "" && gdiff == "" - }, - 3*time.Second, - 300*time.Millisecond, - ) + for _, tc := range testCases { + t.Run( + tc.name, + func(tc testCase) func(t *testing.T) { + return func(t *testing.T) { + if tc.testTime.IsZero() { + tc.testTime = initialTestTime } - }(tc), - ) - } + clock := testclock.NewFakePassiveClock(initialTestTime) + + ytLocal := NewYtsaurusLocal() + defer func() { require.NoError(t, ytLocal.Stop()) }() + require.NoError(t, ytLocal.Start()) + + azure := NewAzureFake() + azure.setUsers(tc.azureUsersSetUp) + azure.setGroups(tc.azureGroupsSetUp) + + ytClient, err := ytLocal.GetClient() + require.NoError(t, err) + + initialYtUsers, initialYtGroups := getAllYtsaurusObjects(t, ytClient) + setupYtsaurusObjects(t, ytClient, tc.ytUsersSetUp, tc.ytGroupsSetUp) + + if tc.appConfig == nil { + tc.appConfig = defaultAppConfig + } + app, err := NewAppCustomized( + &Config{ + App: *tc.appConfig, + Azure: &AzureConfig{}, + Ytsaurus: YtsaurusConfig{ + Proxy: ytLocal.GetProxy(), + ApplyUserChanges: true, + ApplyGroupChanges: true, + ApplyMemberChanges: true, + LogLevel: "DEBUG", + SourceAttributeName: "azure", + }, + }, getDevelopmentLogger(), + azure, + clock, + ) + require.NoError(t, err) + + app.syncOnce() + + // we have eventually here, because user removal takes some time. + require.Eventually( + t, + func() bool { + udiff, gdiff := diffYtsaurusObjects(t, ytClient, tc.ytUsersExpected, initialYtUsers, tc.ytGroupsExpected, initialYtGroups) + actualUsers, actualGroups := getAllYtsaurusObjects(t, ytClient) + if udiff != "" { + t.Log("Users diff is not empty yet:", udiff) + t.Log("expected users", tc.ytUsersExpected) + t.Log("actual users", actualUsers) + } + if gdiff != "" { + t.Log("Groups diff is not empty yet:", gdiff) + t.Log("expected groups", tc.ytGroupsExpected) + t.Log("actual groups", actualGroups) + } + return udiff == "" && gdiff == "" + }, + 3*time.Second, + 300*time.Millisecond, + ) + } + }(tc), + ) } } @@ -644,121 +653,14 @@ func TestManageUnmanagedUsersIsForbidden(t *testing.T) { } func getAllYtsaurusObjects(t *testing.T, client yt.Client) (users []YtsaurusUser, groups []YtsaurusGroupWithMembers) { - allUsers, err := doGetAllYtsaurusUsers(context.Background(), client) + allUsers, err := doGetAllYtsaurusUsers(context.Background(), client, "azure") require.NoError(t, err) + fmt.Printf("HERE USERS: %v", allUsers) allGroups, err := doGetAllYtsaurusGroupsWithMembers(context.Background(), client) require.NoError(t, err) return allUsers, allGroups } -func setupLdapObjects(t *testing.T, conn *ldap.Conn, users []SourceUser, groups []SourceGroupWithMembers) { - require.NoError(t, conn.Add(&ldap.AddRequest{ - DN: "ou=People,dc=example,dc=org", - Attributes: []ldap.Attribute{ - { - Type: "objectClass", - Vals: []string{"organizationalUnit"}, - }, - { - Type: "ou", - Vals: []string{"People"}, - }, - }, - })) - - require.NoError(t, conn.Add(&ldap.AddRequest{ - DN: "ou=Group,dc=example,dc=org", - Attributes: []ldap.Attribute{ - { - Type: "objectClass", - Vals: []string{"organizationalUnit"}, - }, - { - Type: "ou", - Vals: []string{"Group"}, - }, - }, - })) - - for _, user := range users { - ldapUser := user.(LdapUser) - addRequest := ldap.AddRequest{ - DN: fmt.Sprintf("uid=%s,ou=People,dc=example,dc=org", user.GetID()), - Attributes: []ldap.Attribute{ - { - Type: "objectClass", - Vals: []string{"top", "posixAccount", "inetOrgPerson"}, - }, - { - Type: "ou", - Vals: []string{"People"}, - }, - { - Type: "cn", - Vals: []string{user.GetName()}, - }, - { - Type: "uid", - Vals: []string{user.GetID()}, - }, - { - Type: "uidNumber", - Vals: []string{user.GetID()}, - }, - { - Type: "gidNumber", - Vals: []string{user.GetID()}, - }, - { - Type: "givenName", - Vals: []string{ldapUser.FirstName}, - }, - { - Type: "homeDirectory", - Vals: []string{ldapUser.GetID()}, - }, - { - Type: "sn", - Vals: []string{ldapUser.GetName() + "-surname"}, - }, - }, - } - require.NoError(t, conn.Add(&addRequest)) - } - - for groupID, group := range groups { - ldapGroup := group.SourceGroup.(LdapGroup) - - members := make([]string, 0) - for member := range group.Members.Iter() { - members = append(members, member) - } - - addRequest := ldap.AddRequest{ - DN: fmt.Sprintf("cn=%s,ou=Group,dc=example,dc=org", ldapGroup.GetID()), - Attributes: []ldap.Attribute{ - { - Type: "objectClass", - Vals: []string{"top", "posixGroup"}, - }, - { - Type: "cn", - Vals: []string{ldapGroup.GetName()}, - }, - { - Type: "gidNumber", - Vals: []string{fmt.Sprint(groupID)}, - }, - { - Type: "memberUid", - Vals: members, - }, - }, - } - require.NoError(t, conn.Add(&addRequest)) - } -} - func setupYtsaurusObjects(t *testing.T, client yt.Client, users []YtsaurusUser, groups []YtsaurusGroupWithMembers) { t.Log("Setting up yt for test") for _, user := range users { diff --git a/config.go b/config.go index 8d5baac..0ef2038 100644 --- a/config.go +++ b/config.go @@ -9,9 +9,7 @@ type Config struct { Ytsaurus YtsaurusConfig `yaml:"ytsaurus"` Logging LoggingConfig `yaml:"logging"` - // One of them should be specified. Azure *AzureConfig `yaml:"azure,omitempty"` - Ldap *LdapConfig `yaml:"ldap,omitempty"` } type AppConfig struct { @@ -19,7 +17,7 @@ type AppConfig struct { // If it is not speciied or value is zero than auto-sync disabled (sync can be invoked only manually). SyncInterval time.Duration `yaml:"sync_interval"` - // UsernameReplacements is a list of replaces which will be applied to a username for source (Source or Ldap). + // UsernameReplacements is a list of replaces which will be applied to a username for source (Azure). // For example, you may use it to strip off characters like @ which are not recommended for use // in usernames as they are required to be escaped in YPath. UsernameReplacements []ReplacementPair `yaml:"username_replacements"` @@ -31,7 +29,7 @@ type AppConfig struct { RemoveLimit int `yaml:"remove_limit,omitempty"` // BanBeforeRemoveDuration is a duration of a graceful ban before finally removing the user from YTsaurus. - // If it is not specified, user will be removed straight after user was found to be missing from source (Source or Ldap). + // If it is not specified, user will be removed straight after user was found to be missing from source (Azure). BanBeforeRemoveDuration time.Duration `yaml:"ban_before_remove_duration"` } @@ -53,53 +51,13 @@ type AzureConfig struct { GroupsFilter string `yaml:"groups_filter"` Timeout time.Duration `yaml:"timeout"` - // TODO(nadya73): support for ldap also, but with other name. // GroupsDisplayNameSuffixPostFilter applied to the fetched groups display names. GroupsDisplayNameSuffixPostFilter string `yaml:"groups_display_name_suffix_post_filter"` - // TODO(nadya73): support for ldap also, but with other name. // DebugAzureIDs is a list of ids for which app will print more debug info in logs. DebugAzureIDs []string `yaml:"debug_azure_ids"` } -type LdapUsersConfig struct { - // A filter for getting users. - // For example, `(objectClass=account)`. - Filter string `yaml:"filter"` - // An attribute type which will be used as @name attribute. - // For example, `cn`. - UsernameAttributeType string `yaml:"username_attribute_type"` - // For example, `uid`. - UIDAttributeType string `yaml:"uid_attribute_type"` - FirstNameAttributeType *string `yaml:"first_name_attribute_type"` - // A list of usernames for which app will print more debug info in logs. - DebugUsernames []string `yaml:"debug_usernames"` -} - -type LdapGroupsConfig struct { - // A filter for getting groups. - // For example, `(objectClass=posixGroup)`. - Filter string `yaml:"filter"` - // An attribute type which will be used as @name attribute. - // For example, `cn`. - GroupnameAttributeType string `yaml:"groupname_attribute_type"` - // An attribute type which will be used for getting group members. - // For example, `memberUid`. - MemberUIDAttributeType string `yaml:"member_uid_attribute_type"` - - // A list of groupnames for which app will print more debug info in logs. - DebugGroupnames []string `yaml:"debug_groupnames"` -} - -type LdapConfig struct { - Address string `yaml:"address"` - BindDN string `yaml:"bind_dn"` - BindPasswordEnvVar string `yaml:"bind_password_env_var"` - Users LdapUsersConfig `yaml:"users"` - Groups LdapGroupsConfig `yaml:"groups"` - BaseDN string `yaml:"base_dn"` -} - type YtsaurusConfig struct { Proxy string `yaml:"proxy"` // SecretEnvVar is a name of env variable with YTsaurus token. Default: "YT_TOKEN". @@ -118,6 +76,8 @@ type YtsaurusConfig struct { DebugUsernames []string `yaml:"debug_usernames"` // DebugGroupnames is a list of YTsaurus groupnames for which app will print more debug info in logs. DebugGroupnames []string `yaml:"debug_groupnames"` + + SourceAttributeName string `yaml:"source_attribute_name"` } type LoggingConfig struct { diff --git a/config_test.go b/config_test.go index 7c01e8e..b54fdad 100644 --- a/config_test.go +++ b/config_test.go @@ -5,12 +5,10 @@ import ( "testing" "time" - "go.ytsaurus.tech/library/go/ptr" - "github.com/stretchr/testify/require" ) -//go:embed azure_config.example.yaml ldap_config.example.yaml +//go:embed azure_config.example.yaml var _ embed.FS func TestAzureConfig(t *testing.T) { @@ -30,8 +28,6 @@ func TestAzureConfig(t *testing.T) { require.Equal(t, 10, cfg.App.RemoveLimit) require.Equal(t, 7*24*time.Hour, cfg.App.BanBeforeRemoveDuration) - require.True(t, cfg.Ldap == nil) - require.Equal(t, "acme.onmicrosoft.com", cfg.Azure.Tenant) require.Equal(t, "abcdefgh-a000-b111-c222-abcdef123456", cfg.Azure.ClientID) require.Equal(t, 1*time.Second, cfg.Azure.Timeout) @@ -53,51 +49,3 @@ func TestAzureConfig(t *testing.T) { require.NoError(t, err) logger.Debugw("test logging message", "key", "val") } - -func TestLdapConfig(t *testing.T) { - configPath := "ldap_config.example.yaml" - - cfg, err := loadConfig(configPath) - require.NoError(t, err) - - require.Equal(t, ptr.Duration(5*time.Minute), cfg.App.SyncInterval) - require.Equal(t, []ReplacementPair{ - {From: "@acme.com", To: ""}, - {From: "@", To: ":"}, - }, cfg.App.UsernameReplacements) - require.Equal(t, []ReplacementPair{ - {From: "|all", To: ""}, - }, cfg.App.GroupnameReplacements) - require.Equal(t, ptr.Int(10), cfg.App.RemoveLimit) - require.Equal(t, ptr.Duration(7*24*time.Hour), cfg.App.BanBeforeRemoveDuration) - - require.True(t, cfg.Azure == nil) - - require.Equal(t, "dc=example,dc=org", cfg.Ldap.BaseDN) - require.Equal(t, "cn=admin,dc=example,dc=org", cfg.Ldap.BindDN) - require.Equal(t, "localhost:10210", cfg.Ldap.Address) - require.Equal(t, "LDAP_PASSWORD", cfg.Ldap.BindPasswordEnvVar) - - require.Equal(t, "(&(objectClass=posixAccount)(ou=People))", cfg.Ldap.Users.Filter) - require.Equal(t, "cn", cfg.Ldap.Users.UsernameAttributeType) - require.Equal(t, "uid", cfg.Ldap.Users.UIDAttributeType) - require.Equal(t, ptr.String("givenName"), cfg.Ldap.Users.FirstNameAttributeType) - - require.Equal(t, "(objectClass=posixGroup)", cfg.Ldap.Groups.Filter) - require.Equal(t, "cn", cfg.Ldap.Groups.GroupnameAttributeType) - require.Equal(t, "memberUid", cfg.Ldap.Groups.MemberUIDAttributeType) - - require.Equal(t, "localhost:10110", cfg.Ytsaurus.Proxy) - require.Equal(t, true, cfg.Ytsaurus.ApplyUserChanges) - require.Equal(t, true, cfg.Ytsaurus.ApplyGroupChanges) - require.Equal(t, true, cfg.Ytsaurus.ApplyMemberChanges) - require.Equal(t, 1*time.Second, cfg.Ytsaurus.Timeout) - require.Equal(t, "DEBUG", cfg.Ytsaurus.LogLevel) - - require.Equal(t, "WARN", cfg.Logging.Level) - require.Equal(t, true, cfg.Logging.IsProduction) - - logger, err := configureLogger(&cfg.Logging) - require.NoError(t, err) - logger.Debugw("test logging message", "key", "val") -} diff --git a/go.mod b/go.mod index 7cece01..cad6dfe 100644 --- a/go.mod +++ b/go.mod @@ -5,7 +5,6 @@ go 1.20 require ( github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.4.0 github.com/deckarep/golang-set/v2 v2.3.1 - github.com/go-ldap/ldap/v3 v3.4.6 github.com/google/go-cmp v0.6.0 github.com/jessevdk/go-flags v1.5.0 github.com/microsoft/kiota-abstractions-go v1.3.0 @@ -14,9 +13,7 @@ require ( github.com/pkg/errors v0.9.1 github.com/stretchr/testify v1.8.4 github.com/testcontainers/testcontainers-go v0.28.0 - github.com/testcontainers/testcontainers-go/modules/openldap v0.28.0 go.uber.org/zap v1.26.0 - go.ytsaurus.tech/library/go/ptr v0.0.1 go.ytsaurus.tech/yt/go v0.0.13 gopkg.in/yaml.v3 v3.0.1 k8s.io/utils v0.0.0-20230220204549-a5ecb0141aa5 @@ -27,7 +24,6 @@ require ( github.com/Azure/azure-sdk-for-go/sdk/azcore v1.8.0 // indirect github.com/Azure/azure-sdk-for-go/sdk/internal v1.3.0 // indirect github.com/Azure/go-ansiterm v0.0.0-20210617225240-d185dfc1b5a1 // indirect - github.com/Azure/go-ntlmssp v0.0.0-20221128193559-754e69321358 // indirect github.com/AzureAD/microsoft-authentication-library-for-go v1.2.0 // indirect github.com/Microsoft/go-winio v0.6.1 // indirect github.com/Microsoft/hcsshim v0.11.4 // indirect @@ -43,7 +39,6 @@ require ( github.com/docker/go-connections v0.5.0 // indirect github.com/docker/go-units v0.5.0 // indirect github.com/felixge/httpsnoop v1.0.3 // indirect - github.com/go-asn1-ber/asn1-ber v1.5.5 // indirect github.com/go-logr/logr v1.2.4 // indirect github.com/go-logr/stdr v1.2.2 // indirect github.com/go-ole/go-ole v1.2.6 // indirect @@ -94,6 +89,7 @@ require ( go.ytsaurus.tech/library/go/blockcodecs v0.0.2 // indirect go.ytsaurus.tech/library/go/core/log v0.0.3 // indirect go.ytsaurus.tech/library/go/core/xerrors v0.0.3 // indirect + go.ytsaurus.tech/library/go/ptr v0.0.1 // indirect go.ytsaurus.tech/library/go/x/xreflect v0.0.2 // indirect go.ytsaurus.tech/library/go/x/xruntime v0.0.3 // indirect golang.org/x/crypto v0.17.0 // indirect diff --git a/go.sum b/go.sum index 6073294..795c6e1 100644 --- a/go.sum +++ b/go.sum @@ -9,16 +9,12 @@ github.com/Azure/azure-sdk-for-go/sdk/internal v1.3.0 h1:sXr+ck84g/ZlZUOZiNELInm github.com/Azure/azure-sdk-for-go/sdk/internal v1.3.0/go.mod h1:okt5dMMTOFjX/aovMlrjvvXoPMBVSPzk9185BT0+eZM= github.com/Azure/go-ansiterm v0.0.0-20210617225240-d185dfc1b5a1 h1:UQHMgLO+TxOElx5B5HZ4hJQsoJ/PvUvKRhJHDQXO8P8= github.com/Azure/go-ansiterm v0.0.0-20210617225240-d185dfc1b5a1/go.mod h1:xomTg63KZ2rFqZQzSB4Vz2SUXa1BpHTVz9L5PTmPC4E= -github.com/Azure/go-ntlmssp v0.0.0-20221128193559-754e69321358 h1:mFRzDkZVAjdal+s7s0MwaRv9igoPqLRdzOLzw/8Xvq8= -github.com/Azure/go-ntlmssp v0.0.0-20221128193559-754e69321358/go.mod h1:chxPXzSsl7ZWRAuOIE23GDNzjWuZquvFlgA8xmpunjU= github.com/AzureAD/microsoft-authentication-library-for-go v1.2.0 h1:hVeq+yCyUi+MsoO/CU95yqCIcdzra5ovzk8Q2BBpV2M= github.com/AzureAD/microsoft-authentication-library-for-go v1.2.0/go.mod h1:wP83P5OoQ5p6ip3ScPr0BAq0BvuPAvacpEuSzyouqAI= github.com/Microsoft/go-winio v0.6.1 h1:9/kr64B9VUZrLm5YYwbGtUJnMgqWVOdUAXu6Migciow= github.com/Microsoft/go-winio v0.6.1/go.mod h1:LRdKpFKfdobln8UmuiYcKPot9D2v6svN5+sAH+4kjUM= github.com/Microsoft/hcsshim v0.11.4 h1:68vKo2VN8DE9AdN4tnkWnmdhqdbpUFM8OF3Airm7fz8= github.com/Microsoft/hcsshim v0.11.4/go.mod h1:smjE4dvqPX9Zldna+t5FG3rnoHhaB7QYxPRqGcpAD9w= -github.com/alexbrainman/sspi v0.0.0-20210105120005-909beea2cc74 h1:Kk6a4nehpJ3UuJRqlA3JxYxBZEqCeOmATOvrbT4p9RA= -github.com/alexbrainman/sspi v0.0.0-20210105120005-909beea2cc74/go.mod h1:cEWa1LVoE5KvSD9ONXsZrj0z6KqySlCCNKHlLzbqAt4= github.com/andybalholm/brotli v1.0.4 h1:V7DdXeJtZscaqfNuAdSRuRFzuiKlHSC/Zh3zl9qY3JY= github.com/andybalholm/brotli v1.0.4/go.mod h1:fO7iG3H7G2nSZ7m0zPUDn85XEX2GTukHGRSepvi9Eig= github.com/cenkalti/backoff/v4 v4.2.1 h1:y4OZtCnogmCPw98Zjyt5a6+QwPLGkiQsYW5oUqylYbM= @@ -49,10 +45,6 @@ github.com/docker/go-units v0.5.0/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDD github.com/felixge/httpsnoop v1.0.3 h1:s/nj+GCswXYzN5v2DpNMuMQYe+0DDwt5WVCU6CWBdXk= github.com/felixge/httpsnoop v1.0.3/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U= github.com/frankban/quicktest v1.14.5 h1:dfYrrRyLtiqT9GyKXgdh+k4inNeTvmGbuSgZ3lx3GhA= -github.com/go-asn1-ber/asn1-ber v1.5.5 h1:MNHlNMBDgEKD4TcKr36vQN68BA00aDfjIt3/bD50WnA= -github.com/go-asn1-ber/asn1-ber v1.5.5/go.mod h1:hEBeB/ic+5LoWskz+yKT7vGhhPYkProFKoKdwZRWMe0= -github.com/go-ldap/ldap/v3 v3.4.6 h1:ert95MdbiG7aWo/oPYp9btL3KJlMPKnP58r09rI8T+A= -github.com/go-ldap/ldap/v3 v3.4.6/go.mod h1:IGMQANNtxpsOzj7uUAMjpGBaOVTC4DYyIy8VsTdxmtc= github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= github.com/go-logr/logr v1.2.4 h1:g01GSCwiDw2xSZfjJ2/T9M+S6pFdcNtFYsp+Y43HYDQ= github.com/go-logr/logr v1.2.4/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= @@ -78,7 +70,6 @@ github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/google/tink/go v1.7.0 h1:6Eox8zONGebBFcCBqkVmt60LaWZa6xg1cl/DwAh/J1w= github.com/google/tink/go v1.7.0/go.mod h1:GAUOd+QE3pgj9q8VKIGTCP33c/B7eb4NhxLcgTJZStM= -github.com/google/uuid v1.3.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/grpc-ecosystem/grpc-gateway/v2 v2.16.0 h1:YBftPWNWd4WwGqtY2yeZL2ef8rHAxPBD8KFhJpmcqms= @@ -166,15 +157,12 @@ github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcU github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= github.com/testcontainers/testcontainers-go v0.28.0 h1:1HLm9qm+J5VikzFDYhOd+Zw12NtOl+8drH2E8nTY1r8= github.com/testcontainers/testcontainers-go v0.28.0/go.mod h1:COlDpUXbwW3owtpMkEB1zo9gwb1CoKVKlyrVPejF4AU= -github.com/testcontainers/testcontainers-go/modules/openldap v0.28.0 h1:hR3BtDFSPUXvMZNuVbw7YcqAFsvjwt/VNA6ZTjtDZiM= -github.com/testcontainers/testcontainers-go/modules/openldap v0.28.0/go.mod h1:8/VlUrL0D5j0G3Vy3c+APgdBH3RZJcIETkualsRA8W4= github.com/tklauser/go-sysconf v0.3.12 h1:0QaGUFOdQaIVdPgfITYzaTegZvdCjmYO52cSFAEVmqU= github.com/tklauser/go-sysconf v0.3.12/go.mod h1:Ho14jnntGE1fpdOqQEEaiKRpvIavV0hSfmBq8nJbHYI= github.com/tklauser/numcpus v0.6.1 h1:ng9scYS7az0Bk4OZLvrNXNSAO2Pxr1XXRAPyjhIx+Fk= github.com/tklauser/numcpus v0.6.1/go.mod h1:1XfjsgE2zo8GVw7POkMbHENHzVg3GzmoZ9fESEdAacY= github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= -github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= github.com/yusufpapurcu/wmi v1.2.3 h1:E1ctvB7uKFMOJw3fdOW32DwGE9I7t++CRUEMKvFoFiw= github.com/yusufpapurcu/wmi v1.2.3/go.mod h1:SBZ9tNy3G9/m5Oi98Zks0QjeHVDvuK0qfxQmPyzfmi0= go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.45.0 h1:x8Z78aZx8cOF0+Kkazoc7lwUNMGy0LrzEMxTm4BbTxg= @@ -214,65 +202,40 @@ go.ytsaurus.tech/yt/go v0.0.13/go.mod h1:r3puwhqLgS/TD9VZ8/3rhuxtdD02bQ+yuZWncNu golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= -golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= -golang.org/x/crypto v0.13.0/go.mod h1:y6Z2r+Rw4iayiXXAIxJIDAJ1zMW4yaTpebo8fPOliYc= golang.org/x/crypto v0.17.0 h1:r8bRNjWL3GshPW3gkd+RpvzWrZAwPS49OmTGZ/uhM4k= golang.org/x/crypto v0.17.0/go.mod h1:gCAAfMLgwOJRpTjQ2zCCt2OcSfYMTeZVSRtQlPC7Nq4= golang.org/x/exp v0.0.0-20230725093048-515e97ebf090 h1:Di6/M8l0O2lCLc6VVRWhgCiApHV8MnQurBnFSHsQtNY= golang.org/x/exp v0.0.0-20230725093048-515e97ebf090/go.mod h1:FXUEEKJgO7OQYeo8N01OfiKP8RXMtf6e8aTskBGqWdc= golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= -golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= -golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= golang.org/x/mod v0.11.0 h1:bUO06HqtnRcc/7l71XBe4WcqTZ+3AH1J59zWDDwLKgU= golang.org/x/mod v0.11.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= -golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= -golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= -golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= -golang.org/x/net v0.10.0/go.mod h1:0qNGK6F8kojg2nk9dLZ2mShWaEBan6FAoqfSigmmuDg= golang.org/x/net v0.17.0 h1:pVaXccu2ozPjCXewfr1S7xza/zcXTity9cCdXQYSjIM= golang.org/x/net v0.17.0/go.mod h1:NxSsAGuq816PNPmqtQdLE42eU2Fs7NoRIZrHJAlaCOE= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.3.0 h1:ftCYgMx6zT/asHUrPw8BLLscYtGznsLAnjq5RH9P66E= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190916202348-b4ddaad3f8a3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201204225414-ed752295db88/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210320140829-1e4c9ba3b0c4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210616045830-e2b7044e8c71/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210616094352-59db8d763f22/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.11.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.12.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.15.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/sys v0.16.0 h1:xWw16ngr6ZMtmxDyKyIgsE93KNKz5HKmMa3b8ALHidU= golang.org/x/sys v0.16.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= -golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= -golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= -golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k= -golang.org/x/term v0.8.0/go.mod h1:xPskH00ivmX89bAKVGSKKtLOWNx2+17Eiy94tnKShWo= -golang.org/x/term v0.12.0/go.mod h1:owVbMEjm3cBLCHdkQu9b1opXd4ETQWc3BhuQGKgXgvU= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= -golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= -golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= -golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= -golang.org/x/text v0.13.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= golang.org/x/text v0.14.0 h1:ScX5w1eTa3QqT8oi6+ziP7dTV1S2+ALU0bI+0zXKWiQ= golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= golang.org/x/time v0.3.0 h1:rg5rLMjNzMS1RkNLzCG38eapWhnYLFYXDXj2gOlr8j4= @@ -280,8 +243,6 @@ golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGm golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= -golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= -golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU= golang.org/x/tools v0.10.0 h1:tvDr/iQoUqNdohiYm0LmmKcBk+q86lb9EprIUFhHHGg= golang.org/x/tools v0.10.0/go.mod h1:UJwyiVBsOA2uwvK/e5OY3GTpDUJriEd+/YlqAwLPmyM= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= diff --git a/ldap.go b/ldap.go deleted file mode 100644 index 0c9f736..0000000 --- a/ldap.go +++ /dev/null @@ -1,90 +0,0 @@ -package main - -import ( - "log" - - "github.com/go-ldap/ldap/v3" - "k8s.io/utils/env" -) - -type Ldap struct { - Connection *ldap.Conn - Config *LdapConfig -} - -func NewLdap(cfg *LdapConfig, logger appLoggerType) (*Ldap, error) { - conn, err := ldap.DialURL(cfg.Address) - if err != nil { - log.Fatalf("Failed to connect: %s\n", err) - return nil, err - } - - _, err = conn.SimpleBind(&ldap.SimpleBindRequest{ - Username: cfg.BindDN, - Password: env.GetString(cfg.BindPasswordEnvVar, "adminpassword"), - }) - if err != nil { - log.Fatalf("Failed to bind: %s\n", err) - return nil, err - } - return &Ldap{ - Connection: conn, - Config: cfg, - }, nil -} - -func (l *Ldap) GetSourceType() SourceType { - return LdapSourceType -} - -func (l *Ldap) GetUsers() ([]SourceUser, error) { - res, err := l.Connection.Search(&ldap.SearchRequest{ - BaseDN: l.Config.BaseDN, - Filter: l.Config.Users.Filter, - Attributes: []string{"*"}, - Scope: ldap.ScopeWholeSubtree, - }) - if err != nil { - return nil, err - } - - var users []SourceUser - for _, entry := range res.Entries { - username := entry.GetAttributeValue(l.Config.Users.UsernameAttributeType) - uid := entry.GetAttributeValue(l.Config.Users.UIDAttributeType) - var firstName string - if l.Config.Users.FirstNameAttributeType != nil { - firstName = entry.GetAttributeValue(*l.Config.Users.FirstNameAttributeType) - } - users = append(users, LdapUser{ - Username: username, - UID: uid, - FirstName: firstName}) - } - return users, nil -} - -func (l *Ldap) GetGroupsWithMembers() ([]SourceGroupWithMembers, error) { - res, err := l.Connection.Search(&ldap.SearchRequest{ - BaseDN: l.Config.BaseDN, - Filter: l.Config.Groups.Filter, - Attributes: []string{"*"}, - Scope: ldap.ScopeWholeSubtree, - }) - if err != nil { - return nil, err - } - - var groups []SourceGroupWithMembers - for _, entry := range res.Entries { - groupname := entry.GetAttributeValue(l.Config.Groups.GroupnameAttributeType) - members := entry.GetAttributeValues(l.Config.Groups.MemberUIDAttributeType) - groups = append(groups, SourceGroupWithMembers{ - SourceGroup: LdapGroup{ - Groupname: groupname, - }, - Members: NewStringSetFromItems(members...), - }) - } - return groups, nil -} diff --git a/ldap_config.example.yaml b/ldap_config.example.yaml deleted file mode 100644 index 06fd81f..0000000 --- a/ldap_config.example.yaml +++ /dev/null @@ -1,39 +0,0 @@ -app: - sync_interval: 5m - username_replacements: - - from: "@acme.com" - to: "" - - from: "@" - to: ":" - groupname_replacements: - - from: "|all" - to: "" - remove_limit: 10 - ban_before_remove_duration: 168h # 7d - -ldap: - address: "localhost:10210" - bind_dn: "cn=admin,dc=example,dc=org" - bind_password_env_var: "LDAP_PASSWORD" - base_dn: "dc=example,dc=org" - users: - filter: "(&(objectClass=posixAccount)(ou=People))" - username_attribute_type: "cn" - uid_attribute_type: "uid" - first_name_attribute_type: "givenName" - groups: - filter: "(objectClass=posixGroup)" - groupname_attribute_type: "cn" - member_uid_attribute_type: "memberUid" - -ytsaurus: - proxy: localhost:10110 - apply_user_changes: true - apply_group_changes: true - apply_member_changes: true - timeout: 1s - log_level: DEBUG - -logging: - level: WARN - is_production: true diff --git a/source.go b/source.go index 753fd71..2d20da1 100644 --- a/source.go +++ b/source.go @@ -10,7 +10,6 @@ type ObjectID = string type SourceType string const ( - LdapSourceType SourceType = "ldap" AzureSourceType SourceType = "azure" ) @@ -33,13 +32,6 @@ func NewSourceUser(sourceType SourceType, attributes map[string]any) (SourceUser } switch sourceType { - case LdapSourceType: - var ldapUser LdapUser - err = yson.Unmarshal(bytes, &ldapUser) - if err != nil { - return nil, err - } - return ldapUser, nil case AzureSourceType: var azureUser AzureUser err = yson.Unmarshal(bytes, &azureUser) @@ -75,25 +67,6 @@ func (user AzureUser) GetSourceType() SourceType { return AzureSourceType } -type LdapUser struct { - Username string `yson:"username"` - UID string `yson:"uid"` - FirstName string `yson:"first_name"` - // TODO(nadya73): Add more fields. -} - -func (user LdapUser) GetID() ObjectID { - return user.UID -} - -func (user LdapUser) GetName() string { - return user.Username -} - -func (user LdapUser) GetSourceType() SourceType { - return LdapSourceType -} - type SourceGroup interface { GetID() ObjectID GetName() string @@ -107,13 +80,6 @@ func NewSourceGroup(sourceType SourceType, attributes map[string]any) (SourceGro } switch sourceType { - case LdapSourceType: - var ldapGroup LdapGroup - err = yson.Unmarshal(bytes, &ldapGroup) - if err != nil { - return nil, err - } - return ldapGroup, nil case AzureSourceType: var azureGroup AzureGroup err = yson.Unmarshal(bytes, &azureGroup) @@ -146,22 +112,6 @@ func (ag AzureGroup) GetSourceType() SourceType { return AzureSourceType } -type LdapGroup struct { - Groupname string `yson:"groupname"` -} - -func (lg LdapGroup) GetID() ObjectID { - return lg.Groupname -} - -func (lg LdapGroup) GetName() string { - return lg.Groupname -} - -func (lg LdapGroup) GetSourceType() SourceType { - return LdapSourceType -} - type SourceGroupWithMembers struct { SourceGroup SourceGroup // Members is a set of strings, representing users' ObjectID. diff --git a/testcontainer_openldap.go b/testcontainer_openldap.go deleted file mode 100644 index ab74f93..0000000 --- a/testcontainer_openldap.go +++ /dev/null @@ -1,66 +0,0 @@ -package main - -import ( - "context" - "errors" - - "github.com/testcontainers/testcontainers-go" - "github.com/testcontainers/testcontainers-go/modules/openldap" - - "go.ytsaurus.tech/library/go/ptr" -) - -type OpenLdapLocal struct { - container *openldap.OpenLDAPContainer -} - -func NewOpenLdapLocal() *OpenLdapLocal { - return &OpenLdapLocal{} -} - -func (y *OpenLdapLocal) Start() error { - ctx := context.Background() - container, err := openldap.RunContainer(ctx, testcontainers.WithImage("bitnami/openldap:2.6.6")) - if err != nil { - return err - } - y.container = container - return y.container.Start(ctx) -} - -func (y *OpenLdapLocal) GetConfig() (*LdapConfig, error) { - connectionString, err := y.container.ConnectionString(context.Background()) - if err != nil { - return nil, err - } - return &LdapConfig{ - Address: connectionString, - BaseDN: "dc=example,dc=org", - BindDN: "cn=admin,dc=example,dc=org", - BindPasswordEnvVar: "LDAP_PASSWORD", - Users: LdapUsersConfig{ - Filter: "(&(objectClass=posixAccount)(ou=People))", - UsernameAttributeType: "cn", - UIDAttributeType: "uid", - FirstNameAttributeType: ptr.String("givenName"), - }, - Groups: LdapGroupsConfig{ - Filter: "(objectClass=posixGroup)", - GroupnameAttributeType: "cn", - MemberUIDAttributeType: "memberUid", - }, - }, nil -} - -func (y *OpenLdapLocal) Stop() error { - ctx := context.Background() - if y.container == nil { - return errors.New("container not started") - } - err := y.container.Terminate(ctx) - if err != nil { - return err - } - y.container = nil - return nil -} diff --git a/ytsaurus.go b/ytsaurus.go index 740bcef..95553a6 100644 --- a/ytsaurus.go +++ b/ytsaurus.go @@ -31,6 +31,8 @@ type Ytsaurus struct { debugUsernames []string debugGroupnames []string + + sourceAttributeName string } func NewYtsaurus(cfg *YtsaurusConfig, logger appLoggerType, clock clock.PassiveClock) (*Ytsaurus, error) { @@ -70,8 +72,9 @@ func NewYtsaurus(cfg *YtsaurusConfig, logger appLoggerType, clock clock.PassiveC timeout: cfg.Timeout, clock: clock, - debugUsernames: cfg.DebugUsernames, - debugGroupnames: cfg.DebugGroupnames, + debugUsernames: cfg.DebugUsernames, + debugGroupnames: cfg.DebugGroupnames, + sourceAttributeName: cfg.SourceAttributeName, }, nil } @@ -79,7 +82,7 @@ func (y *Ytsaurus) GetUsers(sourceType SourceType) ([]YtsaurusUser, error) { ctx, cancel := context.WithTimeout(context.Background(), y.timeout) defer cancel() - users, err := doGetAllYtsaurusUsers(ctx, y.client) + users, err := doGetAllYtsaurusUsers(ctx, y.client, y.sourceAttributeName) if err != nil { return nil, errors.Wrap(err, "failed to get ytsaurus users") } diff --git a/ytsaurus_helpers.go b/ytsaurus_helpers.go index 3b92b83..4a6106f 100644 --- a/ytsaurus_helpers.go +++ b/ytsaurus_helpers.go @@ -12,14 +12,21 @@ import ( // Lower level functions for reusing in tests. -func doGetAllYtsaurusUsers(ctx context.Context, client yt.Client) ([]YtsaurusUser, error) { +const ( + bannedSinceAttributeName = "banned_since" + bannedAttributeName = "banned" + sourceTypeAttributeName = "source_type" +) + +func doGetAllYtsaurusUsers(ctx context.Context, client yt.Client, sourceAttributeName string) ([]YtsaurusUser, error) { type YtsaurusUserResponse struct { - Name string `yson:",value"` - Azure *AzureUser `yson:"azure,attr"` - SourceType *SourceType `yson:"source_type,attr"` + Name string `yson:",value"` + Attrs map[string]any `yson:",attrs"` + /*Azure *AzureUser `yson:"azure,attr"` Source map[string]any `yson:"source,attr"` + SourceType *SourceType `yson:"source_type,attr"` Banned bool `yson:"banned,attr"` - BannedSince string `yson:"banned_since,attr"` + BannedSince string `yson:"banned_since,attr"`*/ } var response []YtsaurusUserResponse @@ -29,11 +36,10 @@ func doGetAllYtsaurusUsers(ctx context.Context, client yt.Client) ([]YtsaurusUse &response, &yt.ListNodeOptions{ Attributes: []string{ - "azure", - "banned", - "banned_since", - "source_type", - "source", + bannedAttributeName, + bannedSinceAttributeName, + sourceTypeAttributeName, + sourceAttributeName, }, }, ) @@ -44,21 +50,29 @@ func doGetAllYtsaurusUsers(ctx context.Context, client yt.Client) ([]YtsaurusUse var users []YtsaurusUser for _, ytUser := range response { var bannedSince time.Time - if ytUser.BannedSince != "" { - bannedSince, err = time.Parse(appTimeFormat, ytUser.BannedSince) + if bannedSinceRaw, ok := ytUser.Attrs[bannedSinceAttributeName]; ok && bannedSinceRaw != "" { + bannedSince, err = time.Parse(appTimeFormat, bannedSinceRaw.(string)) if err != nil { return nil, errors.Wrapf(err, "failed to parse @banned_since. %v", ytUser) } } var sourceUser SourceUser - if ytUser.Azure != nil { + if sourceRaw, ok := ytUser.Attrs[sourceAttributeName]; ok { + sourceType := AzureSourceType + if sourceTypeRaw, ok := ytUser.Attrs[sourceTypeAttributeName]; ok { + sourceType = SourceType(sourceTypeRaw.(string)) + } + sourceUser, err = NewSourceUser(sourceType, sourceRaw.(map[string]any)) + } + + /*if ytUser.Azure != nil { sourceUser = *ytUser.Azure } else if ytUser.SourceType != nil && ytUser.Source != nil { sourceUser, err = NewSourceUser(*ytUser.SourceType, ytUser.Source) if err != nil { return nil, errors.Wrapf(err, "failed to create source user. %v", ytUser) } - } + }*/ users = append(users, YtsaurusUser{ Username: ytUser.Name, diff --git a/ytsaurus_models.go b/ytsaurus_models.go index 43dd96a..bb90320 100644 --- a/ytsaurus_models.go +++ b/ytsaurus_models.go @@ -15,8 +15,6 @@ func (u YtsaurusUser) GetSourceAttributeName() string { switch u.SourceUser.GetSourceType() { case AzureSourceType: return "azure" - case LdapSourceType: - return "source" } return "source" } @@ -47,8 +45,6 @@ func (g YtsaurusGroup) GetSourceAttributeName() string { switch g.SourceGroup.GetSourceType() { case AzureSourceType: return "azure" - case LdapSourceType: - return "source" } return "source" } From 5ca528bcb53c95a4e9720c2fb0fd8538dfc11b6d Mon Sep 17 00:00:00 2001 From: Nadezhda Savchenko Date: Wed, 28 Feb 2024 17:26:05 +0300 Subject: [PATCH 11/21] Add default value for source_attribute_name --- config.go | 2 +- ytsaurus.go | 6 +++++- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/config.go b/config.go index 0ef2038..3539514 100644 --- a/config.go +++ b/config.go @@ -77,7 +77,7 @@ type YtsaurusConfig struct { // DebugGroupnames is a list of YTsaurus groupnames for which app will print more debug info in logs. DebugGroupnames []string `yaml:"debug_groupnames"` - SourceAttributeName string `yaml:"source_attribute_name"` + SourceAttributeName *string `yaml:"source_attribute_name"` } type LoggingConfig struct { diff --git a/ytsaurus.go b/ytsaurus.go index 95553a6..8407cef 100644 --- a/ytsaurus.go +++ b/ytsaurus.go @@ -2,6 +2,7 @@ package main import ( "context" + "go.ytsaurus.tech/library/go/ptr" "os" "time" @@ -62,6 +63,9 @@ func NewYtsaurus(cfg *YtsaurusConfig, logger appLoggerType, clock clock.PassiveC if cfg.Timeout == 0 { cfg.Timeout = defaultYtsaurusTimeout } + if cfg.SourceAttributeName == nil { + cfg.SourceAttributeName = ptr.String("source") + } return &Ytsaurus{ client: client, dryRunUsers: !cfg.ApplyUserChanges, @@ -74,7 +78,7 @@ func NewYtsaurus(cfg *YtsaurusConfig, logger appLoggerType, clock clock.PassiveC debugUsernames: cfg.DebugUsernames, debugGroupnames: cfg.DebugGroupnames, - sourceAttributeName: cfg.SourceAttributeName, + sourceAttributeName: *cfg.SourceAttributeName, }, nil } From 95714f97519fff3e8991b40a62e88709cf917d16 Mon Sep 17 00:00:00 2001 From: Nadezhda Savchenko Date: Wed, 28 Feb 2024 17:27:57 +0300 Subject: [PATCH 12/21] Rename source.go -> source_models.go --- source.go => source_models.go | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename source.go => source_models.go (100%) diff --git a/source.go b/source_models.go similarity index 100% rename from source.go rename to source_models.go From 1c6952dcc92dd70161961d59e816281bc932ff8b Mon Sep 17 00:00:00 2001 From: Nadezhda Savchenko Date: Wed, 28 Feb 2024 17:29:21 +0300 Subject: [PATCH 13/21] Remove commented code --- ytsaurus_helpers.go | 14 -------------- 1 file changed, 14 deletions(-) diff --git a/ytsaurus_helpers.go b/ytsaurus_helpers.go index 4a6106f..10303ae 100644 --- a/ytsaurus_helpers.go +++ b/ytsaurus_helpers.go @@ -22,11 +22,6 @@ func doGetAllYtsaurusUsers(ctx context.Context, client yt.Client, sourceAttribut type YtsaurusUserResponse struct { Name string `yson:",value"` Attrs map[string]any `yson:",attrs"` - /*Azure *AzureUser `yson:"azure,attr"` - Source map[string]any `yson:"source,attr"` - SourceType *SourceType `yson:"source_type,attr"` - Banned bool `yson:"banned,attr"` - BannedSince string `yson:"banned_since,attr"`*/ } var response []YtsaurusUserResponse @@ -65,15 +60,6 @@ func doGetAllYtsaurusUsers(ctx context.Context, client yt.Client, sourceAttribut sourceUser, err = NewSourceUser(sourceType, sourceRaw.(map[string]any)) } - /*if ytUser.Azure != nil { - sourceUser = *ytUser.Azure - } else if ytUser.SourceType != nil && ytUser.Source != nil { - sourceUser, err = NewSourceUser(*ytUser.SourceType, ytUser.Source) - if err != nil { - return nil, errors.Wrapf(err, "failed to create source user. %v", ytUser) - } - }*/ - users = append(users, YtsaurusUser{ Username: ytUser.Name, SourceUser: sourceUser, From 2dcbefafb6d32e5cafcc4fe23823f749812bdd4f Mon Sep 17 00:00:00 2001 From: Nadezhda Savchenko Date: Wed, 28 Feb 2024 19:11:23 +0300 Subject: [PATCH 14/21] Fix build --- app_test.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/app_test.go b/app_test.go index bcbebd0..ec2603f 100644 --- a/app_test.go +++ b/app_test.go @@ -3,6 +3,7 @@ package main import ( "context" "fmt" + "go.ytsaurus.tech/library/go/ptr" "os" "testing" "time" @@ -570,7 +571,7 @@ func TestAppSync(t *testing.T) { ApplyGroupChanges: true, ApplyMemberChanges: true, LogLevel: "DEBUG", - SourceAttributeName: "azure", + SourceAttributeName: ptr.String("azure"), }, }, getDevelopmentLogger(), azure, From 614ce40906685f44aaebacf28a24dc2102aad573 Mon Sep 17 00:00:00 2001 From: Nadezhda Savchenko Date: Tue, 5 Mar 2024 18:40:30 +0300 Subject: [PATCH 15/21] Review fixes --- app.go | 6 + app_test.go | 186 +++++++++++++++------------- azure_models.go | 106 ++++++++++++++++ diff.go | 227 +++++++++++++++++++++++++--------- migration_integration_test.go | 4 +- source_models.go | 111 ----------------- ytsaurus.go | 14 +-- ytsaurus_helpers.go | 116 +++++++++-------- ytsaurus_models.go | 48 +++---- ytsaurus_test.go | 64 +++++----- 10 files changed, 511 insertions(+), 371 deletions(-) create mode 100644 azure_models.go diff --git a/app.go b/app.go index 7736082..171f634 100644 --- a/app.go +++ b/app.go @@ -11,6 +11,12 @@ import ( "k8s.io/utils/clock" ) +type Source interface { + GetUsers() ([]SourceUser, error) + GetGroupsWithMembers() ([]SourceGroupWithMembers, error) + GetSourceType() SourceType +} + type App struct { syncInterval time.Duration usernameReplaces []ReplacementPair diff --git a/app_test.go b/app_test.go index ec2603f..871fa84 100644 --- a/app_test.go +++ b/app_test.go @@ -2,7 +2,6 @@ package main import ( "context" - "fmt" "go.ytsaurus.tech/library/go/ptr" "os" "testing" @@ -100,122 +99,134 @@ var ( } aliceYtsaurus = YtsaurusUser{ - Username: "alice", - SourceUser: AzureUser{ - AzureID: aliceAzure.AzureID, - PrincipalName: aliceAzure.PrincipalName, - Email: aliceAzure.Email, - FirstName: aliceAzure.FirstName, - LastName: aliceAzure.LastName, - DisplayName: aliceAzure.DisplayName, + Username: "alice", + SourceType: ptr.String("azure"), + SourceRaw: map[string]any{ + "id": aliceAzure.AzureID, + "principal_name": aliceAzure.PrincipalName, + "email": aliceAzure.Email, + "first_name": aliceAzure.FirstName, + "last_name": aliceAzure.LastName, + "display_name": aliceAzure.DisplayName, }, } bobYtsaurus = YtsaurusUser{ - Username: "bob", - SourceUser: AzureUser{ - AzureID: bobAzure.AzureID, - PrincipalName: bobAzure.PrincipalName, - Email: bobAzure.Email, - FirstName: bobAzure.FirstName, - LastName: bobAzure.LastName, - DisplayName: bobAzure.DisplayName, + Username: "bob", + SourceType: ptr.String("azure"), + SourceRaw: map[string]any{ + "id": bobAzure.AzureID, + "principal_name": bobAzure.PrincipalName, + "email": bobAzure.Email, + "first_name": bobAzure.FirstName, + "last_name": bobAzure.LastName, + "display_name": bobAzure.DisplayName, }, } carolYtsaurus = YtsaurusUser{ - Username: "carol", - SourceUser: AzureUser{ - AzureID: carolAzure.AzureID, - PrincipalName: carolAzure.PrincipalName, - Email: carolAzure.Email, - FirstName: carolAzure.FirstName, - LastName: carolAzure.LastName, - DisplayName: carolAzure.DisplayName, + Username: "carol", + SourceType: ptr.String("azure"), + SourceRaw: map[string]any{ + "id": carolAzure.AzureID, + "principal_name": carolAzure.PrincipalName, + "email": carolAzure.Email, + "first_name": carolAzure.FirstName, + "last_name": carolAzure.LastName, + "display_name": carolAzure.DisplayName, }, } aliceYtsaurusChangedLastName = YtsaurusUser{ - Username: aliceYtsaurus.Username, - SourceUser: AzureUser{ - AzureID: aliceYtsaurus.SourceUser.(AzureUser).AzureID, - PrincipalName: aliceYtsaurus.SourceUser.(AzureUser).PrincipalName, - Email: aliceYtsaurus.SourceUser.(AzureUser).Email, - FirstName: aliceYtsaurus.SourceUser.(AzureUser).FirstName, - LastName: aliceAzureChangedLastName.LastName, - DisplayName: aliceYtsaurus.SourceUser.(AzureUser).DisplayName, + Username: aliceYtsaurus.Username, + SourceType: ptr.String("azure"), + SourceRaw: map[string]any{ + "id": aliceYtsaurus.SourceRaw["id"], + "principal_name": aliceYtsaurus.SourceRaw["principal_name"], + "email": aliceYtsaurus.SourceRaw["email"], + "first_name": aliceYtsaurus.SourceRaw["first_name"], + "last_name": aliceAzureChangedLastName.LastName, + "display_name": aliceYtsaurus.SourceRaw["display_name"], }, } bobYtsaurusChangedEmail = YtsaurusUser{ - Username: "bobby:example.com", - SourceUser: AzureUser{ - AzureID: bobYtsaurus.SourceUser.(AzureUser).AzureID, - PrincipalName: bobAzureChangedEmail.PrincipalName, - Email: bobAzureChangedEmail.Email, - FirstName: bobYtsaurus.SourceUser.(AzureUser).FirstName, - LastName: bobYtsaurus.SourceUser.(AzureUser).LastName, - DisplayName: bobYtsaurus.SourceUser.(AzureUser).DisplayName, + Username: "bobby:example.com", + SourceType: ptr.String("azure"), + SourceRaw: map[string]any{ + "id": bobYtsaurus.SourceRaw["id"], + "principal_name": bobAzureChangedEmail.PrincipalName, + "email": bobAzureChangedEmail.Email, + "first_name": bobYtsaurus.SourceRaw["first_name"], + "last_name": bobYtsaurus.SourceRaw["last_name"], + "display_name": bobYtsaurus.SourceRaw["display_name"], }, } bobYtsaurusBanned = YtsaurusUser{ - Username: bobYtsaurus.Username, - SourceUser: AzureUser{ - AzureID: bobYtsaurus.SourceUser.(AzureUser).AzureID, - PrincipalName: bobYtsaurus.SourceUser.(AzureUser).PrincipalName, - Email: bobYtsaurus.SourceUser.(AzureUser).Email, - FirstName: bobYtsaurus.SourceUser.(AzureUser).FirstName, - LastName: bobYtsaurus.SourceUser.(AzureUser).LastName, - DisplayName: bobYtsaurus.SourceUser.(AzureUser).DisplayName, + Username: bobYtsaurus.Username, + SourceType: ptr.String("azure"), + SourceRaw: map[string]any{ + "id": bobYtsaurus.SourceRaw["id"], + "principal_name": bobYtsaurus.SourceRaw["principal_name"], + "email": bobYtsaurus.SourceRaw["email"], + "first_name": bobYtsaurus.SourceRaw["first_name"], + "last_name": bobYtsaurus.SourceRaw["last_name"], + "display_name": bobYtsaurus.SourceRaw["display_name"], }, BannedSince: initialTestTime, } carolYtsaurusBanned = YtsaurusUser{ - Username: carolYtsaurus.Username, - SourceUser: AzureUser{ - AzureID: carolYtsaurus.SourceUser.(AzureUser).AzureID, - PrincipalName: carolYtsaurus.SourceUser.(AzureUser).PrincipalName, - Email: carolYtsaurus.SourceUser.(AzureUser).Email, - FirstName: carolYtsaurus.SourceUser.(AzureUser).FirstName, - LastName: carolYtsaurus.SourceUser.(AzureUser).LastName, - DisplayName: carolYtsaurus.SourceUser.(AzureUser).DisplayName, + Username: carolYtsaurus.Username, + SourceType: ptr.String("azure"), + SourceRaw: map[string]any{ + "id": carolYtsaurus.SourceRaw["id"], + "principal_name": carolYtsaurus.SourceRaw["principal_name"], + "email": carolYtsaurus.SourceRaw["email"], + "first_name": carolYtsaurus.SourceRaw["first_name"], + "last_name": carolYtsaurus.SourceRaw["last_name"], + "display_name": carolYtsaurus.SourceRaw["display_name"], }, BannedSince: initialTestTime.Add(40 * time.Hour), } devsYtsaurusGroup = YtsaurusGroup{ - Name: "acme.devs", - SourceGroup: AzureGroup{ - AzureID: devsAzureGroup.AzureID, - DisplayName: "acme.devs|all", - Identity: "acme.devs|all", + Name: "acme.devs", + SourceType: ptr.String("azure"), + SourceRaw: map[string]any{ + "id": devsAzureGroup.AzureID, + "display_name": "acme.devs|all", + "identity": "acme.devs|all", }, } qaYtsaurusGroup = YtsaurusGroup{ - Name: "acme.qa", - SourceGroup: AzureGroup{ - AzureID: "fake-az-acme.qa", - DisplayName: "acme.qa|all", - Identity: "acme.qa", + Name: "acme.qa", + SourceType: ptr.String("azure"), + SourceRaw: map[string]any{ + "id": "fake-az-acme.qa", + "display_name": "acme.qa|all", + "identity": "acme.qa", }, } hqYtsaurusGroup = YtsaurusGroup{ - Name: "acme.hq", - SourceGroup: AzureGroup{ - AzureID: hqAzureGroup.AzureID, - DisplayName: "acme.hq", - Identity: "acme.hq", + Name: "acme.hq", + SourceType: ptr.String("azure"), + SourceRaw: map[string]any{ + "id": hqAzureGroup.AzureID, + "display_name": "acme.hq", + "identity": "acme.hq", }, } devsYtsaurusGroupChangedDisplayName = YtsaurusGroup{ - Name: "acme.developers", - SourceGroup: AzureGroup{ - AzureID: devsAzureGroup.AzureID, - DisplayName: "acme.developers|all", - Identity: "acme.developers|all", + Name: "acme.developers", + SourceType: ptr.String("azure"), + SourceRaw: map[string]any{ + "id": devsAzureGroup.AzureID, + "display_name": "acme.developers|all", + "identity": "acme.developers|all", }, } hqYtsaurusGroupChangedBackwardCompatible = YtsaurusGroup{ - Name: "acme.hq", - SourceGroup: AzureGroup{ - AzureID: hqAzureGroup.AzureID, - DisplayName: "acme.hq|all", - Identity: "acme.hq|all", + Name: "acme.hq", + SourceType: ptr.String("azure"), + SourceRaw: map[string]any{ + "id": hqAzureGroup.AzureID, + "display_name": "acme.hq|all", + "identity": "acme.hq|all", }, } @@ -647,7 +658,9 @@ func TestManageUnmanagedUsersIsForbidden(t *testing.T) { "Prevented attempt to change manual managed user", ) require.ErrorContains(t, - ytsaurus.UpdateUser(username, YtsaurusUser{Username: username, SourceUser: AzureUser{Email: "dummy@acme.com"}}), + ytsaurus.UpdateUser(username, YtsaurusUser{Username: username, SourceRaw: map[string]any{ + "email": "dummy@acme.com", + }}), "Prevented attempt to change manual managed user", ) } @@ -656,8 +669,7 @@ func TestManageUnmanagedUsersIsForbidden(t *testing.T) { func getAllYtsaurusObjects(t *testing.T, client yt.Client) (users []YtsaurusUser, groups []YtsaurusGroupWithMembers) { allUsers, err := doGetAllYtsaurusUsers(context.Background(), client, "azure") require.NoError(t, err) - fmt.Printf("HERE USERS: %v", allUsers) - allGroups, err := doGetAllYtsaurusGroupsWithMembers(context.Background(), client) + allGroups, err := doGetAllYtsaurusGroupsWithMembers(context.Background(), client, "azure") require.NoError(t, err) return allUsers, allGroups } @@ -670,7 +682,7 @@ func setupYtsaurusObjects(t *testing.T, client yt.Client, users []YtsaurusUser, context.Background(), client, user.Username, - buildUserAttributes(user), + buildUserAttributes(user, "azure"), ) require.NoError(t, err) } @@ -681,7 +693,7 @@ func setupYtsaurusObjects(t *testing.T, client yt.Client, users []YtsaurusUser, context.Background(), client, group.Name, - buildGroupAttributes(group.YtsaurusGroup), + buildGroupAttributes(group.YtsaurusGroup, "azure"), ) require.NoError(t, err) for member := range group.Members.Iter() { diff --git a/azure_models.go b/azure_models.go new file mode 100644 index 0000000..ac5aab5 --- /dev/null +++ b/azure_models.go @@ -0,0 +1,106 @@ +package main + +import ( + "go.ytsaurus.tech/yt/go/yson" +) + +type AzureUser struct { + // PrincipalName is unique human-readable Azure user field, used (possibly with changes) + // for the corresponding YTsaurus user's `name` attribute. + PrincipalName string `yson:"principal_name"` + + AzureID ObjectID `yson:"id"` + Email string `yson:"email"` + FirstName string `yson:"first_name"` + LastName string `yson:"last_name"` + DisplayName string `yson:"display_name"` +} + +func NewAzureUser(attributes map[string]any) (*AzureUser, error) { + bytes, err := yson.Marshal(attributes) + if err != nil { + return nil, err + } + + var azureUser AzureUser + err = yson.Unmarshal(bytes, &azureUser) + if err != nil { + return nil, err + } + return &azureUser, nil +} + +func (au AzureUser) GetID() ObjectID { + return au.AzureID +} + +func (au AzureUser) GetName() string { + return au.PrincipalName +} + +func (au AzureUser) GetSourceType() SourceType { + return AzureSourceType +} + +func (au AzureUser) GetRaw() (map[string]any, error) { + bytes, err := yson.Marshal(au) + if err != nil { + return nil, err + } + + raw := make(map[string]any) + err = yson.Unmarshal(bytes, &raw) + if err != nil { + return nil, err + } + return raw, nil +} + +type AzureGroup struct { + // Identity is unique human-readable Source user field, used (possibly with changes) + // for the corresponding YTsaurus user's `name` attribute. + Identity string `yson:"identity"` + + AzureID ObjectID `yson:"id"` + DisplayName string `yson:"display_name"` +} + +func NewAzureGroup(attributes map[string]any) (*AzureGroup, error) { + bytes, err := yson.Marshal(attributes) + if err != nil { + return nil, err + } + + var azureGroup AzureGroup + err = yson.Unmarshal(bytes, &azureGroup) + if err != nil { + return nil, err + } + return &azureGroup, nil +} + +func (ag AzureGroup) GetID() ObjectID { + return ag.AzureID +} + +func (ag AzureGroup) GetName() string { + return ag.Identity +} + +func (ag AzureGroup) GetSourceType() SourceType { + return AzureSourceType +} + +func (ag AzureGroup) GetRaw() (map[string]any, error) { + bytes, err := yson.Marshal(ag) + if err != nil { + return nil, err + } + + raw := make(map[string]any) + err = yson.Unmarshal(bytes, &raw) + if err != nil { + return nil, err + } + return raw, nil +} \ No newline at end of file diff --git a/diff.go b/diff.go index 84a7dfa..e5315aa 100644 --- a/diff.go +++ b/diff.go @@ -1,7 +1,9 @@ package main import ( + "bytes" "fmt" + "go.ytsaurus.tech/yt/go/yson" "strings" "time" @@ -48,17 +50,10 @@ func (a *App) syncUsers() (map[ObjectID]YtsaurusUser, error) { return nil, errors.Wrap(err, "failed to get YTsaurus users") } - sourceUsersMap := make(map[ObjectID]SourceUser) - ytUsersMap := make(map[ObjectID]YtsaurusUser) - - for _, user := range sourceUsers { - sourceUsersMap[user.GetID()] = user - } - for _, user := range ytUsers { - ytUsersMap[user.SourceUser.GetID()] = user + diff, err := a.diffUsers(sourceUsers, ytUsers) + if err != nil { + return nil, errors.Wrap(err, "failed to calculate users diff") } - - diff := a.diffUsers(sourceUsersMap, ytUsersMap) if a.isRemoveLimitReached(len(diff.remove)) { return nil, fmt.Errorf("delete limit in one cycle reached: %d %v", len(diff.remove), diff) } @@ -77,8 +72,6 @@ func (a *App) syncUsers() (map[ObjectID]YtsaurusUser, error) { if wasRemoved { removedCount++ } - // Actualizing user map for group sync later. - delete(ytUsersMap, user.SourceUser.GetID()) } for _, user := range diff.create { err = a.ytsaurus.CreateUser(user) @@ -86,8 +79,6 @@ func (a *App) syncUsers() (map[ObjectID]YtsaurusUser, error) { createErrCount++ a.logger.Errorw("failed to create user", zap.Error(err), "user", user) } - // Actualizing user map for group sync later. - ytUsersMap[user.SourceUser.GetID()] = user } for _, updatedUser := range diff.update { err = a.ytsaurus.UpdateUser(updatedUser.OldUsername, updatedUser.YtsaurusUser) @@ -95,8 +86,6 @@ func (a *App) syncUsers() (map[ObjectID]YtsaurusUser, error) { updateErrCount++ a.logger.Errorw("failed to update user", zap.Error(err), "user", updatedUser) } - // Actualizing user map for group sync later. - ytUsersMap[updatedUser.SourceUser.GetID()] = updatedUser.YtsaurusUser } a.logger.Infow("Finish syncing users", "created", len(diff.create)-createErrCount, @@ -107,7 +96,7 @@ func (a *App) syncUsers() (map[ObjectID]YtsaurusUser, error) { "banned", bannedCount, "ban_or_remove_errors", banOrremoveErrCount, ) - return ytUsersMap, nil + return diff.result, nil } func (a *App) syncGroups(usersMap map[ObjectID]YtsaurusUser) error { @@ -121,7 +110,10 @@ func (a *App) syncGroups(usersMap map[ObjectID]YtsaurusUser) error { return errors.Wrap(err, "failed to get YTsaurus groups") } - diff := a.diffGroups(azureGroups, ytGroups, usersMap) + diff, err := a.diffGroups(azureGroups, ytGroups, usersMap) + if err != nil { + return errors.Wrap(err, "failed to calculate groups diff") + } if a.isRemoveLimitReached(len(diff.groupsToRemove)) { return fmt.Errorf("delete limit in one cycle reached: %d %v", len(diff.groupsToRemove), diff) } @@ -192,29 +184,48 @@ type groupDiff struct { membersToRemove []YtsaurusMembership } +type existedGroup struct { + ytsaurusGroup YtsaurusGroupWithMembers + sourceGroup SourceGroup +} + func (a *App) diffGroups( sourceGroups []SourceGroupWithMembers, ytGroups []YtsaurusGroupWithMembers, usersMap map[ObjectID]YtsaurusUser, -) groupDiff { +) (*groupDiff, error) { var groupsToCreate, groupsToRemove []YtsaurusGroup var groupsToUpdate []UpdatedYtsaurusGroup var membersToAdd, membersToRemove []YtsaurusMembership sourceGroupsWithMembersMap := make(map[ObjectID]SourceGroupWithMembers) - ytGroupsWithMembersMap := make(map[ObjectID]YtsaurusGroupWithMembers) - for _, group := range sourceGroups { sourceGroupsWithMembersMap[group.SourceGroup.GetID()] = group } + + existedGroupsWithMembersMap := make(map[ObjectID]existedGroup) for _, group := range ytGroups { - ytGroupsWithMembersMap[group.SourceGroup.GetID()] = group + if group.IsManuallyManaged(a.source.GetSourceType()) { + continue + } + sourceType := SourceType(*group.SourceType) + switch sourceType { + case AzureSourceType: + sourceGroup, err := NewAzureGroup(group.SourceRaw) + if err != nil { + return nil, errors.Wrap(err, "failed to create azure group from source") + } + existedGroupsWithMembersMap[sourceGroup.GetID()] = existedGroup{group, sourceGroup} + } } // Collecting groups to create (the ones that exist in Source but not in YTsaurus). for objectID, sourceGroupWithMembers := range sourceGroupsWithMembersMap { - if _, ok := ytGroupsWithMembersMap[objectID]; !ok { - newYtsaurusGroup := a.buildYtsaurusGroup(sourceGroupWithMembers.SourceGroup) + if _, ok := existedGroupsWithMembersMap[objectID]; !ok { + newYtsaurusGroup, err := a.buildYtsaurusGroup(sourceGroupWithMembers.SourceGroup) + if err != nil { + return nil, errors.Wrap(err, "failed to build Ytsaurus group") + } groupsToCreate = append(groupsToCreate, newYtsaurusGroup) for username := range a.buildYtsaurusGroupMembers(sourceGroupWithMembers, usersMap).Iter() { membersToAdd = append(membersToAdd, YtsaurusMembership{ @@ -225,19 +236,22 @@ func (a *App) diffGroups( } } - for objectID, ytGroupWithMembers := range ytGroupsWithMembersMap { + for objectID, ytGroupWithMembers := range existedGroupsWithMembersMap { // Collecting groups to remove (the ones that exist in YTsaurus and not in Azure). sourceGroupWithMembers, ok := sourceGroupsWithMembersMap[objectID] if !ok { - groupsToRemove = append(groupsToRemove, ytGroupWithMembers.YtsaurusGroup) + groupsToRemove = append(groupsToRemove, ytGroupWithMembers.ytsaurusGroup.YtsaurusGroup) continue } // Collecting groups with changed Source fields (actually we have only displayName for now which // should change, though we still handle that just in case). - groupChanged, updatedYtGroup := a.isGroupChanged(sourceGroupWithMembers.SourceGroup, ytGroupWithMembers.YtsaurusGroup) + groupChanged, updatedYtGroup, err := a.isGroupChanged(sourceGroupWithMembers.SourceGroup, ytGroupWithMembers.ytsaurusGroup.YtsaurusGroup) + if err != nil { + return nil, errors.Wrap(err, "failed to check if group is changed") + } // Group name can change after update, so we ensure that correct one is used for membership updates. - actualGroupname := ytGroupWithMembers.YtsaurusGroup.Name + actualGroupname := ytGroupWithMembers.ytsaurusGroup.YtsaurusGroup.Name if groupChanged { // This shouldn't happen until we add more fields in YTsaurus' group @azure attribute. a.logger.Warnw( @@ -248,7 +262,7 @@ func (a *App) diffGroups( actualGroupname = updatedYtGroup.YtsaurusGroup.Name } - membersCreate, membersRemove := a.isGroupMembersChanged(sourceGroupWithMembers, ytGroupWithMembers, usersMap) + membersCreate, membersRemove := a.isGroupMembersChanged(sourceGroupWithMembers, ytGroupWithMembers.ytsaurusGroup, usersMap) for _, username := range membersCreate { membersToAdd = append(membersToAdd, YtsaurusMembership{ GroupName: actualGroupname, @@ -263,50 +277,100 @@ func (a *App) diffGroups( } } - return groupDiff{ + return &groupDiff{ groupsToCreate: groupsToCreate, groupsToUpdate: groupsToUpdate, groupsToRemove: groupsToRemove, membersToAdd: membersToAdd, membersToRemove: membersToRemove, - } + }, nil } type usersDiff struct { create []YtsaurusUser update []UpdatedYtsaurusUser remove []YtsaurusUser + result map[ObjectID]YtsaurusUser +} + +type existedUser struct { + ytsaurusUser YtsaurusUser + sourceUser SourceUser } func (a *App) diffUsers( - sourceUsersMap map[ObjectID]SourceUser, - ytUsersMap map[ObjectID]YtsaurusUser, -) usersDiff { + sourceUsers []SourceUser, + ytUsers []YtsaurusUser, +) (*usersDiff, error) { + sourceUsersMap := make(map[ObjectID]SourceUser) + for _, user := range sourceUsers { + sourceUsersMap[user.GetID()] = user + } + + ytUsersFromSourceMap := make(map[ObjectID]YtsaurusUser) + for _, user := range sourceUsers { + ytUser, err := a.buildYtsaurusUser(user) + if err != nil { + return nil, errors.Wrap(err, "failed to create Ytsaurus user from source user") + } + ytUsersFromSourceMap[user.GetID()] = ytUser + } + + resultUsersMap := make(map[ObjectID]YtsaurusUser) + + existedUsersMap := make(map[ObjectID]existedUser) + for _, user := range ytUsers { + if user.IsManuallyManaged(a.source.GetSourceType()) { + continue + } + sourceType := SourceType(*user.SourceType) + switch sourceType { + case AzureSourceType: + sourceUser, err := NewAzureUser(user.SourceRaw) + if err != nil { + return nil, errors.Wrap(err, "failed to create azure user from source") + } + existedUsersMap[sourceUser.GetID()] = existedUser{user, sourceUser} + resultUsersMap[sourceUser.GetID()] = user + } + } + var create, remove []YtsaurusUser var update []UpdatedYtsaurusUser for objectID, sourceUser := range sourceUsersMap { - if _, ok := ytUsersMap[objectID]; !ok { - create = append(create, a.buildYtsaurusUser(sourceUser)) + if _, ok := existedUsersMap[objectID]; !ok { + ytUser, err := a.buildYtsaurusUser(sourceUser) + if err != nil { + return nil, errors.Wrap(err, "failed to create Ytsaurus user from source user") + } + create = append(create, ytUser) + resultUsersMap[objectID] = ytUser } } - for objectID, ytUser := range ytUsersMap { - sourceUser, ok := sourceUsersMap[objectID] + + for objectID, user := range existedUsersMap { + ytUserFromSource, ok := ytUsersFromSourceMap[objectID] if !ok { - remove = append(remove, ytUser) + remove = append(remove, user.ytsaurusUser) continue } - userChanged, updatedYtUser := a.isUserChanged(sourceUser, ytUser) + userChanged, updatedYtUser, err := a.isUserChanged(ytUserFromSource, user.ytsaurusUser) + if err != nil { + return nil, errors.Wrap(err, "failed to check if user was changed") + } if !userChanged { continue } update = append(update, updatedYtUser) + resultUsersMap[objectID] = updatedYtUser.YtsaurusUser } - return usersDiff{ + return &usersDiff{ create: create, update: update, remove: remove, - } + result: resultUsersMap, + }, nil } func (a *App) buildUsername(sourceUser SourceUser) string { @@ -331,20 +395,47 @@ func (a *App) buildGroupName(sourceGroup SourceGroup) string { return name } -func (a *App) buildYtsaurusUser(sourceUser SourceUser) YtsaurusUser { +func (a *App) buildSourceUser(ytUser *YtsaurusUser) (SourceUser, error) { + if ytUser.IsManuallyManaged(a.source.GetSourceType()) { + return nil, errors.New("user is manually managed and can't be converted to source user") + } + sourceType := SourceType(*ytUser.SourceType) + switch sourceType { + case AzureSourceType: + { + return NewAzureUser(ytUser.SourceRaw) + } + } + return nil, errors.New("unknown user source type") +} + +func (a *App) buildYtsaurusUser(sourceUser SourceUser) (YtsaurusUser, error) { + sourceType := string(sourceUser.GetSourceType()) + sourceRaw, err := sourceUser.GetRaw() + if err != nil { + return YtsaurusUser{}, err + } return YtsaurusUser{ Username: a.buildUsername(sourceUser), - SourceUser: sourceUser, + SourceType: &sourceType, + SourceRaw: sourceRaw, // If we have Source user —> he is not banned. BannedSince: time.Time{}, - } + }, nil } -func (a *App) buildYtsaurusGroup(sourceGroup SourceGroup) YtsaurusGroup { - return YtsaurusGroup{ - Name: a.buildGroupName(sourceGroup), - SourceGroup: sourceGroup, +func (a *App) buildYtsaurusGroup(sourceGroup SourceGroup) (YtsaurusGroup, error) { + sourceType := string(sourceGroup.GetSourceType()) + sourceRaw, err := sourceGroup.GetRaw() + if err != nil { + return YtsaurusGroup{}, err } + + return YtsaurusGroup{ + Name: a.buildGroupName(sourceGroup), + SourceType: &sourceType, + SourceRaw: sourceRaw, + }, nil } func (a *App) buildYtsaurusGroupMembers(sourceGroupWithMembers SourceGroupWithMembers, usersMap map[ObjectID]YtsaurusUser) StringSet { @@ -368,12 +459,19 @@ type UpdatedYtsaurusUser struct { } // If isUserChanged detects that user is changed, it returns UpdatedYtsaurusUser. -func (a *App) isUserChanged(sourceUser SourceUser, ytUser YtsaurusUser) (bool, UpdatedYtsaurusUser) { - newUser := a.buildYtsaurusUser(sourceUser) - if newUser == ytUser { - return false, UpdatedYtsaurusUser{} +func (a *App) isUserChanged(newYtUser YtsaurusUser, ytUser YtsaurusUser) (bool, UpdatedYtsaurusUser, error) { + newSourceRaw, err := yson.Marshal(newYtUser.SourceRaw) + if err != nil { + return false, UpdatedYtsaurusUser{}, err } - return true, UpdatedYtsaurusUser{YtsaurusUser: newUser, OldUsername: ytUser.Username} + oldSourceRaw, err := yson.Marshal(ytUser.SourceRaw) + if err != nil { + return false, UpdatedYtsaurusUser{}, err + } + if newYtUser.Username == ytUser.Username && bytes.Equal(newSourceRaw, oldSourceRaw) && newYtUser.BannedSince == ytUser.BannedSince { + return false, UpdatedYtsaurusUser{}, nil + } + return true, UpdatedYtsaurusUser{YtsaurusUser: newYtUser, OldUsername: ytUser.Username}, nil } // UpdatedYtsaurusGroup is a wrapper for YtsaurusGroup, because it is handy to store old groupname for update, @@ -384,12 +482,23 @@ type UpdatedYtsaurusGroup struct { } // If isGroupChanged detects that group itself (not members) is changed, it returns UpdatedYtsaurusGroup. -func (a *App) isGroupChanged(sourceGroup SourceGroup, ytGroup YtsaurusGroup) (bool, UpdatedYtsaurusGroup) { - newGroup := a.buildYtsaurusGroup(sourceGroup) - if newGroup == ytGroup { - return false, UpdatedYtsaurusGroup{} +func (a *App) isGroupChanged(sourceGroup SourceGroup, ytGroup YtsaurusGroup) (bool, UpdatedYtsaurusGroup, error) { + newGroup, err := a.buildYtsaurusGroup(sourceGroup) + if err != nil { + return false, UpdatedYtsaurusGroup{}, errors.Wrap(err, "failed to build Ytsaurus group") + } + newSourceRaw, err := yson.Marshal(newGroup.SourceRaw) + if err != nil { + return false, UpdatedYtsaurusGroup{}, err + } + oldSourceRaw, err := yson.Marshal(ytGroup.SourceRaw) + if err != nil { + return false, UpdatedYtsaurusGroup{}, err + } + if bytes.Equal(newSourceRaw, oldSourceRaw) { + return false, UpdatedYtsaurusGroup{}, nil } - return true, UpdatedYtsaurusGroup{YtsaurusGroup: newGroup, OldName: ytGroup.Name} + return true, UpdatedYtsaurusGroup{YtsaurusGroup: newGroup, OldName: ytGroup.Name}, nil } // If isGroupMembersChanged detects that group members are changed, it returns lists of usernames to create and remove. diff --git a/migration_integration_test.go b/migration_integration_test.go index fded182..9050876 100644 --- a/migration_integration_test.go +++ b/migration_integration_test.go @@ -73,7 +73,7 @@ func TestUsersMigration(t *testing.T) { context.Background(), yt.client, username, - user.GetSourceAttributeName(), + "azure", attrValue, ) require.NoError(t, err) @@ -137,7 +137,7 @@ func TestGroupsMigration(t *testing.T) { context.Background(), yt.client, groupname, - group.GetSourceAttributeName(), + "azure", attrValue, ) require.NoError(t, err) diff --git a/source_models.go b/source_models.go index 2d20da1..83731ef 100644 --- a/source_models.go +++ b/source_models.go @@ -1,119 +1,8 @@ package main -import ( - "fmt" - - "go.ytsaurus.tech/yt/go/yson" -) - type ObjectID = string type SourceType string const ( AzureSourceType SourceType = "azure" ) - -type Source interface { - GetUsers() ([]SourceUser, error) - GetGroupsWithMembers() ([]SourceGroupWithMembers, error) - GetSourceType() SourceType -} - -type SourceUser interface { - GetID() ObjectID - GetName() string - GetSourceType() SourceType -} - -func NewSourceUser(sourceType SourceType, attributes map[string]any) (SourceUser, error) { - bytes, err := yson.Marshal(attributes) - if err != nil { - return nil, err - } - - switch sourceType { - case AzureSourceType: - var azureUser AzureUser - err = yson.Unmarshal(bytes, &azureUser) - if err != nil { - return nil, err - } - return azureUser, nil - } - return nil, fmt.Errorf("unknown source type: %v", sourceType) -} - -type AzureUser struct { - // PrincipalName is unique human-readable Azure user field, used (possibly with changes) - // for the corresponding YTsaurus user's `name` attribute. - PrincipalName string `yson:"principal_name"` - - AzureID ObjectID `yson:"id"` - Email string `yson:"email"` - FirstName string `yson:"first_name"` - LastName string `yson:"last_name"` - DisplayName string `yson:"display_name"` -} - -func (user AzureUser) GetID() ObjectID { - return user.AzureID -} - -func (user AzureUser) GetName() string { - return user.PrincipalName -} - -func (user AzureUser) GetSourceType() SourceType { - return AzureSourceType -} - -type SourceGroup interface { - GetID() ObjectID - GetName() string - GetSourceType() SourceType -} - -func NewSourceGroup(sourceType SourceType, attributes map[string]any) (SourceGroup, error) { - bytes, err := yson.Marshal(attributes) - if err != nil { - return nil, err - } - - switch sourceType { - case AzureSourceType: - var azureGroup AzureGroup - err = yson.Unmarshal(bytes, &azureGroup) - if err != nil { - return nil, err - } - return azureGroup, nil - } - return nil, fmt.Errorf("unknown source type: %v", sourceType) -} - -type AzureGroup struct { - // Identity is unique human-readable Source user field, used (possibly with changes) - // for the corresponding YTsaurus user's `name` attribute. - Identity string `yson:"identity"` - - AzureID ObjectID `yson:"id"` - DisplayName string `yson:"display_name"` -} - -func (ag AzureGroup) GetID() ObjectID { - return ag.AzureID -} - -func (ag AzureGroup) GetName() string { - return ag.Identity -} - -func (ag AzureGroup) GetSourceType() SourceType { - return AzureSourceType -} - -type SourceGroupWithMembers struct { - SourceGroup SourceGroup - // Members is a set of strings, representing users' ObjectID. - Members StringSet -} diff --git a/ytsaurus.go b/ytsaurus.go index 8407cef..fa32d8e 100644 --- a/ytsaurus.go +++ b/ytsaurus.go @@ -122,8 +122,8 @@ func (y *Ytsaurus) CreateUser(user YtsaurusUser) error { y.client, user.Username, map[string]any{ - user.GetSourceAttributeName(): user.SourceUser, - "source_type": user.SourceUser.GetSourceType(), + y.sourceAttributeName: user.SourceRaw, + "source_type": user.SourceType, }, ) } @@ -151,7 +151,7 @@ func (y *Ytsaurus) UpdateUser(username string, user YtsaurusUser) error { ctx, y.client, username, - buildUserAttributes(user), + buildUserAttributes(user, y.sourceAttributeName), ) } @@ -207,7 +207,7 @@ func (y *Ytsaurus) GetGroupsWithMembers(sourceType SourceType) ([]YtsaurusGroupW ctx, cancel := context.WithTimeout(context.Background(), y.timeout) defer cancel() - groups, err := doGetAllYtsaurusGroupsWithMembers(ctx, y.client) + groups, err := doGetAllYtsaurusGroupsWithMembers(ctx, y.client, y.sourceAttributeName) if err != nil { return nil, errors.Wrap(err, "failed to get ytsaurus groups") } @@ -242,8 +242,8 @@ func (y *Ytsaurus) CreateGroup(group YtsaurusGroup) error { y.client, group.Name, map[string]any{ - "source_type": group.SourceGroup.GetSourceType(), - group.GetSourceAttributeName(): group.SourceGroup, + "source_type": group.SourceType, + y.sourceAttributeName: group.SourceRaw, }, ) } @@ -270,7 +270,7 @@ func (y *Ytsaurus) UpdateGroup(groupname string, group YtsaurusGroup) error { ctx, y.client, groupname, - buildGroupAttributes(group), + buildGroupAttributes(group, y.sourceAttributeName), ) } diff --git a/ytsaurus_helpers.go b/ytsaurus_helpers.go index 10303ae..352ee31 100644 --- a/ytsaurus_helpers.go +++ b/ytsaurus_helpers.go @@ -16,6 +16,8 @@ const ( bannedSinceAttributeName = "banned_since" bannedAttributeName = "banned" sourceTypeAttributeName = "source_type" + membersAttributeName = "members" + nameAttributeName = "name" ) func doGetAllYtsaurusUsers(ctx context.Context, client yt.Client, sourceAttributeName string) ([]YtsaurusUser, error) { @@ -44,38 +46,38 @@ func doGetAllYtsaurusUsers(ctx context.Context, client yt.Client, sourceAttribut var users []YtsaurusUser for _, ytUser := range response { - var bannedSince time.Time - if bannedSinceRaw, ok := ytUser.Attrs[bannedSinceAttributeName]; ok && bannedSinceRaw != "" { - bannedSince, err = time.Parse(appTimeFormat, bannedSinceRaw.(string)) - if err != nil { - return nil, errors.Wrapf(err, "failed to parse @banned_since. %v", ytUser) - } + user := YtsaurusUser{ + Username: ytUser.Name, } - var sourceUser SourceUser - if sourceRaw, ok := ytUser.Attrs[sourceAttributeName]; ok { - sourceType := AzureSourceType - if sourceTypeRaw, ok := ytUser.Attrs[sourceTypeAttributeName]; ok { - sourceType = SourceType(sourceTypeRaw.(string)) + + if ytUser.Attrs != nil { + if bannedSinceRaw, ok := ytUser.Attrs[bannedSinceAttributeName]; ok && bannedSinceRaw != "" { + user.BannedSince, err = time.Parse(appTimeFormat, bannedSinceRaw.(string)) + if err != nil { + return nil, errors.Wrapf(err, "failed to parse @banned_since. %v", ytUser) + } + } + if sourceRaw, ok := ytUser.Attrs[sourceAttributeName]; ok { + // For backward compatibility only. + sourceType := string(AzureSourceType) + if sourceTypeRaw, ok := ytUser.Attrs[sourceTypeAttributeName]; ok { + sourceType = sourceTypeRaw.(string) + } + + user.SourceType = &sourceType + user.SourceRaw = sourceRaw.(map[string]any) } - sourceUser, err = NewSourceUser(sourceType, sourceRaw.(map[string]any)) } - users = append(users, YtsaurusUser{ - Username: ytUser.Name, - SourceUser: sourceUser, - BannedSince: bannedSince, - }) + users = append(users, user) } return users, nil } -func doGetAllYtsaurusGroupsWithMembers(ctx context.Context, client yt.Client) ([]YtsaurusGroupWithMembers, error) { +func doGetAllYtsaurusGroupsWithMembers(ctx context.Context, client yt.Client, sourceAttributeName string) ([]YtsaurusGroupWithMembers, error) { type YtsaurusGroupReponse struct { - Name string `yson:",value"` - SourceType *SourceType `yson:"source_type,attr"` - Azure *AzureGroup `yson:"azure,attr"` - Source map[string]any `yson:"source,attr"` - Members []string `yson:"members,attr"` + Name string `yson:",value"` + Attrs map[string]any `yson:",attrs"` } var response []YtsaurusGroupReponse @@ -85,10 +87,9 @@ func doGetAllYtsaurusGroupsWithMembers(ctx context.Context, client yt.Client) ([ &response, &yt.ListNodeOptions{ Attributes: []string{ - "members", - "azure", - "source", - "source_type", + membersAttributeName, + sourceAttributeName, + sourceTypeAttributeName, }, }, ) @@ -99,26 +100,31 @@ func doGetAllYtsaurusGroupsWithMembers(ctx context.Context, client yt.Client) ([ var groups []YtsaurusGroupWithMembers for _, ytGroup := range response { members := NewStringSet() - for _, m := range ytGroup.Members { - members.Add(m) - } - var sourceGroup SourceGroup - if ytGroup.Azure != nil { - sourceGroup = *ytGroup.Azure - } else if ytGroup.SourceType != nil && ytGroup.Source != nil { - sourceGroup, err = NewSourceGroup(*ytGroup.SourceType, ytGroup.Source) - if err != nil { - return nil, errors.Wrapf(err, "failed to create source group. %v", ytGroup) + group := YtsaurusGroup{Name: ytGroup.Name} + + if ytGroup.Attrs != nil { + if membersRaw, ok := ytGroup.Attrs[membersAttributeName]; ok { + for _, m := range membersRaw.([]interface{}) { + members.Add(m.(string)) + } + } + + if sourceRaw, ok := ytGroup.Attrs[sourceAttributeName]; ok { + // For backward compatibility only. + sourceType := string(AzureSourceType) + if sourceTypeRaw, ok := ytGroup.Attrs[sourceTypeAttributeName]; ok { + sourceType = sourceTypeRaw.(string) + } + + group.SourceType = &sourceType + group.SourceRaw = sourceRaw.(map[string]any) } } groups = append(groups, YtsaurusGroupWithMembers{ - YtsaurusGroup: YtsaurusGroup{ - Name: ytGroup.Name, - SourceGroup: sourceGroup, - }, - Members: members, + YtsaurusGroup: group, + Members: members, }) } return groups, nil @@ -128,7 +134,7 @@ func doCreateYtsaurusUser(ctx context.Context, client yt.Client, username string if attrs == nil { attrs = make(map[string]any) } - attrs["name"] = username + attrs[nameAttributeName] = username _, err := client.CreateObject( ctx, yt.NodeUser, @@ -141,7 +147,7 @@ func doCreateYtsaurusGroup(ctx context.Context, client yt.Client, name string, a if attrs == nil { attrs = make(map[string]any) } - attrs["name"] = name + attrs[nameAttributeName] = name _, err := client.CreateObject( ctx, yt.NodeGroup, @@ -168,21 +174,21 @@ func doRemoveMemberYtsaurusGroup(ctx context.Context, client yt.Client, username ) } -func buildUserAttributes(user YtsaurusUser) map[string]any { +func buildUserAttributes(user YtsaurusUser, sourceAttributeName string) map[string]any { return map[string]any{ - "name": user.Username, - "banned_since": user.BannedSinceString(), - "banned": user.IsBanned(), - user.GetSourceAttributeName(): user.SourceUser, - "source_type": user.SourceUser.GetSourceType(), + nameAttributeName: user.Username, + bannedSinceAttributeName: user.BannedSinceString(), + bannedAttributeName: user.IsBanned(), + sourceAttributeName: user.SourceRaw, + sourceTypeAttributeName: user.SourceType, } } -func buildGroupAttributes(group YtsaurusGroup) map[string]any { +func buildGroupAttributes(group YtsaurusGroup, sourceAttributeName string) map[string]any { return map[string]any{ - group.GetSourceAttributeName(): group.SourceGroup, - "name": group.Name, - "source_type": group.SourceGroup.GetSourceType(), + sourceAttributeName: group.SourceRaw, + nameAttributeName: group.Name, + sourceTypeAttributeName: group.SourceType, } } @@ -199,7 +205,7 @@ func doSetAzureAttributeForYtsaurusUser(ctx context.Context, client yt.Client, u func doSetAttributesForYtsaurusUser(ctx context.Context, client yt.Client, username string, attrs map[string]any) error { attrsCopy := make(map[string]any) for key, value := range attrs { - if key == "name" && value == username { + if key == nameAttributeName && value == username { // multiset_attributes returns an error: // `setting builtin attribute "name" ... user ... already exists` // on attempt to set same name for the existing user. diff --git a/ytsaurus_models.go b/ytsaurus_models.go index bb90320..326a0f7 100644 --- a/ytsaurus_models.go +++ b/ytsaurus_models.go @@ -4,24 +4,37 @@ import ( "time" ) +type SourceUser interface { + GetID() ObjectID + GetName() string + GetSourceType() SourceType + GetRaw() (map[string]any, error) +} + +type SourceGroup interface { + GetID() ObjectID + GetName() string + GetSourceType() SourceType + GetRaw() (map[string]any, error) +} + +type SourceGroupWithMembers struct { + SourceGroup SourceGroup + // Members is a set of strings, representing users' ObjectID. + Members StringSet +} + type YtsaurusUser struct { // Username is a unique @name attribute of a user. Username string - SourceUser SourceUser + SourceType *string + SourceRaw map[string]any BannedSince time.Time } -func (u YtsaurusUser) GetSourceAttributeName() string { - switch u.SourceUser.GetSourceType() { - case AzureSourceType: - return "azure" - } - return "source" -} - // IsManuallyManaged true if user doesn't have @azure attribute (system or manually created user). func (u YtsaurusUser) IsManuallyManaged(sourceType SourceType) bool { - return u.SourceUser == nil || u.SourceUser.GetSourceType() != sourceType + return u.SourceRaw == nil || u.SourceType == nil || *u.SourceType != string(sourceType) } func (u YtsaurusUser) IsBanned() bool { @@ -37,21 +50,14 @@ func (u YtsaurusUser) BannedSinceString() string { type YtsaurusGroup struct { // Name is a unique @name attribute of a group. - Name string - SourceGroup SourceGroup -} - -func (g YtsaurusGroup) GetSourceAttributeName() string { - switch g.SourceGroup.GetSourceType() { - case AzureSourceType: - return "azure" - } - return "source" + Name string + SourceType *string + SourceRaw map[string]any } // IsManuallyManaged true if group doesn't have @azure attribute (system or manually created group). func (g YtsaurusGroup) IsManuallyManaged(sourceType SourceType) bool { - return g.SourceGroup == nil || g.SourceGroup.GetSourceType() != sourceType + return g.SourceRaw == nil || g.SourceType == nil || *g.SourceType != string(sourceType) } type YtsaurusGroupWithMembers struct { diff --git a/ytsaurus_test.go b/ytsaurus_test.go index 32eb425..37d247d 100644 --- a/ytsaurus_test.go +++ b/ytsaurus_test.go @@ -2,6 +2,7 @@ package main import ( "context" + "go.ytsaurus.tech/library/go/ptr" "os" "testing" "time" @@ -18,12 +19,13 @@ func getYtsaurus(t *testing.T, ytLocal *YtsaurusLocal) *Ytsaurus { require.NoError(t, os.Setenv("YT_TOKEN", ytDevToken)) yt, err := NewYtsaurus( &YtsaurusConfig{ - Proxy: ytLocal.GetProxy(), - Timeout: 10 * time.Minute, - LogLevel: "DEBUG", - ApplyUserChanges: true, - ApplyGroupChanges: true, - ApplyMemberChanges: true, + Proxy: ytLocal.GetProxy(), + Timeout: 10 * time.Minute, + LogLevel: "DEBUG", + ApplyUserChanges: true, + ApplyGroupChanges: true, + ApplyMemberChanges: true, + SourceAttributeName: ptr.String("azure"), }, getDevelopmentLogger(), clock.RealClock{}, ) @@ -44,20 +46,20 @@ func TestUpdateUserFirstName(t *testing.T) { const azureID = "fake-az-id-old" managedOleg := YtsaurusUser{ - Username: "oleg", - SourceUser: AzureUser{ - AzureID: azureID, - FirstName: "Lego", + Username: "oleg", + SourceType: ptr.String("azure"), + SourceRaw: map[string]any{ + "id": azureID, + "first_name": "Lego", }, } err := yt.CreateUser(managedOleg) require.NoError(t, err) - updateSourceUser := AzureUser{ - AzureID: azureID, - FirstName: "Oleg", + managedOleg.SourceRaw = map[string]any{ + "id": azureID, + "first_name": "Oleg", } - managedOleg.SourceUser = updateSourceUser updErr := yt.UpdateUser(managedOleg.Username, managedOleg) @@ -87,19 +89,21 @@ func TestGroups(t *testing.T) { require.Empty(t, groupsInitial) managedOleg := YtsaurusUser{ - Username: "oleg", - SourceUser: AzureUser{ - AzureID: "fake-az-id-oleg", + Username: "oleg", + SourceType: ptr.String("azure"), + SourceRaw: map[string]any{ + "id": "fake-az-id-oleg", }, } err = yt.CreateUser(managedOleg) require.NoError(t, err) managedOlegsGroup := YtsaurusGroup{ - Name: "olegs", - SourceGroup: AzureGroup{ - AzureID: "fake-az-id-olegs", - DisplayName: "This is group is for Olegs only", + Name: "olegs", + SourceType: ptr.String("azure"), + SourceRaw: map[string]any{ + "id": "fake-az-id-olegs", + "display_name": "This is group is for Olegs only", }, } err = yt.CreateGroup(managedOlegsGroup) @@ -115,10 +119,11 @@ func TestGroups(t *testing.T) { require.Equal(t, []YtsaurusGroupWithMembers{ { YtsaurusGroup: YtsaurusGroup{ - Name: managedOlegsGroup.Name, - SourceGroup: AzureGroup{ - AzureID: managedOlegsGroup.SourceGroup.(AzureGroup).AzureID, - DisplayName: managedOlegsGroup.SourceGroup.(AzureGroup).DisplayName, + Name: managedOlegsGroup.Name, + SourceType: ptr.String("azure"), + SourceRaw: map[string]any{ + "id": managedOlegsGroup.SourceRaw["id"], + "display_name": managedOlegsGroup.SourceRaw["display_name"], }, }, Members: members, @@ -133,10 +138,11 @@ func TestGroups(t *testing.T) { require.Equal(t, []YtsaurusGroupWithMembers{ { YtsaurusGroup: YtsaurusGroup{ - Name: managedOlegsGroup.Name, - SourceGroup: AzureGroup{ - AzureID: managedOlegsGroup.SourceGroup.(AzureGroup).AzureID, - DisplayName: managedOlegsGroup.SourceGroup.(AzureGroup).DisplayName, + Name: managedOlegsGroup.Name, + SourceType: ptr.String("azure"), + SourceRaw: map[string]any{ + "id": managedOlegsGroup.SourceRaw["id"], + "display_name": managedOlegsGroup.SourceRaw["display_name"], }, }, Members: NewStringSet(), From c5b4fee373ca70820b1f0ffb4546030bece6b02c Mon Sep 17 00:00:00 2001 From: Nadezhda Savchenko Date: Tue, 5 Mar 2024 18:42:31 +0300 Subject: [PATCH 16/21] Fix golangci --- .golangci.yml | 3 --- go.mod | 2 +- 2 files changed, 1 insertion(+), 4 deletions(-) diff --git a/.golangci.yml b/.golangci.yml index 6799ee0..dcb74cc 100644 --- a/.golangci.yml +++ b/.golangci.yml @@ -89,9 +89,6 @@ linters-settings: gofmt: simplify: false # gofmt with `-s` option, true by default - goimports: - local-prefixes: go.ytsaurus.tech - goconst: # minimal length of string constant, 3 by default min-len: 3 diff --git a/go.mod b/go.mod index cad6dfe..baf7029 100644 --- a/go.mod +++ b/go.mod @@ -14,6 +14,7 @@ require ( github.com/stretchr/testify v1.8.4 github.com/testcontainers/testcontainers-go v0.28.0 go.uber.org/zap v1.26.0 + go.ytsaurus.tech/library/go/ptr v0.0.1 go.ytsaurus.tech/yt/go v0.0.13 gopkg.in/yaml.v3 v3.0.1 k8s.io/utils v0.0.0-20230220204549-a5ecb0141aa5 @@ -89,7 +90,6 @@ require ( go.ytsaurus.tech/library/go/blockcodecs v0.0.2 // indirect go.ytsaurus.tech/library/go/core/log v0.0.3 // indirect go.ytsaurus.tech/library/go/core/xerrors v0.0.3 // indirect - go.ytsaurus.tech/library/go/ptr v0.0.1 // indirect go.ytsaurus.tech/library/go/x/xreflect v0.0.2 // indirect go.ytsaurus.tech/library/go/x/xruntime v0.0.3 // indirect golang.org/x/crypto v0.17.0 // indirect From 9261dc18844f5b5e33f5bf1dfa32e4099e9956b0 Mon Sep 17 00:00:00 2001 From: Nadezhda Savchenko Date: Wed, 6 Mar 2024 11:28:09 +0300 Subject: [PATCH 17/21] Add buildYtsaurusGroup helper --- app_test.go | 3 ++- azure_models.go | 2 +- diff.go | 44 ++++++++++++++++++++++++++------------------ ytsaurus.go | 3 ++- ytsaurus_test.go | 3 ++- 5 files changed, 33 insertions(+), 22 deletions(-) diff --git a/app_test.go b/app_test.go index 871fa84..770b975 100644 --- a/app_test.go +++ b/app_test.go @@ -2,11 +2,12 @@ package main import ( "context" - "go.ytsaurus.tech/library/go/ptr" "os" "testing" "time" + "go.ytsaurus.tech/library/go/ptr" + "github.com/google/go-cmp/cmp" "github.com/google/go-cmp/cmp/cmpopts" "github.com/stretchr/testify/require" diff --git a/azure_models.go b/azure_models.go index ac5aab5..83b5c76 100644 --- a/azure_models.go +++ b/azure_models.go @@ -103,4 +103,4 @@ func (ag AzureGroup) GetRaw() (map[string]any, error) { return nil, err } return raw, nil -} \ No newline at end of file +} diff --git a/diff.go b/diff.go index e5315aa..3956d19 100644 --- a/diff.go +++ b/diff.go @@ -3,10 +3,11 @@ package main import ( "bytes" "fmt" - "go.ytsaurus.tech/yt/go/yson" "strings" "time" + "go.ytsaurus.tech/yt/go/yson" + "github.com/pkg/errors" "go.uber.org/zap" ) @@ -208,15 +209,12 @@ func (a *App) diffGroups( if group.IsManuallyManaged(a.source.GetSourceType()) { continue } - sourceType := SourceType(*group.SourceType) - switch sourceType { - case AzureSourceType: - sourceGroup, err := NewAzureGroup(group.SourceRaw) - if err != nil { - return nil, errors.Wrap(err, "failed to create azure group from source") - } - existedGroupsWithMembersMap[sourceGroup.GetID()] = existedGroup{group, sourceGroup} + + sourceGroup, err := a.buildSourceGroup(&group) + if err != nil { + return nil, errors.Wrap(err, "failed to create azure group from source") } + existedGroupsWithMembersMap[sourceGroup.GetID()] = existedGroup{group, sourceGroup} } // Collecting groups to create (the ones that exist in Source but not in YTsaurus). @@ -323,16 +321,12 @@ func (a *App) diffUsers( if user.IsManuallyManaged(a.source.GetSourceType()) { continue } - sourceType := SourceType(*user.SourceType) - switch sourceType { - case AzureSourceType: - sourceUser, err := NewAzureUser(user.SourceRaw) - if err != nil { - return nil, errors.Wrap(err, "failed to create azure user from source") - } - existedUsersMap[sourceUser.GetID()] = existedUser{user, sourceUser} - resultUsersMap[sourceUser.GetID()] = user + sourceUser, err := a.buildSourceUser(&user) + if err != nil { + return nil, errors.Wrap(err, "failed to create azure user from source") } + existedUsersMap[sourceUser.GetID()] = existedUser{user, sourceUser} + resultUsersMap[sourceUser.GetID()] = user } var create, remove []YtsaurusUser @@ -409,6 +403,20 @@ func (a *App) buildSourceUser(ytUser *YtsaurusUser) (SourceUser, error) { return nil, errors.New("unknown user source type") } +func (a *App) buildSourceGroup(ytGroup *YtsaurusGroupWithMembers) (SourceUser, error) { + if ytGroup.IsManuallyManaged(a.source.GetSourceType()) { + return nil, errors.New("user is manually managed and can't be converted to source user") + } + sourceType := SourceType(*ytGroup.SourceType) + switch sourceType { + case AzureSourceType: + { + return NewAzureGroup(ytGroup.SourceRaw) + } + } + return nil, errors.New("unknown user source type") +} + func (a *App) buildYtsaurusUser(sourceUser SourceUser) (YtsaurusUser, error) { sourceType := string(sourceUser.GetSourceType()) sourceRaw, err := sourceUser.GetRaw() diff --git a/ytsaurus.go b/ytsaurus.go index fa32d8e..b9f65c3 100644 --- a/ytsaurus.go +++ b/ytsaurus.go @@ -2,10 +2,11 @@ package main import ( "context" - "go.ytsaurus.tech/library/go/ptr" "os" "time" + "go.ytsaurus.tech/library/go/ptr" + "github.com/pkg/errors" "k8s.io/utils/clock" diff --git a/ytsaurus_test.go b/ytsaurus_test.go index 37d247d..909901c 100644 --- a/ytsaurus_test.go +++ b/ytsaurus_test.go @@ -2,11 +2,12 @@ package main import ( "context" - "go.ytsaurus.tech/library/go/ptr" "os" "testing" "time" + "go.ytsaurus.tech/library/go/ptr" + "github.com/stretchr/testify/require" "k8s.io/utils/clock" From 2690bae9ca59dc94843b8278ef5b1cfd3e256a82 Mon Sep 17 00:00:00 2001 From: Nadezhda Savchenko Date: Thu, 7 Mar 2024 16:08:07 +0300 Subject: [PATCH 18/21] Review fixes --- app.go | 5 ++- app_test.go | 44 ++++++++--------------- azure_fake.go | 8 +++-- azure_models.go | 8 ----- azure_real.go | 14 +++++--- azure_real_integration_test.go | 4 +-- config.go | 4 +-- diff.go | 65 +++++++++++++++------------------- source_models.go | 7 ---- util.go | 5 +-- ytsaurus.go | 52 +++++++-------------------- ytsaurus_helpers.go | 23 ++---------- ytsaurus_models.go | 34 ++++-------------- ytsaurus_test.go | 27 ++++++-------- 14 files changed, 99 insertions(+), 201 deletions(-) diff --git a/app.go b/app.go index 171f634..5aca535 100644 --- a/app.go +++ b/app.go @@ -11,10 +11,13 @@ import ( "k8s.io/utils/clock" ) +type ObjectID = string + type Source interface { GetUsers() ([]SourceUser, error) GetGroupsWithMembers() ([]SourceGroupWithMembers, error) - GetSourceType() SourceType + CreateUserFromRaw(raw map[string]any) (SourceUser, error) + CreateGroupFromRaw(raw map[string]any) (SourceGroup, error) } type App struct { diff --git a/app_test.go b/app_test.go index 770b975..704a714 100644 --- a/app_test.go +++ b/app_test.go @@ -6,8 +6,6 @@ import ( "testing" "time" - "go.ytsaurus.tech/library/go/ptr" - "github.com/google/go-cmp/cmp" "github.com/google/go-cmp/cmp/cmpopts" "github.com/stretchr/testify/require" @@ -100,8 +98,7 @@ var ( } aliceYtsaurus = YtsaurusUser{ - Username: "alice", - SourceType: ptr.String("azure"), + Username: "alice", SourceRaw: map[string]any{ "id": aliceAzure.AzureID, "principal_name": aliceAzure.PrincipalName, @@ -112,8 +109,7 @@ var ( }, } bobYtsaurus = YtsaurusUser{ - Username: "bob", - SourceType: ptr.String("azure"), + Username: "bob", SourceRaw: map[string]any{ "id": bobAzure.AzureID, "principal_name": bobAzure.PrincipalName, @@ -124,8 +120,7 @@ var ( }, } carolYtsaurus = YtsaurusUser{ - Username: "carol", - SourceType: ptr.String("azure"), + Username: "carol", SourceRaw: map[string]any{ "id": carolAzure.AzureID, "principal_name": carolAzure.PrincipalName, @@ -136,8 +131,7 @@ var ( }, } aliceYtsaurusChangedLastName = YtsaurusUser{ - Username: aliceYtsaurus.Username, - SourceType: ptr.String("azure"), + Username: aliceYtsaurus.Username, SourceRaw: map[string]any{ "id": aliceYtsaurus.SourceRaw["id"], "principal_name": aliceYtsaurus.SourceRaw["principal_name"], @@ -148,8 +142,7 @@ var ( }, } bobYtsaurusChangedEmail = YtsaurusUser{ - Username: "bobby:example.com", - SourceType: ptr.String("azure"), + Username: "bobby:example.com", SourceRaw: map[string]any{ "id": bobYtsaurus.SourceRaw["id"], "principal_name": bobAzureChangedEmail.PrincipalName, @@ -160,8 +153,7 @@ var ( }, } bobYtsaurusBanned = YtsaurusUser{ - Username: bobYtsaurus.Username, - SourceType: ptr.String("azure"), + Username: bobYtsaurus.Username, SourceRaw: map[string]any{ "id": bobYtsaurus.SourceRaw["id"], "principal_name": bobYtsaurus.SourceRaw["principal_name"], @@ -173,8 +165,7 @@ var ( BannedSince: initialTestTime, } carolYtsaurusBanned = YtsaurusUser{ - Username: carolYtsaurus.Username, - SourceType: ptr.String("azure"), + Username: carolYtsaurus.Username, SourceRaw: map[string]any{ "id": carolYtsaurus.SourceRaw["id"], "principal_name": carolYtsaurus.SourceRaw["principal_name"], @@ -186,8 +177,7 @@ var ( BannedSince: initialTestTime.Add(40 * time.Hour), } devsYtsaurusGroup = YtsaurusGroup{ - Name: "acme.devs", - SourceType: ptr.String("azure"), + Name: "acme.devs", SourceRaw: map[string]any{ "id": devsAzureGroup.AzureID, "display_name": "acme.devs|all", @@ -195,8 +185,7 @@ var ( }, } qaYtsaurusGroup = YtsaurusGroup{ - Name: "acme.qa", - SourceType: ptr.String("azure"), + Name: "acme.qa", SourceRaw: map[string]any{ "id": "fake-az-acme.qa", "display_name": "acme.qa|all", @@ -204,8 +193,7 @@ var ( }, } hqYtsaurusGroup = YtsaurusGroup{ - Name: "acme.hq", - SourceType: ptr.String("azure"), + Name: "acme.hq", SourceRaw: map[string]any{ "id": hqAzureGroup.AzureID, "display_name": "acme.hq", @@ -213,8 +201,7 @@ var ( }, } devsYtsaurusGroupChangedDisplayName = YtsaurusGroup{ - Name: "acme.developers", - SourceType: ptr.String("azure"), + Name: "acme.developers", SourceRaw: map[string]any{ "id": devsAzureGroup.AzureID, "display_name": "acme.developers|all", @@ -222,8 +209,7 @@ var ( }, } hqYtsaurusGroupChangedBackwardCompatible = YtsaurusGroup{ - Name: "acme.hq", - SourceType: ptr.String("azure"), + Name: "acme.hq", SourceRaw: map[string]any{ "id": hqAzureGroup.AzureID, "display_name": "acme.hq|all", @@ -497,9 +483,9 @@ var ( ), }, { - // for this group only displayName should be updated + // For this group only displayName should be updated. SourceGroup: hqAzureGroupChangedBackwardCompatible, - // members also changed + // Members also changed. Members: NewStringSetFromItems( aliceAzure.AzureID, carolAzure.AzureID, @@ -583,7 +569,7 @@ func TestAppSync(t *testing.T) { ApplyGroupChanges: true, ApplyMemberChanges: true, LogLevel: "DEBUG", - SourceAttributeName: ptr.String("azure"), + SourceAttributeName: "azure", }, }, getDevelopmentLogger(), azure, diff --git a/azure_fake.go b/azure_fake.go index 9986bde..528e8b6 100644 --- a/azure_fake.go +++ b/azure_fake.go @@ -17,8 +17,12 @@ func (a *AzureFake) setGroups(groups []SourceGroupWithMembers) { a.groups = groups } -func (a *AzureFake) GetSourceType() SourceType { - return AzureSourceType +func (a *AzureFake) CreateUserFromRaw(raw map[string]any) (SourceUser, error) { + return NewAzureUser(raw) +} + +func (a *AzureFake) CreateGroupFromRaw(raw map[string]any) (SourceGroup, error) { + return NewAzureGroup(raw) } func (a *AzureFake) GetUsers() ([]SourceUser, error) { diff --git a/azure_models.go b/azure_models.go index 83b5c76..07d7caf 100644 --- a/azure_models.go +++ b/azure_models.go @@ -38,10 +38,6 @@ func (au AzureUser) GetName() string { return au.PrincipalName } -func (au AzureUser) GetSourceType() SourceType { - return AzureSourceType -} - func (au AzureUser) GetRaw() (map[string]any, error) { bytes, err := yson.Marshal(au) if err != nil { @@ -87,10 +83,6 @@ func (ag AzureGroup) GetName() string { return ag.Identity } -func (ag AzureGroup) GetSourceType() SourceType { - return AzureSourceType -} - func (ag AzureGroup) GetRaw() (map[string]any, error) { bytes, err := yson.Marshal(ag) if err != nil { diff --git a/azure_real.go b/azure_real.go index 28fd89e..a3e29ea 100644 --- a/azure_real.go +++ b/azure_real.go @@ -17,8 +17,10 @@ import ( ) const ( - scope = "https://graph.microsoft.com/.default" - msgraphExpandLimit = 20 + scope = "https://graph.microsoft.com/.default" + msgraphExpandLimit = 20 + defaultAzureTimeout = 3 * time.Second + defaultAzureSecretEnvVar = "AZURE_CLIENT_SECRET" ) var ( @@ -98,8 +100,12 @@ func handleNil[T any](s *T) T { return result } -func (a *AzureReal) GetSourceType() SourceType { - return AzureSourceType +func (a *AzureReal) CreateUserFromRaw(raw map[string]any) (SourceUser, error) { + return NewAzureUser(raw) +} + +func (a *AzureReal) CreateGroupFromRaw(raw map[string]any) (SourceGroup, error) { + return NewAzureGroup(raw) } func (a *AzureReal) GetUsers() ([]SourceUser, error) { diff --git a/azure_real_integration_test.go b/azure_real_integration_test.go index 5864cc6..18037b2 100644 --- a/azure_real_integration_test.go +++ b/azure_real_integration_test.go @@ -14,7 +14,7 @@ import ( //go:embed config.local.yaml var _localConfig embed.FS -// TestPrintAzureUsersIntegration tests nothing, but can be used to debug Source users retrieved from ms graph api. +// TestPrintAzureUsersIntegration tests nothing, but can be used to debug Azure users retrieved from ms graph api. // In particular, it can be used to tune userFilter for production use. // It requires AZURE_CLIENT_SECRET env var and `config.local.yaml` file (which is .gitignored). func TestPrintAzureUsersIntegration(t *testing.T) { @@ -79,7 +79,7 @@ func TestPrintAzureUsersIntegration(t *testing.T) { require.NotEmpty(t, usersRaw) } -// TestPrintAzureGroupsIntegration tests nothing, but can be used to debug Source groups retrieved from ms graph api. +// TestPrintAzureGroupsIntegration tests nothing, but can be used to debug Azure groups retrieved from ms graph api. // In particular, it can be used to tune groupsFilter for production use. // It requires AZURE_CLIENT_SECRET env var and `config.local.yaml` file (which is .gitignored). func TestPrintAzureGroupsIntegrationRaw(t *testing.T) { diff --git a/config.go b/config.go index 3539514..d8589bf 100644 --- a/config.go +++ b/config.go @@ -76,8 +76,8 @@ type YtsaurusConfig struct { DebugUsernames []string `yaml:"debug_usernames"` // DebugGroupnames is a list of YTsaurus groupnames for which app will print more debug info in logs. DebugGroupnames []string `yaml:"debug_groupnames"` - - SourceAttributeName *string `yaml:"source_attribute_name"` + // The attribute name of user/group object in YTsaurus. + SourceAttributeName string `yaml:"source_attribute_name"` } type LoggingConfig struct { diff --git a/diff.go b/diff.go index 3956d19..13987a1 100644 --- a/diff.go +++ b/diff.go @@ -12,6 +12,24 @@ import ( "go.uber.org/zap" ) +type SourceUser interface { + GetID() ObjectID + GetName() string + GetRaw() (map[string]any, error) +} + +type SourceGroup interface { + GetID() ObjectID + GetName() string + GetRaw() (map[string]any, error) +} + +type SourceGroupWithMembers struct { + SourceGroup SourceGroup + // Members is a set of strings, representing users' ObjectID. + Members StringSet +} + func (a *App) syncOnce() { a.logger.Info("Start syncing") defer a.logger.Info("Finish syncing") @@ -46,7 +64,7 @@ func (a *App) syncUsers() (map[ObjectID]YtsaurusUser, error) { return nil, errors.Wrap(err, "failed to get Source users") } - ytUsers, err := a.ytsaurus.GetUsers(a.source.GetSourceType()) + ytUsers, err := a.ytsaurus.GetUsers() if err != nil { return nil, errors.Wrap(err, "failed to get YTsaurus users") } @@ -106,7 +124,7 @@ func (a *App) syncGroups(usersMap map[ObjectID]YtsaurusUser) error { if err != nil { return errors.Wrap(err, "failed to get Source groups") } - ytGroups, err := a.ytsaurus.GetGroupsWithMembers(a.source.GetSourceType()) + ytGroups, err := a.ytsaurus.GetGroupsWithMembers() if err != nil { return errors.Wrap(err, "failed to get YTsaurus groups") } @@ -206,10 +224,6 @@ func (a *App) diffGroups( existedGroupsWithMembersMap := make(map[ObjectID]existedGroup) for _, group := range ytGroups { - if group.IsManuallyManaged(a.source.GetSourceType()) { - continue - } - sourceGroup, err := a.buildSourceGroup(&group) if err != nil { return nil, errors.Wrap(err, "failed to create azure group from source") @@ -318,12 +332,9 @@ func (a *App) diffUsers( existedUsersMap := make(map[ObjectID]existedUser) for _, user := range ytUsers { - if user.IsManuallyManaged(a.source.GetSourceType()) { - continue - } sourceUser, err := a.buildSourceUser(&user) if err != nil { - return nil, errors.Wrap(err, "failed to create azure user from source") + return nil, errors.Wrap(err, "failed to build source user") } existedUsersMap[sourceUser.GetID()] = existedUser{user, sourceUser} resultUsersMap[sourceUser.GetID()] = user @@ -390,59 +401,41 @@ func (a *App) buildGroupName(sourceGroup SourceGroup) string { } func (a *App) buildSourceUser(ytUser *YtsaurusUser) (SourceUser, error) { - if ytUser.IsManuallyManaged(a.source.GetSourceType()) { + if ytUser.IsManuallyManaged() { return nil, errors.New("user is manually managed and can't be converted to source user") } - sourceType := SourceType(*ytUser.SourceType) - switch sourceType { - case AzureSourceType: - { - return NewAzureUser(ytUser.SourceRaw) - } - } - return nil, errors.New("unknown user source type") + return a.source.CreateUserFromRaw(ytUser.SourceRaw) } func (a *App) buildSourceGroup(ytGroup *YtsaurusGroupWithMembers) (SourceUser, error) { - if ytGroup.IsManuallyManaged(a.source.GetSourceType()) { + if ytGroup.IsManuallyManaged() { return nil, errors.New("user is manually managed and can't be converted to source user") } - sourceType := SourceType(*ytGroup.SourceType) - switch sourceType { - case AzureSourceType: - { - return NewAzureGroup(ytGroup.SourceRaw) - } - } - return nil, errors.New("unknown user source type") + return a.source.CreateGroupFromRaw(ytGroup.SourceRaw) } func (a *App) buildYtsaurusUser(sourceUser SourceUser) (YtsaurusUser, error) { - sourceType := string(sourceUser.GetSourceType()) sourceRaw, err := sourceUser.GetRaw() if err != nil { return YtsaurusUser{}, err } return YtsaurusUser{ - Username: a.buildUsername(sourceUser), - SourceType: &sourceType, - SourceRaw: sourceRaw, + Username: a.buildUsername(sourceUser), + SourceRaw: sourceRaw, // If we have Source user —> he is not banned. BannedSince: time.Time{}, }, nil } func (a *App) buildYtsaurusGroup(sourceGroup SourceGroup) (YtsaurusGroup, error) { - sourceType := string(sourceGroup.GetSourceType()) sourceRaw, err := sourceGroup.GetRaw() if err != nil { return YtsaurusGroup{}, err } return YtsaurusGroup{ - Name: a.buildGroupName(sourceGroup), - SourceType: &sourceType, - SourceRaw: sourceRaw, + Name: a.buildGroupName(sourceGroup), + SourceRaw: sourceRaw, }, nil } diff --git a/source_models.go b/source_models.go index 83731ef..06ab7d0 100644 --- a/source_models.go +++ b/source_models.go @@ -1,8 +1 @@ package main - -type ObjectID = string -type SourceType string - -const ( - AzureSourceType SourceType = "azure" -) diff --git a/util.go b/util.go index 1331ff1..1d33be7 100644 --- a/util.go +++ b/util.go @@ -12,10 +12,7 @@ import ( ) const ( - appTimeFormat = "2006-01-02T15:04:05Z0700" - defaultAzureTimeout = 3 * time.Second - defaultAzureSecretEnvVar = "AZURE_CLIENT_SECRET" - defaultAppRemoveLimit = 10 + appTimeFormat = "2006-01-02T15:04:05Z0700" ) type appLoggerType = *zap.SugaredLogger diff --git a/ytsaurus.go b/ytsaurus.go index b9f65c3..aca1b4f 100644 --- a/ytsaurus.go +++ b/ytsaurus.go @@ -2,11 +2,10 @@ package main import ( "context" + "fmt" "os" "time" - "go.ytsaurus.tech/library/go/ptr" - "github.com/pkg/errors" "k8s.io/utils/clock" @@ -64,8 +63,8 @@ func NewYtsaurus(cfg *YtsaurusConfig, logger appLoggerType, clock clock.PassiveC if cfg.Timeout == 0 { cfg.Timeout = defaultYtsaurusTimeout } - if cfg.SourceAttributeName == nil { - cfg.SourceAttributeName = ptr.String("source") + if cfg.SourceAttributeName == "" { + cfg.SourceAttributeName = "source" } return &Ytsaurus{ client: client, @@ -79,11 +78,11 @@ func NewYtsaurus(cfg *YtsaurusConfig, logger appLoggerType, clock clock.PassiveC debugUsernames: cfg.DebugUsernames, debugGroupnames: cfg.DebugGroupnames, - sourceAttributeName: *cfg.SourceAttributeName, + sourceAttributeName: cfg.SourceAttributeName, }, nil } -func (y *Ytsaurus) GetUsers(sourceType SourceType) ([]YtsaurusUser, error) { +func (y *Ytsaurus) GetUsers() ([]YtsaurusUser, error) { ctx, cancel := context.WithTimeout(context.Background(), y.timeout) defer cancel() @@ -94,7 +93,7 @@ func (y *Ytsaurus) GetUsers(sourceType SourceType) ([]YtsaurusUser, error) { var managedUsers []YtsaurusUser for _, user := range users { y.maybePrintExtraLogs(user.Username, "get_user", "user", user) - if user.IsManuallyManaged(sourceType) { + if user.IsManuallyManaged() { continue } managedUsers = append(managedUsers, user) @@ -124,7 +123,6 @@ func (y *Ytsaurus) CreateUser(user YtsaurusUser) error { user.Username, map[string]any{ y.sourceAttributeName: user.SourceRaw, - "source_type": user.SourceType, }, ) } @@ -204,7 +202,7 @@ func (y *Ytsaurus) BanUser(username string) error { ) } -func (y *Ytsaurus) GetGroupsWithMembers(sourceType SourceType) ([]YtsaurusGroupWithMembers, error) { +func (y *Ytsaurus) GetGroupsWithMembers() ([]YtsaurusGroupWithMembers, error) { ctx, cancel := context.WithTimeout(context.Background(), y.timeout) defer cancel() @@ -215,7 +213,7 @@ func (y *Ytsaurus) GetGroupsWithMembers(sourceType SourceType) ([]YtsaurusGroupW var managedGroups []YtsaurusGroupWithMembers for _, group := range groups { y.maybePrintExtraLogs(group.Name, "get_group", "group", group) - if group.IsManuallyManaged(sourceType) { + if group.IsManuallyManaged() { continue } managedGroups = append(managedGroups, group) @@ -243,7 +241,6 @@ func (y *Ytsaurus) CreateGroup(group YtsaurusGroup) error { y.client, group.Name, map[string]any{ - "source_type": group.SourceType, y.sourceAttributeName: group.SourceRaw, }, ) @@ -343,23 +340,11 @@ func (y *Ytsaurus) isUserManaged(username string) (bool, error) { ctx, cancel := context.WithTimeout(context.Background(), y.timeout) defer cancel() - attrAzureExists, err := y.client.NodeExists( - ctx, - ypath.Path("//sys/users/"+username+"/@azure"), - nil, - ) - if err != nil { - return false, err - } - attrSourceExists, err := y.client.NodeExists( + return y.client.NodeExists( ctx, - ypath.Path("//sys/users/"+username+"/@source"), + ypath.Path(fmt.Sprintf("//sys/users/%v/@%v", username, y.sourceAttributeName)), nil, ) - if err != nil { - return false, err - } - return attrAzureExists || attrSourceExists, nil } func (y *Ytsaurus) ensureUserManaged(username string) error { @@ -377,24 +362,11 @@ func (y *Ytsaurus) isGroupManaged(name string) (bool, error) { ctx, cancel := context.WithTimeout(context.Background(), y.timeout) defer cancel() - attrAzureExists, err := y.client.NodeExists( + return y.client.NodeExists( ctx, - ypath.Path("//sys/groups/"+name+"/@azure"), + ypath.Path(fmt.Sprintf("//sys/groups/%v/@%v", name, y.sourceAttributeName)), nil, ) - if err != nil { - return false, err - } - attrSourceExists, err := y.client.NodeExists( - ctx, - ypath.Path("//sys/groups/"+name+"/@source"), - nil, - ) - if err != nil { - return false, err - } - - return attrAzureExists || attrSourceExists, nil } func (y *Ytsaurus) ensureGroupManaged(groupname string) error { diff --git a/ytsaurus_helpers.go b/ytsaurus_helpers.go index 352ee31..667435c 100644 --- a/ytsaurus_helpers.go +++ b/ytsaurus_helpers.go @@ -15,7 +15,6 @@ import ( const ( bannedSinceAttributeName = "banned_since" bannedAttributeName = "banned" - sourceTypeAttributeName = "source_type" membersAttributeName = "members" nameAttributeName = "name" ) @@ -35,7 +34,6 @@ func doGetAllYtsaurusUsers(ctx context.Context, client yt.Client, sourceAttribut Attributes: []string{ bannedAttributeName, bannedSinceAttributeName, - sourceTypeAttributeName, sourceAttributeName, }, }, @@ -58,13 +56,6 @@ func doGetAllYtsaurusUsers(ctx context.Context, client yt.Client, sourceAttribut } } if sourceRaw, ok := ytUser.Attrs[sourceAttributeName]; ok { - // For backward compatibility only. - sourceType := string(AzureSourceType) - if sourceTypeRaw, ok := ytUser.Attrs[sourceTypeAttributeName]; ok { - sourceType = sourceTypeRaw.(string) - } - - user.SourceType = &sourceType user.SourceRaw = sourceRaw.(map[string]any) } } @@ -89,7 +80,6 @@ func doGetAllYtsaurusGroupsWithMembers(ctx context.Context, client yt.Client, so Attributes: []string{ membersAttributeName, sourceAttributeName, - sourceTypeAttributeName, }, }, ) @@ -111,13 +101,6 @@ func doGetAllYtsaurusGroupsWithMembers(ctx context.Context, client yt.Client, so } if sourceRaw, ok := ytGroup.Attrs[sourceAttributeName]; ok { - // For backward compatibility only. - sourceType := string(AzureSourceType) - if sourceTypeRaw, ok := ytGroup.Attrs[sourceTypeAttributeName]; ok { - sourceType = sourceTypeRaw.(string) - } - - group.SourceType = &sourceType group.SourceRaw = sourceRaw.(map[string]any) } } @@ -180,15 +163,13 @@ func buildUserAttributes(user YtsaurusUser, sourceAttributeName string) map[stri bannedSinceAttributeName: user.BannedSinceString(), bannedAttributeName: user.IsBanned(), sourceAttributeName: user.SourceRaw, - sourceTypeAttributeName: user.SourceType, } } func buildGroupAttributes(group YtsaurusGroup, sourceAttributeName string) map[string]any { return map[string]any{ - sourceAttributeName: group.SourceRaw, - nameAttributeName: group.Name, - sourceTypeAttributeName: group.SourceType, + sourceAttributeName: group.SourceRaw, + nameAttributeName: group.Name, } } diff --git a/ytsaurus_models.go b/ytsaurus_models.go index 326a0f7..dc34e12 100644 --- a/ytsaurus_models.go +++ b/ytsaurus_models.go @@ -4,37 +4,16 @@ import ( "time" ) -type SourceUser interface { - GetID() ObjectID - GetName() string - GetSourceType() SourceType - GetRaw() (map[string]any, error) -} - -type SourceGroup interface { - GetID() ObjectID - GetName() string - GetSourceType() SourceType - GetRaw() (map[string]any, error) -} - -type SourceGroupWithMembers struct { - SourceGroup SourceGroup - // Members is a set of strings, representing users' ObjectID. - Members StringSet -} - type YtsaurusUser struct { // Username is a unique @name attribute of a user. Username string - SourceType *string SourceRaw map[string]any BannedSince time.Time } // IsManuallyManaged true if user doesn't have @azure attribute (system or manually created user). -func (u YtsaurusUser) IsManuallyManaged(sourceType SourceType) bool { - return u.SourceRaw == nil || u.SourceType == nil || *u.SourceType != string(sourceType) +func (u YtsaurusUser) IsManuallyManaged() bool { + return u.SourceRaw == nil } func (u YtsaurusUser) IsBanned() bool { @@ -50,14 +29,13 @@ func (u YtsaurusUser) BannedSinceString() string { type YtsaurusGroup struct { // Name is a unique @name attribute of a group. - Name string - SourceType *string - SourceRaw map[string]any + Name string + SourceRaw map[string]any } // IsManuallyManaged true if group doesn't have @azure attribute (system or manually created group). -func (g YtsaurusGroup) IsManuallyManaged(sourceType SourceType) bool { - return g.SourceRaw == nil || g.SourceType == nil || *g.SourceType != string(sourceType) +func (g YtsaurusGroup) IsManuallyManaged() bool { + return g.SourceRaw == nil } type YtsaurusGroupWithMembers struct { diff --git a/ytsaurus_test.go b/ytsaurus_test.go index 909901c..9482229 100644 --- a/ytsaurus_test.go +++ b/ytsaurus_test.go @@ -6,8 +6,6 @@ import ( "testing" "time" - "go.ytsaurus.tech/library/go/ptr" - "github.com/stretchr/testify/require" "k8s.io/utils/clock" @@ -26,7 +24,7 @@ func getYtsaurus(t *testing.T, ytLocal *YtsaurusLocal) *Ytsaurus { ApplyUserChanges: true, ApplyGroupChanges: true, ApplyMemberChanges: true, - SourceAttributeName: ptr.String("azure"), + SourceAttributeName: "azure", }, getDevelopmentLogger(), clock.RealClock{}, ) @@ -47,8 +45,7 @@ func TestUpdateUserFirstName(t *testing.T) { const azureID = "fake-az-id-old" managedOleg := YtsaurusUser{ - Username: "oleg", - SourceType: ptr.String("azure"), + Username: "oleg", SourceRaw: map[string]any{ "id": azureID, "first_name": "Lego", @@ -85,13 +82,12 @@ func TestGroups(t *testing.T) { defer func() { require.NoError(t, ytLocal.Stop()) }() yt := getYtsaurus(t, ytLocal) - groupsInitial, err := yt.GetGroupsWithMembers(AzureSourceType) + groupsInitial, err := yt.GetGroupsWithMembers() require.NoError(t, err) require.Empty(t, groupsInitial) managedOleg := YtsaurusUser{ - Username: "oleg", - SourceType: ptr.String("azure"), + Username: "oleg", SourceRaw: map[string]any{ "id": "fake-az-id-oleg", }, @@ -100,8 +96,7 @@ func TestGroups(t *testing.T) { require.NoError(t, err) managedOlegsGroup := YtsaurusGroup{ - Name: "olegs", - SourceType: ptr.String("azure"), + Name: "olegs", SourceRaw: map[string]any{ "id": "fake-az-id-olegs", "display_name": "This is group is for Olegs only", @@ -113,15 +108,14 @@ func TestGroups(t *testing.T) { err = yt.AddMember(managedOleg.Username, managedOlegsGroup.Name) require.NoError(t, err) - groupsAfterCreate, err := yt.GetGroupsWithMembers(AzureSourceType) + groupsAfterCreate, err := yt.GetGroupsWithMembers() require.NoError(t, err) members := NewStringSet() members.Add(managedOleg.Username) require.Equal(t, []YtsaurusGroupWithMembers{ { YtsaurusGroup: YtsaurusGroup{ - Name: managedOlegsGroup.Name, - SourceType: ptr.String("azure"), + Name: managedOlegsGroup.Name, SourceRaw: map[string]any{ "id": managedOlegsGroup.SourceRaw["id"], "display_name": managedOlegsGroup.SourceRaw["display_name"], @@ -134,13 +128,12 @@ func TestGroups(t *testing.T) { err = yt.RemoveMember(managedOleg.Username, managedOlegsGroup.Name) require.NoError(t, err) - groupsAfterRemoveMember, err := yt.GetGroupsWithMembers(AzureSourceType) + groupsAfterRemoveMember, err := yt.GetGroupsWithMembers() require.NoError(t, err) require.Equal(t, []YtsaurusGroupWithMembers{ { YtsaurusGroup: YtsaurusGroup{ - Name: managedOlegsGroup.Name, - SourceType: ptr.String("azure"), + Name: managedOlegsGroup.Name, SourceRaw: map[string]any{ "id": managedOlegsGroup.SourceRaw["id"], "display_name": managedOlegsGroup.SourceRaw["display_name"], @@ -153,7 +146,7 @@ func TestGroups(t *testing.T) { err = yt.RemoveGroup(managedOlegsGroup.Name) require.NoError(t, err) - groupsAfterRemove, err := yt.GetGroupsWithMembers(AzureSourceType) + groupsAfterRemove, err := yt.GetGroupsWithMembers() require.NoError(t, err) require.Empty(t, groupsAfterRemove) From 1ba54fa64b139daec9c92e27cbebcdf863384588 Mon Sep 17 00:00:00 2001 From: Nadezhda Savchenko Date: Thu, 14 Mar 2024 12:53:50 +0300 Subject: [PATCH 19/21] Review fixes --- azure_real_integration_test.go | 2 +- config.go | 6 ++-- diff.go | 61 ++++++++++++++++------------------ source_models.go | 1 - ytsaurus.go | 3 +- 5 files changed, 34 insertions(+), 39 deletions(-) delete mode 100644 source_models.go diff --git a/azure_real_integration_test.go b/azure_real_integration_test.go index 18037b2..0027f14 100644 --- a/azure_real_integration_test.go +++ b/azure_real_integration_test.go @@ -86,7 +86,7 @@ func TestPrintAzureGroupsIntegrationRaw(t *testing.T) { cfg, err := loadConfig("config.local.yaml") require.NoError(t, err) - logger, err := configureLogger(cfg.Logging) + logger, err := configureLogger(&cfg.Logging) require.NoError(t, err) azure, err := NewAzureReal(cfg.Azure, logger) require.NoError(t, err) diff --git a/config.go b/config.go index d8589bf..08bbf22 100644 --- a/config.go +++ b/config.go @@ -48,11 +48,11 @@ type AzureConfig struct { UsersFilter string `yaml:"users_filter"` // GroupsFilter is MS Graph $filter value used for group fetching requests. // See https://learn.microsoft.com/en-us/graph/api/group-list - GroupsFilter string `yaml:"groups_filter"` - Timeout time.Duration `yaml:"timeout"` + GroupsFilter string `yaml:"groups_filter"` // GroupsDisplayNameSuffixPostFilter applied to the fetched groups display names. - GroupsDisplayNameSuffixPostFilter string `yaml:"groups_display_name_suffix_post_filter"` + GroupsDisplayNameSuffixPostFilter string `yaml:"groups_display_name_suffix_post_filter"` + Timeout time.Duration `yaml:"timeout"` // DebugAzureIDs is a list of ids for which app will print more debug info in logs. DebugAzureIDs []string `yaml:"debug_azure_ids"` diff --git a/diff.go b/diff.go index 13987a1..b9789cd 100644 --- a/diff.go +++ b/diff.go @@ -186,6 +186,7 @@ func (a *App) syncGroups(usersMap map[ObjectID]YtsaurusUser) error { // TODO: alerts } } + a.logger.Infow("Finish syncing group memberships", "added", len(diff.membersToAdd)-addMemberErrCount, "add_errors", addMemberErrCount, @@ -203,11 +204,6 @@ type groupDiff struct { membersToRemove []YtsaurusMembership } -type existedGroup struct { - ytsaurusGroup YtsaurusGroupWithMembers - sourceGroup SourceGroup -} - func (a *App) diffGroups( sourceGroups []SourceGroupWithMembers, ytGroups []YtsaurusGroupWithMembers, @@ -222,18 +218,18 @@ func (a *App) diffGroups( sourceGroupsWithMembersMap[group.SourceGroup.GetID()] = group } - existedGroupsWithMembersMap := make(map[ObjectID]existedGroup) + ytGroupsWithMembersMap := make(map[ObjectID]YtsaurusGroupWithMembers) for _, group := range ytGroups { sourceGroup, err := a.buildSourceGroup(&group) if err != nil { return nil, errors.Wrap(err, "failed to create azure group from source") } - existedGroupsWithMembersMap[sourceGroup.GetID()] = existedGroup{group, sourceGroup} + ytGroupsWithMembersMap[sourceGroup.GetID()] = group } // Collecting groups to create (the ones that exist in Source but not in YTsaurus). for objectID, sourceGroupWithMembers := range sourceGroupsWithMembersMap { - if _, ok := existedGroupsWithMembersMap[objectID]; !ok { + if _, ok := ytGroupsWithMembersMap[objectID]; !ok { newYtsaurusGroup, err := a.buildYtsaurusGroup(sourceGroupWithMembers.SourceGroup) if err != nil { return nil, errors.Wrap(err, "failed to build Ytsaurus group") @@ -248,22 +244,22 @@ func (a *App) diffGroups( } } - for objectID, ytGroupWithMembers := range existedGroupsWithMembersMap { + for objectID, ytGroupWithMembers := range ytGroupsWithMembersMap { // Collecting groups to remove (the ones that exist in YTsaurus and not in Azure). sourceGroupWithMembers, ok := sourceGroupsWithMembersMap[objectID] if !ok { - groupsToRemove = append(groupsToRemove, ytGroupWithMembers.ytsaurusGroup.YtsaurusGroup) + groupsToRemove = append(groupsToRemove, ytGroupWithMembers.YtsaurusGroup) continue } // Collecting groups with changed Source fields (actually we have only displayName for now which // should change, though we still handle that just in case). - groupChanged, updatedYtGroup, err := a.isGroupChanged(sourceGroupWithMembers.SourceGroup, ytGroupWithMembers.ytsaurusGroup.YtsaurusGroup) + groupChanged, updatedYtGroup, err := a.isGroupChanged(sourceGroupWithMembers.SourceGroup, ytGroupWithMembers.YtsaurusGroup) if err != nil { return nil, errors.Wrap(err, "failed to check if group is changed") } // Group name can change after update, so we ensure that correct one is used for membership updates. - actualGroupname := ytGroupWithMembers.ytsaurusGroup.YtsaurusGroup.Name + actualGroupname := ytGroupWithMembers.YtsaurusGroup.Name if groupChanged { // This shouldn't happen until we add more fields in YTsaurus' group @azure attribute. a.logger.Warnw( @@ -274,7 +270,7 @@ func (a *App) diffGroups( actualGroupname = updatedYtGroup.YtsaurusGroup.Name } - membersCreate, membersRemove := a.isGroupMembersChanged(sourceGroupWithMembers, ytGroupWithMembers.ytsaurusGroup, usersMap) + membersCreate, membersRemove := a.isGroupMembersChanged(sourceGroupWithMembers, ytGroupWithMembers, usersMap) for _, username := range membersCreate { membersToAdd = append(membersToAdd, YtsaurusMembership{ GroupName: actualGroupname, @@ -305,11 +301,6 @@ type usersDiff struct { result map[ObjectID]YtsaurusUser } -type existedUser struct { - ytsaurusUser YtsaurusUser - sourceUser SourceUser -} - func (a *App) diffUsers( sourceUsers []SourceUser, ytUsers []YtsaurusUser, @@ -319,6 +310,15 @@ func (a *App) diffUsers( sourceUsersMap[user.GetID()] = user } + ytUsersMap := make(map[ObjectID]YtsaurusUser) + for _, user := range ytUsers { + sourceUser, err := a.buildSourceUser(&user) + if err != nil { + return nil, errors.Wrap(err, "failed to build source user") + } + ytUsersMap[sourceUser.GetID()] = user + } + ytUsersFromSourceMap := make(map[ObjectID]YtsaurusUser) for _, user := range sourceUsers { ytUser, err := a.buildYtsaurusUser(user) @@ -330,21 +330,11 @@ func (a *App) diffUsers( resultUsersMap := make(map[ObjectID]YtsaurusUser) - existedUsersMap := make(map[ObjectID]existedUser) - for _, user := range ytUsers { - sourceUser, err := a.buildSourceUser(&user) - if err != nil { - return nil, errors.Wrap(err, "failed to build source user") - } - existedUsersMap[sourceUser.GetID()] = existedUser{user, sourceUser} - resultUsersMap[sourceUser.GetID()] = user - } - var create, remove []YtsaurusUser var update []UpdatedYtsaurusUser for objectID, sourceUser := range sourceUsersMap { - if _, ok := existedUsersMap[objectID]; !ok { + if _, ok := ytUsersMap[objectID]; !ok { ytUser, err := a.buildYtsaurusUser(sourceUser) if err != nil { return nil, errors.Wrap(err, "failed to create Ytsaurus user from source user") @@ -354,13 +344,18 @@ func (a *App) diffUsers( } } - for objectID, user := range existedUsersMap { - ytUserFromSource, ok := ytUsersFromSourceMap[objectID] + for objectID, ytUser := range ytUsersMap { + sourceUser, ok := sourceUsersMap[objectID] if !ok { - remove = append(remove, user.ytsaurusUser) + remove = append(remove, ytUser) + delete(resultUsersMap, objectID) continue } - userChanged, updatedYtUser, err := a.isUserChanged(ytUserFromSource, user.ytsaurusUser) + newYtUser, err := a.buildYtsaurusUser(sourceUser) + if err != nil { + return nil, errors.Wrap(err, "failed to create Ytsaurus user from source user") + } + userChanged, updatedYtUser, err := a.isUserChanged(newYtUser, ytUser) if err != nil { return nil, errors.Wrap(err, "failed to check if user was changed") } diff --git a/source_models.go b/source_models.go deleted file mode 100644 index 06ab7d0..0000000 --- a/source_models.go +++ /dev/null @@ -1 +0,0 @@ -package main diff --git a/ytsaurus.go b/ytsaurus.go index aca1b4f..b38a4c8 100644 --- a/ytsaurus.go +++ b/ytsaurus.go @@ -17,6 +17,7 @@ import ( const ( defaultYtsaurusTimeout = 3 * time.Second defaultYtsaurusSecretEnvVar = "YT_TOKEN" + defaultSourceAttributeName = "source" ) type Ytsaurus struct { @@ -64,7 +65,7 @@ func NewYtsaurus(cfg *YtsaurusConfig, logger appLoggerType, clock clock.PassiveC cfg.Timeout = defaultYtsaurusTimeout } if cfg.SourceAttributeName == "" { - cfg.SourceAttributeName = "source" + cfg.SourceAttributeName = defaultSourceAttributeName } return &Ytsaurus{ client: client, From b039559e4382019f35a697e11c2bace33a4c0f79 Mon Sep 17 00:00:00 2001 From: Nadezhda Savchenko Date: Thu, 14 Mar 2024 13:00:09 +0300 Subject: [PATCH 20/21] Fix test --- diff.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/diff.go b/diff.go index b9789cd..8c1e96b 100644 --- a/diff.go +++ b/diff.go @@ -311,12 +311,14 @@ func (a *App) diffUsers( } ytUsersMap := make(map[ObjectID]YtsaurusUser) + resultUsersMap := make(map[ObjectID]YtsaurusUser) for _, user := range ytUsers { sourceUser, err := a.buildSourceUser(&user) if err != nil { return nil, errors.Wrap(err, "failed to build source user") } ytUsersMap[sourceUser.GetID()] = user + resultUsersMap[sourceUser.GetID()] = user } ytUsersFromSourceMap := make(map[ObjectID]YtsaurusUser) @@ -328,8 +330,6 @@ func (a *App) diffUsers( ytUsersFromSourceMap[user.GetID()] = ytUser } - resultUsersMap := make(map[ObjectID]YtsaurusUser) - var create, remove []YtsaurusUser var update []UpdatedYtsaurusUser From ae0690d7ef1e909a2c447e3ad300f950bb580ece Mon Sep 17 00:00:00 2001 From: Nadezhda Savchenko Date: Thu, 14 Mar 2024 13:31:51 +0300 Subject: [PATCH 21/21] Remove unused map --- diff.go | 9 --------- 1 file changed, 9 deletions(-) diff --git a/diff.go b/diff.go index 8c1e96b..d8927f7 100644 --- a/diff.go +++ b/diff.go @@ -321,15 +321,6 @@ func (a *App) diffUsers( resultUsersMap[sourceUser.GetID()] = user } - ytUsersFromSourceMap := make(map[ObjectID]YtsaurusUser) - for _, user := range sourceUsers { - ytUser, err := a.buildYtsaurusUser(user) - if err != nil { - return nil, errors.Wrap(err, "failed to create Ytsaurus user from source user") - } - ytUsersFromSourceMap[user.GetID()] = ytUser - } - var create, remove []YtsaurusUser var update []UpdatedYtsaurusUser