LDAP user synchronization (#1478)

conf/app.ini

@@ -442,6 +442,16 @@ SCHEDULE = @every 24h
 ; Archives created more than OLDER_THAN ago are subject to deletion
 OLDER_THAN = 24h
 
+; Synchronize external user data (only LDAP user synchronization is supported)
+[cron.sync_external_users]
+; Syncronize external user data when starting server (default false)
+RUN_AT_START = false
+; Interval as a duration between each synchronization (default every 24h)
+SCHEDULE = @every 24h
+; Create new users, update existing user data and disable users that are not in external source anymore (default)
+;   or only create new users if UPDATE_EXISTING is set to false
+UPDATE_EXISTING = true
+
 [git]
 ; Disables highlight of added and removed changes
 DISABLE_DIFF_HIGHLIGHT = false

models/login_source.go

@@ -140,11 +140,12 @@ func (cfg *OAuth2Config) ToDB() ([]byte, error) {
 
 // LoginSource represents an external way for authorizing users.
 type LoginSource struct {
-	ID        int64 `xorm:"pk autoincr"`
-	Type      LoginType
-	Name      string          `xorm:"UNIQUE"`
-	IsActived bool            `xorm:"INDEX NOT NULL DEFAULT false"`
-	Cfg       core.Conversion `xorm:"TEXT"`
+	ID            int64 `xorm:"pk autoincr"`
+	Type          LoginType
+	Name          string          `xorm:"UNIQUE"`
+	IsActived     bool            `xorm:"INDEX NOT NULL DEFAULT false"`
+	IsSyncEnabled bool            `xorm:"INDEX NOT NULL DEFAULT false"`
+	Cfg           core.Conversion `xorm:"TEXT"`
 
 	Created     time.Time `xorm:"-"`
 	CreatedUnix int64     `xorm:"INDEX"`
@@ -294,6 +295,10 @@ func CreateLoginSource(source *LoginSource) error {
 	} else if has {
 		return ErrLoginSourceAlreadyExist{source.Name}
 	}
+	// Synchronization is only aviable with LDAP for now
+	if !source.IsLDAP() {
+		source.IsSyncEnabled = false
+	}
 
 	_, err = x.Insert(source)
 	if err == nil && source.IsOAuth2() && source.IsActived {
@@ -405,8 +410,8 @@ func composeFullName(firstname, surname, username string) string {
 // LoginViaLDAP queries if login/password is valid against the LDAP directory pool,
 // and create a local user if success when enabled.
 func LoginViaLDAP(user *User, login, password string, source *LoginSource, autoRegister bool) (*User, error) {
-	username, fn, sn, mail, isAdmin, succeed := source.Cfg.(*LDAPConfig).SearchEntry(login, password, source.Type == LoginDLDAP)
-	if !succeed {
+	sr := source.Cfg.(*LDAPConfig).SearchEntry(login, password, source.Type == LoginDLDAP)
+	if sr == nil {
 		// User not in LDAP, do nothing
 		return nil, ErrUserNotExist{0, login, 0}
 	}
@@ -416,28 +421,28 @@ func LoginViaLDAP(user *User, login, password string, source *LoginSource, autoR
 	}
 
 	// Fallback.
-	if len(username) == 0 {
-		username = login
+	if len(sr.Username) == 0 {
+		sr.Username = login
 	}
 	// Validate username make sure it satisfies requirement.
-	if binding.AlphaDashDotPattern.MatchString(username) {
-		return nil, fmt.Errorf("Invalid pattern for attribute 'username' [%s]: must be valid alpha or numeric or dash(-_) or dot characters", username)
+	if binding.AlphaDashDotPattern.MatchString(sr.Username) {
+		return nil, fmt.Errorf("Invalid pattern for attribute 'username' [%s]: must be valid alpha or numeric or dash(-_) or dot characters", sr.Username)
 	}
 
-	if len(mail) == 0 {
-		mail = fmt.Sprintf("%s@localhost", username)
+	if len(sr.Mail) == 0 {
+		sr.Mail = fmt.Sprintf("%s@localhost", sr.Username)
 	}
 
 	user = &User{
-		LowerName:   strings.ToLower(username),
-		Name:        username,
-		FullName:    composeFullName(fn, sn, username),
-		Email:       mail,
+		LowerName:   strings.ToLower(sr.Username),
+		Name:        sr.Username,
+		FullName:    composeFullName(sr.Name, sr.Surname, sr.Username),
+		Email:       sr.Mail,
 		LoginType:   source.Type,
 		LoginSource: source.ID,
 		LoginName:   login,
 		IsActive:    true,
-		IsAdmin:     isAdmin,
+		IsAdmin:     sr.IsAdmin,
 	}
 	return user, CreateUser(user)
 }

models/migrations/migrations.go

@@ -110,6 +110,8 @@ var migrations = []Migration{
 	NewMigration("add commit status table", addCommitStatus),
 	// v30 -> 31
 	NewMigration("add primary key to external login user", addExternalLoginUserPK),
+	// 31 -> 32
+	NewMigration("add field for login source synchronization", addLoginSourceSyncEnabledColumn),
 }
 
 // Migrate database to current version

models/migrations/v31.go

@@ -0,0 +1,35 @@
+// Copyright 2017 The Gogs Authors. All rights reserved.
+// Use of this source code is governed by a MIT-style
+// license that can be found in the LICENSE file.
+
+package migrations
+
+import (
+	"fmt"
+	"time"
+
+	"github.com/go-xorm/core"
+	"github.com/go-xorm/xorm"
+)
+
+func addLoginSourceSyncEnabledColumn(x *xorm.Engine) error {
+	// LoginSource see models/login_source.go
+	type LoginSource struct {
+		ID            int64 `xorm:"pk autoincr"`
+		Type          int
+		Name          string          `xorm:"UNIQUE"`
+		IsActived     bool            `xorm:"INDEX NOT NULL DEFAULT false"`
+		IsSyncEnabled bool            `xorm:"INDEX NOT NULL DEFAULT false"`
+		Cfg           core.Conversion `xorm:"TEXT"`
+
+		Created     time.Time `xorm:"-"`
+		CreatedUnix int64     `xorm:"INDEX"`
+		Updated     time.Time `xorm:"-"`
+		UpdatedUnix int64     `xorm:"INDEX"`
+	}
+
+	if err := x.Sync2(new(LoginSource)); err != nil {
+		return fmt.Errorf("Sync2: %v", err)
+	}
+	return nil
+}

models/user.go

@@ -50,6 +50,8 @@ const (
 	UserTypeOrganization
 )
 
+const syncExternalUsers = "sync_external_users"
+
 var (
 	// ErrUserNotKeyOwner user does not own this key error
 	ErrUserNotKeyOwner = errors.New("User does not own this public key")
@@ -1322,3 +1324,128 @@ func GetWatchedRepos(userID int64, private bool) ([]*Repository, error) {
 	}
 	return repos, nil
 }
+
+// SyncExternalUsers is used to synchronize users with external authorization source
+func SyncExternalUsers() {
+	if taskStatusTable.IsRunning(syncExternalUsers) {
+		return
+	}
+	taskStatusTable.Start(syncExternalUsers)
+	defer taskStatusTable.Stop(syncExternalUsers)
+
+	log.Trace("Doing: SyncExternalUsers")
+
+	ls, err := LoginSources()
+	if err != nil {
+		log.Error(4, "SyncExternalUsers: %v", err)
+		return
+	}
+
+	updateExisting := setting.Cron.SyncExternalUsers.UpdateExisting
+
+	for _, s := range ls {
+		if !s.IsActived || !s.IsSyncEnabled {
+			continue
+		}
+		if s.IsLDAP() {
+			log.Trace("Doing: SyncExternalUsers[%s]", s.Name)
+
+			var existingUsers []int64
+
+			// Find all users with this login type
+			var users []User
+			x.Where("login_type = ?", LoginLDAP).
+				And("login_source = ?", s.ID).
+				Find(&users)
+
+			sr := s.LDAP().SearchEntries()
+
+			for _, su := range sr {
+				if len(su.Username) == 0 {
+					continue
+				}
+
+				if len(su.Mail) == 0 {
+					su.Mail = fmt.Sprintf("%s@localhost", su.Username)
+				}
+
+				var usr *User
+				// Search for existing user
+				for _, du := range users {
+					if du.LowerName == strings.ToLower(su.Username) {
+						usr = &du
+						break
+					}
+				}
+
+				fullName := composeFullName(su.Name, su.Surname, su.Username)
+				// If no existing user found, create one
+				if usr == nil {
+					log.Trace("SyncExternalUsers[%s]: Creating user %s", s.Name, su.Username)
+
+					usr = &User{
+						LowerName:   strings.ToLower(su.Username),
+						Name:        su.Username,
+						FullName:    fullName,
+						LoginType:   s.Type,
+						LoginSource: s.ID,
+						LoginName:   su.Username,
+						Email:       su.Mail,
+						IsAdmin:     su.IsAdmin,
+						IsActive:    true,
+					}
+
+					err = CreateUser(usr)
+					if err != nil {
+						log.Error(4, "SyncExternalUsers[%s]: Error creating user %s: %v", s.Name, su.Username, err)
+					}
+				} else if updateExisting {
+					existingUsers = append(existingUsers, usr.ID)
+					// Check if user data has changed
+					if (len(s.LDAP().AdminFilter) > 0 && usr.IsAdmin != su.IsAdmin) ||
+						strings.ToLower(usr.Email) != strings.ToLower(su.Mail) ||
+						usr.FullName != fullName ||
+						!usr.IsActive {
+
+						log.Trace("SyncExternalUsers[%s]: Updating user %s", s.Name, usr.Name)
+
+						usr.FullName = fullName
+						usr.Email = su.Mail
+						// Change existing admin flag only if AdminFilter option is set
+						if len(s.LDAP().AdminFilter) > 0 {
+							usr.IsAdmin = su.IsAdmin
+						}
+						usr.IsActive = true
+
+						err = UpdateUser(usr)
+						if err != nil {
+							log.Error(4, "SyncExternalUsers[%s]: Error updating user %s: %v", s.Name, usr.Name, err)
+						}
+					}
+				}
+			}
+
+			// Deactivate users not present in LDAP
+			if updateExisting {
+				for _, usr := range users {
+					found := false
+					for _, uid := range existingUsers {
+						if usr.ID == uid {
+							found = true
+							break
+						}
+					}
+					if !found {
+						log.Trace("SyncExternalUsers[%s]: Deactivating user %s", s.Name, usr.Name)
+
+						usr.IsActive = false
+						err = UpdateUser(&usr)
+						if err != nil {
+							log.Error(4, "SyncExternalUsers[%s]: Error deactivating user %s: %v", s.Name, usr.Name, err)
+						}
+					}
+				}
+			}
+		}
+	}
+}

modules/auth/auth_form.go

@@ -28,6 +28,7 @@ type AuthenticationForm struct {
 	Filter                        string
 	AdminFilter                   string
 	IsActive                      bool
+	IsSyncEnabled                 bool
 	SMTPAuth                      string
 	SMTPHost                      string
 	SMTPPort                      int