From dc2fe9801f1a83a5810a778b806dac1bc210f110 Mon Sep 17 00:00:00 2001 From: zeripath Date: Thu, 12 Dec 2019 13:18:07 +0000 Subject: [PATCH] Make repository management section handle lfs locks (#8726) * Make repository maangement section handle lfs locks * Add check attribute handling and handle locking paths better * More cleanly check-attributes * handle error * Check if file exists in default branch before linking to it. * fixup * Properly cleanPath * Use cleanPath * Sigh --- models/lfs_lock.go | 25 +++- models/repo.go | 2 +- modules/git/repo_attribute.go | 84 ++++++++++++ modules/lfs/locks.go | 4 +- options/locale/locale_en-US.ini | 10 ++ routers/repo/lfs.go | 176 +++++++++++++++++++++++++ routers/routes/routes.go | 5 + templates/repo/settings/lfs.tmpl | 5 +- templates/repo/settings/lfs_locks.tmpl | 61 +++++++++ web_src/less/_base.less | 4 + 10 files changed, 367 insertions(+), 9 deletions(-) create mode 100644 modules/git/repo_attribute.go create mode 100644 templates/repo/settings/lfs_locks.tmpl diff --git a/models/lfs_lock.go b/models/lfs_lock.go index ba1a45281..3e56a7960 100644 --- a/models/lfs_lock.go +++ b/models/lfs_lock.go @@ -49,7 +49,7 @@ func (l *LFSLock) AfterLoad(session *xorm.Session) { } func cleanPath(p string) string { - return path.Clean(p) + return path.Clean("/" + p)[1:] } // APIFormat convert a Release to lfs.LFSLock @@ -71,6 +71,8 @@ func CreateLFSLock(lock *LFSLock) (*LFSLock, error) { return nil, err } + lock.Path = cleanPath(lock.Path) + l, err := GetLFSLock(lock.Repo, lock.Path) if err == nil { return l, ErrLFSLockAlreadyExist{lock.RepoID, lock.Path} @@ -110,9 +112,24 @@ func GetLFSLockByID(id int64) (*LFSLock, error) { } // GetLFSLockByRepoID returns a list of locks of repository. -func GetLFSLockByRepoID(repoID int64) (locks []*LFSLock, err error) { - err = x.Where("repo_id = ?", repoID).Find(&locks) - return +func GetLFSLockByRepoID(repoID int64, page, pageSize int) ([]*LFSLock, error) { + sess := x.NewSession() + defer sess.Close() + + if page >= 0 && pageSize > 0 { + start := 0 + if page > 0 { + start = (page - 1) * pageSize + } + sess.Limit(pageSize, start) + } + lfsLocks := make([]*LFSLock, 0, pageSize) + return lfsLocks, sess.Find(&lfsLocks, &LFSLock{RepoID: repoID}) +} + +// CountLFSLockByRepoID returns a count of all LFSLocks associated with a repository. +func CountLFSLockByRepoID(repoID int64) (int64, error) { + return x.Count(&LFSLock{RepoID: repoID}) } // DeleteLFSLockByID deletes a lock by given ID. diff --git a/models/repo.go b/models/repo.go index d5ea29c50..c904449bb 100644 --- a/models/repo.go +++ b/models/repo.go @@ -2913,7 +2913,7 @@ func (repo *Repository) GetOriginalURLHostname() string { // GetTreePathLock returns LSF lock for the treePath func (repo *Repository) GetTreePathLock(treePath string) (*LFSLock, error) { if setting.LFS.StartServer { - locks, err := GetLFSLockByRepoID(repo.ID) + locks, err := GetLFSLockByRepoID(repo.ID, 0, 0) if err != nil { return nil, err } diff --git a/modules/git/repo_attribute.go b/modules/git/repo_attribute.go new file mode 100644 index 000000000..c10c96f55 --- /dev/null +++ b/modules/git/repo_attribute.go @@ -0,0 +1,84 @@ +// Copyright 2019 The Gitea Authors. All rights reserved. +// Use of this source code is governed by a MIT-style +// license that can be found in the LICENSE file. + +package git + +import ( + "bytes" + "fmt" + + "github.com/mcuadros/go-version" +) + +// CheckAttributeOpts represents the possible options to CheckAttribute +type CheckAttributeOpts struct { + CachedOnly bool + AllAttributes bool + Attributes []string + Filenames []string +} + +// CheckAttribute return the Blame object of file +func (repo *Repository) CheckAttribute(opts CheckAttributeOpts) (map[string]map[string]string, error) { + binVersion, err := BinVersion() + if err != nil { + return nil, fmt.Errorf("Git version missing: %v", err) + } + + stdOut := new(bytes.Buffer) + stdErr := new(bytes.Buffer) + + cmdArgs := []string{"check-attr", "-z"} + + if opts.AllAttributes { + cmdArgs = append(cmdArgs, "-a") + } else { + for _, attribute := range opts.Attributes { + if attribute != "" { + cmdArgs = append(cmdArgs, attribute) + } + } + } + + // git check-attr --cached first appears in git 1.7.8 + if opts.CachedOnly && version.Compare(binVersion, "1.7.8", ">=") { + cmdArgs = append(cmdArgs, "--cached") + } + + cmdArgs = append(cmdArgs, "--") + + for _, arg := range opts.Filenames { + if arg != "" { + cmdArgs = append(cmdArgs, arg) + } + } + + cmd := NewCommand(cmdArgs...) + + if err := cmd.RunInDirPipeline(repo.Path, stdOut, stdErr); err != nil { + return nil, fmt.Errorf("Failed to run check-attr: %v\n%s\n%s", err, stdOut.String(), stdErr.String()) + } + + fields := bytes.Split(stdOut.Bytes(), []byte{'\000'}) + + if len(fields)%3 != 1 { + return nil, fmt.Errorf("Wrong number of fields in return from check-attr") + } + + var name2attribute2info = make(map[string]map[string]string) + + for i := 0; i < (len(fields) / 3); i++ { + filename := string(fields[3*i]) + attribute := string(fields[3*i+1]) + info := string(fields[3*i+2]) + attribute2info := name2attribute2info[filename] + if attribute2info == nil { + attribute2info = make(map[string]string) + } + attribute2info[attribute] = info + name2attribute2info[filename] = attribute2info + } + + return name2attribute2info, nil +} diff --git a/modules/lfs/locks.go b/modules/lfs/locks.go index 9ffe6b9d5..b077cd2d0 100644 --- a/modules/lfs/locks.go +++ b/modules/lfs/locks.go @@ -110,7 +110,7 @@ func GetListLockHandler(ctx *context.Context) { } //If no query params path or id - lockList, err := models.GetLFSLockByRepoID(repository.ID) + lockList, err := models.GetLFSLockByRepoID(repository.ID, 0, 0) if err != nil { ctx.JSON(500, api.LFSLockError{ Message: "unable to list locks : " + err.Error(), @@ -220,7 +220,7 @@ func VerifyLockHandler(ctx *context.Context) { } //TODO handle body json cursor and limit - lockList, err := models.GetLFSLockByRepoID(repository.ID) + lockList, err := models.GetLFSLockByRepoID(repository.ID, 0, 0) if err != nil { ctx.JSON(500, api.LFSLockError{ Message: "unable to list locks : " + err.Error(), diff --git a/options/locale/locale_en-US.ini b/options/locale/locale_en-US.ini index 98133cdab..c6fd3b863 100644 --- a/options/locale/locale_en-US.ini +++ b/options/locale/locale_en-US.ini @@ -1438,9 +1438,19 @@ settings.lfs_filelist=LFS files stored in this repository settings.lfs_no_lfs_files=No LFS files stored in this repository settings.lfs_findcommits=Find commits settings.lfs_lfs_file_no_commits=No Commits found for this LFS file +settings.lfs_noattribute=This path does not have the lockable attribute in the default branch settings.lfs_delete=Delete LFS file with OID %s settings.lfs_delete_warning=Deleting an LFS file may cause 'object does not exist' errors on checkout. Are you sure? settings.lfs_findpointerfiles=Find pointer files +settings.lfs_locks=Locks +settings.lfs_invalid_locking_path=Invalid path: %s +settings.lfs_invalid_lock_directory=Cannot lock directory: %s +settings.lfs_lock_already_exists=Lock already exists: %s +settings.lfs_lock=Lock +settings.lfs_lock_path=Filepath to lock... +settings.lfs_locks_no_locks=No Locks +settings.lfs_lock_file_no_exist=Locked file does not exist in default branch +settings.lfs_force_unlock=Force Unlock settings.lfs_pointers.found=Found %d blob pointer(s) - %d associated, %d unassociated (%d missing from store) settings.lfs_pointers.sha=Blob SHA settings.lfs_pointers.oid=OID diff --git a/routers/repo/lfs.go b/routers/repo/lfs.go index de5020c94..c3266844b 100644 --- a/routers/repo/lfs.go +++ b/routers/repo/lfs.go @@ -12,6 +12,7 @@ import ( "io" "io/ioutil" "os" + "path" "path/filepath" "sort" "strconv" @@ -38,6 +39,7 @@ import ( const ( tplSettingsLFS base.TplName = "repo/settings/lfs" + tplSettingsLFSLocks base.TplName = "repo/settings/lfs_locks" tplSettingsLFSFile base.TplName = "repo/settings/lfs_file" tplSettingsLFSFileFind base.TplName = "repo/settings/lfs_file_find" tplSettingsLFSPointers base.TplName = "repo/settings/lfs_pointers" @@ -58,6 +60,7 @@ func LFSFiles(ctx *context.Context) { ctx.ServerError("LFSFiles", err) return } + ctx.Data["Total"] = total pager := context.NewPagination(int(total), setting.UI.ExplorePagingNum, page, 5) ctx.Data["Title"] = ctx.Tr("repo.settings.lfs") @@ -72,6 +75,179 @@ func LFSFiles(ctx *context.Context) { ctx.HTML(200, tplSettingsLFS) } +// LFSLocks shows a repository's LFS locks +func LFSLocks(ctx *context.Context) { + if !setting.LFS.StartServer { + ctx.NotFound("LFSLocks", nil) + return + } + ctx.Data["LFSFilesLink"] = ctx.Repo.RepoLink + "/settings/lfs" + + page := ctx.QueryInt("page") + if page <= 1 { + page = 1 + } + total, err := models.CountLFSLockByRepoID(ctx.Repo.Repository.ID) + if err != nil { + ctx.ServerError("LFSLocks", err) + return + } + ctx.Data["Total"] = total + + pager := context.NewPagination(int(total), setting.UI.ExplorePagingNum, page, 5) + ctx.Data["Title"] = ctx.Tr("repo.settings.lfs_locks") + ctx.Data["PageIsSettingsLFS"] = true + lfsLocks, err := models.GetLFSLockByRepoID(ctx.Repo.Repository.ID, pager.Paginater.Current(), setting.UI.ExplorePagingNum) + if err != nil { + ctx.ServerError("LFSLocks", err) + return + } + ctx.Data["LFSLocks"] = lfsLocks + + if len(lfsLocks) == 0 { + ctx.Data["Page"] = pager + ctx.HTML(200, tplSettingsLFSLocks) + return + } + + // Clone base repo. + tmpBasePath, err := models.CreateTemporaryPath("locks") + if err != nil { + log.Error("Failed to create temporary path: %v", err) + ctx.ServerError("LFSLocks", err) + return + } + defer func() { + if err := models.RemoveTemporaryPath(tmpBasePath); err != nil { + log.Error("LFSLocks: RemoveTemporaryPath: %v", err) + } + }() + + if err := git.Clone(ctx.Repo.Repository.RepoPath(), tmpBasePath, git.CloneRepoOptions{ + Bare: true, + Shared: true, + }); err != nil { + log.Error("Failed to clone repository: %s (%v)", ctx.Repo.Repository.FullName(), err) + ctx.ServerError("LFSLocks", fmt.Errorf("Failed to clone repository: %s (%v)", ctx.Repo.Repository.FullName(), err)) + } + + gitRepo, err := git.OpenRepository(tmpBasePath) + if err != nil { + log.Error("Unable to open temporary repository: %s (%v)", tmpBasePath, err) + ctx.ServerError("LFSLocks", fmt.Errorf("Failed to open new temporary repository in: %s %v", tmpBasePath, err)) + } + + filenames := make([]string, len(lfsLocks)) + + for i, lock := range lfsLocks { + filenames[i] = lock.Path + } + + if err := gitRepo.ReadTreeToIndex(ctx.Repo.Repository.DefaultBranch); err != nil { + log.Error("Unable to read the default branch to the index: %s (%v)", ctx.Repo.Repository.DefaultBranch, err) + ctx.ServerError("LFSLocks", fmt.Errorf("Unable to read the default branch to the index: %s (%v)", ctx.Repo.Repository.DefaultBranch, err)) + } + + name2attribute2info, err := gitRepo.CheckAttribute(git.CheckAttributeOpts{ + Attributes: []string{"lockable"}, + Filenames: filenames, + CachedOnly: true, + }) + if err != nil { + log.Error("Unable to check attributes in %s (%v)", tmpBasePath, err) + ctx.ServerError("LFSLocks", err) + } + + lockables := make([]bool, len(lfsLocks)) + for i, lock := range lfsLocks { + attribute2info, has := name2attribute2info[lock.Path] + if !has { + continue + } + if attribute2info["lockable"] != "set" { + continue + } + lockables[i] = true + } + ctx.Data["Lockables"] = lockables + + filelist, err := gitRepo.LsFiles(filenames...) + if err != nil { + log.Error("Unable to lsfiles in %s (%v)", tmpBasePath, err) + ctx.ServerError("LFSLocks", err) + } + + filemap := make(map[string]bool, len(filelist)) + for _, name := range filelist { + filemap[name] = true + } + + linkable := make([]bool, len(lfsLocks)) + for i, lock := range lfsLocks { + linkable[i] = filemap[lock.Path] + } + ctx.Data["Linkable"] = linkable + + ctx.Data["Page"] = pager + ctx.HTML(200, tplSettingsLFSLocks) +} + +// LFSLockFile locks a file +func LFSLockFile(ctx *context.Context) { + if !setting.LFS.StartServer { + ctx.NotFound("LFSLocks", nil) + return + } + originalPath := ctx.Query("path") + lockPath := originalPath + if len(lockPath) == 0 { + ctx.Flash.Error(ctx.Tr("repo.settings.lfs_invalid_locking_path", originalPath)) + ctx.Redirect(ctx.Repo.RepoLink + "/settings/lfs/locks") + return + } + if lockPath[len(lockPath)-1] == '/' { + ctx.Flash.Error(ctx.Tr("repo.settings.lfs_invalid_lock_directory", originalPath)) + ctx.Redirect(ctx.Repo.RepoLink + "/settings/lfs/locks") + return + } + lockPath = path.Clean("/" + lockPath)[1:] + if len(lockPath) == 0 { + ctx.Flash.Error(ctx.Tr("repo.settings.lfs_invalid_locking_path", originalPath)) + ctx.Redirect(ctx.Repo.RepoLink + "/settings/lfs/locks") + return + } + + _, err := models.CreateLFSLock(&models.LFSLock{ + Repo: ctx.Repo.Repository, + Path: lockPath, + Owner: ctx.User, + }) + if err != nil { + if models.IsErrLFSLockAlreadyExist(err) { + ctx.Flash.Error(ctx.Tr("repo.settings.lfs_lock_already_exists", originalPath)) + ctx.Redirect(ctx.Repo.RepoLink + "/settings/lfs/locks") + return + } + ctx.ServerError("LFSLockFile", err) + return + } + ctx.Redirect(ctx.Repo.RepoLink + "/settings/lfs/locks") +} + +// LFSUnlock forcibly unlocks an LFS lock +func LFSUnlock(ctx *context.Context) { + if !setting.LFS.StartServer { + ctx.NotFound("LFSUnlock", nil) + return + } + _, err := models.DeleteLFSLockByID(ctx.ParamsInt64("lid"), ctx.User, true) + if err != nil { + ctx.ServerError("LFSUnlock", err) + return + } + ctx.Redirect(ctx.Repo.RepoLink + "/settings/lfs/locks") +} + // LFSFileGet serves a single LFS file func LFSFileGet(ctx *context.Context) { if !setting.LFS.StartServer { diff --git a/routers/routes/routes.go b/routers/routes/routes.go index cdbbfaee0..cfd4a6097 100644 --- a/routers/routes/routes.go +++ b/routers/routes/routes.go @@ -685,6 +685,11 @@ func RegisterRoutes(m *macaron.Macaron) { m.Get("/pointers", repo.LFSPointerFiles) m.Post("/pointers/associate", repo.LFSAutoAssociate) m.Get("/find", repo.LFSFileFind) + m.Group("/locks", func() { + m.Get("/", repo.LFSLocks) + m.Post("/", repo.LFSLockFile) + m.Post("/:lid/unlock", repo.LFSUnlock) + }) }) }, func(ctx *context.Context) { diff --git a/templates/repo/settings/lfs.tmpl b/templates/repo/settings/lfs.tmpl index e4480a8b9..f43f9479a 100644 --- a/templates/repo/settings/lfs.tmpl +++ b/templates/repo/settings/lfs.tmpl @@ -5,9 +5,10 @@
{{template "base/alert" .}}

- {{.i18n.Tr "repo.settings.lfs_filelist"}} + {{.i18n.Tr "repo.settings.lfs_filelist"}} ({{.i18n.Tr "admin.total" .Total}})
- {{.i18n.Tr "repo.settings.lfs_findpointerfiles"}} + {{.i18n.Tr "repo.settings.lfs_locks"}} +  {{.i18n.Tr "repo.settings.lfs_findpointerfiles"}}

diff --git a/templates/repo/settings/lfs_locks.tmpl b/templates/repo/settings/lfs_locks.tmpl new file mode 100644 index 000000000..8a5f6e165 --- /dev/null +++ b/templates/repo/settings/lfs_locks.tmpl @@ -0,0 +1,61 @@ +{{template "base/head" .}} +
+ {{template "repo/header" .}} + {{template "repo/settings/navbar" .}} +
+ {{template "base/alert" .}} +
+

+ {{.i18n.Tr "repo.settings.lfs"}} / {{.i18n.Tr "repo.settings.lfs_locks"}} ({{.i18n.Tr "admin.total" .Total}}) +

+
+
+ {{$.CsrfTokenHtml}} +
+ + +
+ +
+
+ + {{range $index, $lock := .LFSLocks}} + + + + + + + {{else}} + + + + {{end}} + +
+ {{if index $.Linkable $index}} + + {{$lock.Path}} + {{else}} + + {{$lock.Path}} + {{end}} + {{if not (index $.Lockables $index)}} + + {{end}} + + + + {{$lock.Owner.DisplayName}} + + {{TimeSince .Created $.Lang}} +
+ {{$.CsrfTokenHtml}} + +
+
{{.i18n.Tr "repo.settings.lfs_locks_no_locks"}}
+ {{template "base/paginate" .}} +
+ + +{{template "base/footer" .}} diff --git a/web_src/less/_base.less b/web_src/less/_base.less index 0fb12878f..34a647f9a 100644 --- a/web_src/less/_base.less +++ b/web_src/less/_base.less @@ -1112,3 +1112,7 @@ i.icon.centerlock { background: #fff866; } } + +.octicon-tiny { + font-size: 0.85714286rem; +}