From 296814e887f9bcf0b1d44552deaf40e89e08ab50 Mon Sep 17 00:00:00 2001 From: zeripath Date: Tue, 12 Feb 2019 13:07:31 +0000 Subject: [PATCH] Refactor editor upload, update and delete to use git plumbing and add LFS support (#5702) * Use git plumbing for upload: #5621 repo_editor.go: UploadRepoFile * Use git plumbing for upload: #5621 repo_editor.go: GetDiffPreview * Use git plumbing for upload: #5621 repo_editor.go: DeleteRepoFile * Use git plumbing for upload: #5621 repo_editor.go: UploadRepoFiles * Move branch checkout functions out of repo_editor.go as they are no longer used there * BUGFIX: The default permissions should be 100644 This is a change from the previous code but is more in keeping with the default behaviour of git. Signed-off-by: Andrew Thornton * Standardise cleanUploadFilename to more closely match git See verify_path in: https://github.com/git/git/blob/7f4e64169352e03476b0ea64e7e2973669e491a2/read-cache.c#L951 Signed-off-by: Andrew Thornton * Redirect on bad paths Signed-off-by: Andrew Thornton * Refactor to move the uploading functions out to a module Signed-off-by: Andrew Thornton * Add LFS support Signed-off-by: Andrew Thornton * Update upload.go attribution header Upload.go is essentially the remnants of repo_editor.go. The remaining code is essentially unchanged from the Gogs code, hence the Gogs attribution. * Delete upload files after session committed * Ensure that GIT_AUTHOR_NAME etc. are valid for git see #5774 Signed-off-by: Andrew Thornton * Add in test cases per @lafriks comment * Add space between gitea and github imports Signed-off-by: Andrew Thornton * more examples in TestCleanUploadName Signed-off-by: Andrew Thornton * fix formatting Signed-off-by: Andrew Thornton * Set the SSH_ORIGINAL_COMMAND to ensure hooks are run Signed-off-by: Andrew Thornton * Switch off SSH_ORIGINAL_COMMAND Signed-off-by: Andrew Thornton --- models/lfs.go | 19 ++ models/repo_branch.go | 40 +++ models/repo_editor.go | 572 ------------------------------------ models/upload.go | 155 ++++++++++ modules/uploader/delete.go | 100 +++++++ modules/uploader/diff.go | 38 +++ modules/uploader/repo.go | 359 ++++++++++++++++++++++ modules/uploader/update.go | 159 ++++++++++ modules/uploader/upload.go | 206 +++++++++++++ routers/repo/editor.go | 59 ++-- routers/repo/editor_test.go | 26 +- 11 files changed, 1135 insertions(+), 598 deletions(-) delete mode 100644 models/repo_editor.go create mode 100644 models/upload.go create mode 100644 modules/uploader/delete.go create mode 100644 modules/uploader/diff.go create mode 100644 modules/uploader/repo.go create mode 100644 modules/uploader/update.go create mode 100644 modules/uploader/upload.go diff --git a/models/lfs.go b/models/lfs.go index 39b0b2dd6..94d3f5790 100644 --- a/models/lfs.go +++ b/models/lfs.go @@ -1,7 +1,11 @@ package models import ( + "crypto/sha256" + "encoding/hex" "errors" + "fmt" + "io" "code.gitea.io/gitea/modules/util" ) @@ -16,6 +20,11 @@ type LFSMetaObject struct { CreatedUnix util.TimeStamp `xorm:"created"` } +// Pointer returns the string representation of an LFS pointer file +func (m *LFSMetaObject) Pointer() string { + return fmt.Sprintf("%s\n%s%s\nsize %d\n", LFSMetaFileIdentifier, LFSMetaFileOidPrefix, m.Oid, m.Size) +} + // LFSTokenResponse defines the JSON structure in which the JWT token is stored. // This structure is fetched via SSH and passed by the Git LFS client to the server // endpoint for authorization. @@ -67,6 +76,16 @@ func NewLFSMetaObject(m *LFSMetaObject) (*LFSMetaObject, error) { return m, sess.Commit() } +// GenerateLFSOid generates a Sha256Sum to represent an oid for arbitrary content +func GenerateLFSOid(content io.Reader) (string, error) { + h := sha256.New() + if _, err := io.Copy(h, content); err != nil { + return "", err + } + sum := h.Sum(nil) + return hex.EncodeToString(sum), nil +} + // GetLFSMetaObjectByOid selects a LFSMetaObject entry from database by its OID. // It may return ErrLFSObjectNotExist or a database error. If the error is nil, // the returned pointer is a valid LFSMetaObject. diff --git a/models/repo_branch.go b/models/repo_branch.go index cd12742ba..88417cbd3 100644 --- a/models/repo_branch.go +++ b/models/repo_branch.go @@ -14,6 +14,46 @@ import ( "github.com/Unknwon/com" ) +// discardLocalRepoBranchChanges discards local commits/changes of +// given branch to make sure it is even to remote branch. +func discardLocalRepoBranchChanges(localPath, branch string) error { + if !com.IsExist(localPath) { + return nil + } + // No need to check if nothing in the repository. + if !git.IsBranchExist(localPath, branch) { + return nil + } + + refName := "origin/" + branch + if err := git.ResetHEAD(localPath, true, refName); err != nil { + return fmt.Errorf("git reset --hard %s: %v", refName, err) + } + return nil +} + +// DiscardLocalRepoBranchChanges discards the local repository branch changes +func (repo *Repository) DiscardLocalRepoBranchChanges(branch string) error { + return discardLocalRepoBranchChanges(repo.LocalCopyPath(), branch) +} + +// checkoutNewBranch checks out to a new branch from the a branch name. +func checkoutNewBranch(repoPath, localPath, oldBranch, newBranch string) error { + if err := git.Checkout(localPath, git.CheckoutOptions{ + Timeout: time.Duration(setting.Git.Timeout.Pull) * time.Second, + Branch: newBranch, + OldBranch: oldBranch, + }); err != nil { + return fmt.Errorf("git checkout -b %s %s: %v", newBranch, oldBranch, err) + } + return nil +} + +// CheckoutNewBranch checks out a new branch +func (repo *Repository) CheckoutNewBranch(oldBranch, newBranch string) error { + return checkoutNewBranch(repo.RepoPath(), repo.LocalCopyPath(), oldBranch, newBranch) +} + // Branch holds the branch information type Branch struct { Path string diff --git a/models/repo_editor.go b/models/repo_editor.go deleted file mode 100644 index 1adaa2c95..000000000 --- a/models/repo_editor.go +++ /dev/null @@ -1,572 +0,0 @@ -// Copyright 2016 The Gogs Authors. All rights reserved. -// Use of this source code is governed by a MIT-style -// license that can be found in the LICENSE file. - -package models - -import ( - "fmt" - "io" - "io/ioutil" - "mime/multipart" - "os" - "os/exec" - "path" - "path/filepath" - "time" - - "github.com/Unknwon/com" - gouuid "github.com/satori/go.uuid" - - "code.gitea.io/git" - - "code.gitea.io/gitea/modules/log" - "code.gitea.io/gitea/modules/process" - "code.gitea.io/gitea/modules/setting" -) - -// ___________ .___.__ __ ___________.__.__ -// \_ _____/ __| _/|__|/ |_ \_ _____/|__| | ____ -// | __)_ / __ | | \ __\ | __) | | | _/ __ \ -// | \/ /_/ | | || | | \ | | |_\ ___/ -// /_______ /\____ | |__||__| \___ / |__|____/\___ > -// \/ \/ \/ \/ - -// discardLocalRepoBranchChanges discards local commits/changes of -// given branch to make sure it is even to remote branch. -func discardLocalRepoBranchChanges(localPath, branch string) error { - if !com.IsExist(localPath) { - return nil - } - // No need to check if nothing in the repository. - if !git.IsBranchExist(localPath, branch) { - return nil - } - - refName := "origin/" + branch - if err := git.ResetHEAD(localPath, true, refName); err != nil { - return fmt.Errorf("git reset --hard %s: %v", refName, err) - } - return nil -} - -// DiscardLocalRepoBranchChanges discards the local repository branch changes -func (repo *Repository) DiscardLocalRepoBranchChanges(branch string) error { - return discardLocalRepoBranchChanges(repo.LocalCopyPath(), branch) -} - -// checkoutNewBranch checks out to a new branch from the a branch name. -func checkoutNewBranch(repoPath, localPath, oldBranch, newBranch string) error { - if err := git.Checkout(localPath, git.CheckoutOptions{ - Timeout: time.Duration(setting.Git.Timeout.Pull) * time.Second, - Branch: newBranch, - OldBranch: oldBranch, - }); err != nil { - return fmt.Errorf("git checkout -b %s %s: %v", newBranch, oldBranch, err) - } - return nil -} - -// CheckoutNewBranch checks out a new branch -func (repo *Repository) CheckoutNewBranch(oldBranch, newBranch string) error { - return checkoutNewBranch(repo.RepoPath(), repo.LocalCopyPath(), oldBranch, newBranch) -} - -// UpdateRepoFileOptions holds the repository file update options -type UpdateRepoFileOptions struct { - LastCommitID string - OldBranch string - NewBranch string - OldTreeName string - NewTreeName string - Message string - Content string - IsNewFile bool -} - -// UpdateRepoFile adds or updates a file in repository. -func (repo *Repository) UpdateRepoFile(doer *User, opts UpdateRepoFileOptions) (err error) { - repoWorkingPool.CheckIn(com.ToStr(repo.ID)) - defer repoWorkingPool.CheckOut(com.ToStr(repo.ID)) - - if err = repo.DiscardLocalRepoBranchChanges(opts.OldBranch); err != nil { - return fmt.Errorf("DiscardLocalRepoBranchChanges [branch: %s]: %v", opts.OldBranch, err) - } else if err = repo.UpdateLocalCopyBranch(opts.OldBranch); err != nil { - return fmt.Errorf("UpdateLocalCopyBranch [branch: %s]: %v", opts.OldBranch, err) - } - - if opts.OldBranch != opts.NewBranch { - if err := repo.CheckoutNewBranch(opts.OldBranch, opts.NewBranch); err != nil { - return fmt.Errorf("CheckoutNewBranch [old_branch: %s, new_branch: %s]: %v", opts.OldBranch, opts.NewBranch, err) - } - } - - localPath := repo.LocalCopyPath() - oldFilePath := path.Join(localPath, opts.OldTreeName) - filePath := path.Join(localPath, opts.NewTreeName) - dir := path.Dir(filePath) - - if err := os.MkdirAll(dir, os.ModePerm); err != nil { - return fmt.Errorf("Failed to create dir %s: %v", dir, err) - } - - // If it's meant to be a new file, make sure it doesn't exist. - if opts.IsNewFile { - if com.IsExist(filePath) { - return ErrRepoFileAlreadyExist{filePath} - } - } - - // Ignore move step if it's a new file under a directory. - // Otherwise, move the file when name changed. - if com.IsFile(oldFilePath) && opts.OldTreeName != opts.NewTreeName { - if err = git.MoveFile(localPath, opts.OldTreeName, opts.NewTreeName); err != nil { - return fmt.Errorf("git mv %s %s: %v", opts.OldTreeName, opts.NewTreeName, err) - } - } - - if err = ioutil.WriteFile(filePath, []byte(opts.Content), 0666); err != nil { - return fmt.Errorf("WriteFile: %v", err) - } - - if err = git.AddChanges(localPath, true); err != nil { - return fmt.Errorf("git add --all: %v", err) - } else if err = git.CommitChanges(localPath, git.CommitChangesOptions{ - Committer: doer.NewGitSig(), - Message: opts.Message, - }); err != nil { - return fmt.Errorf("CommitChanges: %v", err) - } else if err = git.Push(localPath, git.PushOptions{ - Remote: "origin", - Branch: opts.NewBranch, - }); err != nil { - return fmt.Errorf("git push origin %s: %v", opts.NewBranch, err) - } - - gitRepo, err := git.OpenRepository(repo.RepoPath()) - if err != nil { - log.Error(4, "OpenRepository: %v", err) - return nil - } - commit, err := gitRepo.GetBranchCommit(opts.NewBranch) - if err != nil { - log.Error(4, "GetBranchCommit [branch: %s]: %v", opts.NewBranch, err) - return nil - } - - // Simulate push event. - oldCommitID := opts.LastCommitID - if opts.NewBranch != opts.OldBranch { - oldCommitID = git.EmptySHA - } - - if err = repo.GetOwner(); err != nil { - return fmt.Errorf("GetOwner: %v", err) - } - err = PushUpdate( - opts.NewBranch, - PushUpdateOptions{ - PusherID: doer.ID, - PusherName: doer.Name, - RepoUserName: repo.Owner.Name, - RepoName: repo.Name, - RefFullName: git.BranchPrefix + opts.NewBranch, - OldCommitID: oldCommitID, - NewCommitID: commit.ID.String(), - }, - ) - if err != nil { - return fmt.Errorf("PushUpdate: %v", err) - } - UpdateRepoIndexer(repo) - - return nil -} - -// GetDiffPreview produces and returns diff result of a file which is not yet committed. -func (repo *Repository) GetDiffPreview(branch, treePath, content string) (diff *Diff, err error) { - repoWorkingPool.CheckIn(com.ToStr(repo.ID)) - defer repoWorkingPool.CheckOut(com.ToStr(repo.ID)) - - if err = repo.DiscardLocalRepoBranchChanges(branch); err != nil { - return nil, fmt.Errorf("DiscardLocalRepoBranchChanges [branch: %s]: %v", branch, err) - } else if err = repo.UpdateLocalCopyBranch(branch); err != nil { - return nil, fmt.Errorf("UpdateLocalCopyBranch [branch: %s]: %v", branch, err) - } - - localPath := repo.LocalCopyPath() - filePath := path.Join(localPath, treePath) - dir := filepath.Dir(filePath) - - if err := os.MkdirAll(dir, os.ModePerm); err != nil { - return nil, fmt.Errorf("Failed to create dir %s: %v", dir, err) - } - - if err = ioutil.WriteFile(filePath, []byte(content), 0666); err != nil { - return nil, fmt.Errorf("WriteFile: %v", err) - } - - cmd := exec.Command("git", "diff", treePath) - cmd.Dir = localPath - cmd.Stderr = os.Stderr - - stdout, err := cmd.StdoutPipe() - if err != nil { - return nil, fmt.Errorf("StdoutPipe: %v", err) - } - - if err = cmd.Start(); err != nil { - return nil, fmt.Errorf("Start: %v", err) - } - - pid := process.GetManager().Add(fmt.Sprintf("GetDiffPreview [repo_path: %s]", repo.RepoPath()), cmd) - defer process.GetManager().Remove(pid) - - diff, err = ParsePatch(setting.Git.MaxGitDiffLines, setting.Git.MaxGitDiffLineCharacters, setting.Git.MaxGitDiffFiles, stdout) - if err != nil { - return nil, fmt.Errorf("ParsePatch: %v", err) - } - - if err = cmd.Wait(); err != nil { - return nil, fmt.Errorf("Wait: %v", err) - } - - return diff, nil -} - -// ________ .__ __ ___________.__.__ -// \______ \ ____ | | _____/ |_ ____ \_ _____/|__| | ____ -// | | \_/ __ \| | _/ __ \ __\/ __ \ | __) | | | _/ __ \ -// | ` \ ___/| |_\ ___/| | \ ___/ | \ | | |_\ ___/ -// /_______ /\___ >____/\___ >__| \___ > \___ / |__|____/\___ > -// \/ \/ \/ \/ \/ \/ -// - -// DeleteRepoFileOptions holds the repository delete file options -type DeleteRepoFileOptions struct { - LastCommitID string - OldBranch string - NewBranch string - TreePath string - Message string -} - -// DeleteRepoFile deletes a repository file -func (repo *Repository) DeleteRepoFile(doer *User, opts DeleteRepoFileOptions) (err error) { - repoWorkingPool.CheckIn(com.ToStr(repo.ID)) - defer repoWorkingPool.CheckOut(com.ToStr(repo.ID)) - - if err = repo.DiscardLocalRepoBranchChanges(opts.OldBranch); err != nil { - return fmt.Errorf("DiscardLocalRepoBranchChanges [branch: %s]: %v", opts.OldBranch, err) - } else if err = repo.UpdateLocalCopyBranch(opts.OldBranch); err != nil { - return fmt.Errorf("UpdateLocalCopyBranch [branch: %s]: %v", opts.OldBranch, err) - } - - if opts.OldBranch != opts.NewBranch { - if err := repo.CheckoutNewBranch(opts.OldBranch, opts.NewBranch); err != nil { - return fmt.Errorf("CheckoutNewBranch [old_branch: %s, new_branch: %s]: %v", opts.OldBranch, opts.NewBranch, err) - } - } - - localPath := repo.LocalCopyPath() - if err = os.Remove(path.Join(localPath, opts.TreePath)); err != nil { - return fmt.Errorf("Remove: %v", err) - } - - if err = git.AddChanges(localPath, true); err != nil { - return fmt.Errorf("git add --all: %v", err) - } else if err = git.CommitChanges(localPath, git.CommitChangesOptions{ - Committer: doer.NewGitSig(), - Message: opts.Message, - }); err != nil { - return fmt.Errorf("CommitChanges: %v", err) - } else if err = git.Push(localPath, git.PushOptions{ - Remote: "origin", - Branch: opts.NewBranch, - }); err != nil { - return fmt.Errorf("git push origin %s: %v", opts.NewBranch, err) - } - - gitRepo, err := git.OpenRepository(repo.RepoPath()) - if err != nil { - log.Error(4, "OpenRepository: %v", err) - return nil - } - commit, err := gitRepo.GetBranchCommit(opts.NewBranch) - if err != nil { - log.Error(4, "GetBranchCommit [branch: %s]: %v", opts.NewBranch, err) - return nil - } - - // Simulate push event. - oldCommitID := opts.LastCommitID - if opts.NewBranch != opts.OldBranch { - oldCommitID = git.EmptySHA - } - - if err = repo.GetOwner(); err != nil { - return fmt.Errorf("GetOwner: %v", err) - } - err = PushUpdate( - opts.NewBranch, - PushUpdateOptions{ - PusherID: doer.ID, - PusherName: doer.Name, - RepoUserName: repo.Owner.Name, - RepoName: repo.Name, - RefFullName: git.BranchPrefix + opts.NewBranch, - OldCommitID: oldCommitID, - NewCommitID: commit.ID.String(), - }, - ) - if err != nil { - return fmt.Errorf("PushUpdate: %v", err) - } - return nil -} - -// ____ ___ .__ .___ ___________.___.__ -// | | \______ | | _________ __| _/ \_ _____/| | | ____ ______ -// | | /\____ \| | / _ \__ \ / __ | | __) | | | _/ __ \ / ___/ -// | | / | |_> > |_( <_> ) __ \_/ /_/ | | \ | | |_\ ___/ \___ \ -// |______/ | __/|____/\____(____ /\____ | \___ / |___|____/\___ >____ > -// |__| \/ \/ \/ \/ \/ -// - -// Upload represent a uploaded file to a repo to be deleted when moved -type Upload struct { - ID int64 `xorm:"pk autoincr"` - UUID string `xorm:"uuid UNIQUE"` - Name string -} - -// UploadLocalPath returns where uploads is stored in local file system based on given UUID. -func UploadLocalPath(uuid string) string { - return path.Join(setting.Repository.Upload.TempPath, uuid[0:1], uuid[1:2], uuid) -} - -// LocalPath returns where uploads are temporarily stored in local file system. -func (upload *Upload) LocalPath() string { - return UploadLocalPath(upload.UUID) -} - -// NewUpload creates a new upload object. -func NewUpload(name string, buf []byte, file multipart.File) (_ *Upload, err error) { - upload := &Upload{ - UUID: gouuid.NewV4().String(), - Name: name, - } - - localPath := upload.LocalPath() - if err = os.MkdirAll(path.Dir(localPath), os.ModePerm); err != nil { - return nil, fmt.Errorf("MkdirAll: %v", err) - } - - fw, err := os.Create(localPath) - if err != nil { - return nil, fmt.Errorf("Create: %v", err) - } - defer fw.Close() - - if _, err = fw.Write(buf); err != nil { - return nil, fmt.Errorf("Write: %v", err) - } else if _, err = io.Copy(fw, file); err != nil { - return nil, fmt.Errorf("Copy: %v", err) - } - - if _, err := x.Insert(upload); err != nil { - return nil, err - } - - return upload, nil -} - -// GetUploadByUUID returns the Upload by UUID -func GetUploadByUUID(uuid string) (*Upload, error) { - upload := &Upload{UUID: uuid} - has, err := x.Get(upload) - if err != nil { - return nil, err - } else if !has { - return nil, ErrUploadNotExist{0, uuid} - } - return upload, nil -} - -// GetUploadsByUUIDs returns multiple uploads by UUIDS -func GetUploadsByUUIDs(uuids []string) ([]*Upload, error) { - if len(uuids) == 0 { - return []*Upload{}, nil - } - - // Silently drop invalid uuids. - uploads := make([]*Upload, 0, len(uuids)) - return uploads, x.In("uuid", uuids).Find(&uploads) -} - -// DeleteUploads deletes multiple uploads -func DeleteUploads(uploads ...*Upload) (err error) { - if len(uploads) == 0 { - return nil - } - - sess := x.NewSession() - defer sess.Close() - if err = sess.Begin(); err != nil { - return err - } - - ids := make([]int64, len(uploads)) - for i := 0; i < len(uploads); i++ { - ids[i] = uploads[i].ID - } - if _, err = sess. - In("id", ids). - Delete(new(Upload)); err != nil { - return fmt.Errorf("delete uploads: %v", err) - } - - for _, upload := range uploads { - localPath := upload.LocalPath() - if !com.IsFile(localPath) { - continue - } - - if err := os.Remove(localPath); err != nil { - return fmt.Errorf("remove upload: %v", err) - } - } - - return sess.Commit() -} - -// DeleteUpload delete a upload -func DeleteUpload(u *Upload) error { - return DeleteUploads(u) -} - -// DeleteUploadByUUID deletes a upload by UUID -func DeleteUploadByUUID(uuid string) error { - upload, err := GetUploadByUUID(uuid) - if err != nil { - if IsErrUploadNotExist(err) { - return nil - } - return fmt.Errorf("GetUploadByUUID: %v", err) - } - - if err := DeleteUpload(upload); err != nil { - return fmt.Errorf("DeleteUpload: %v", err) - } - - return nil -} - -// UploadRepoFileOptions contains the uploaded repository file options -type UploadRepoFileOptions struct { - LastCommitID string - OldBranch string - NewBranch string - TreePath string - Message string - Files []string // In UUID format. -} - -// UploadRepoFiles uploads files to a repository -func (repo *Repository) UploadRepoFiles(doer *User, opts UploadRepoFileOptions) (err error) { - if len(opts.Files) == 0 { - return nil - } - - uploads, err := GetUploadsByUUIDs(opts.Files) - if err != nil { - return fmt.Errorf("GetUploadsByUUIDs [uuids: %v]: %v", opts.Files, err) - } - - repoWorkingPool.CheckIn(com.ToStr(repo.ID)) - defer repoWorkingPool.CheckOut(com.ToStr(repo.ID)) - - if err = repo.DiscardLocalRepoBranchChanges(opts.OldBranch); err != nil { - return fmt.Errorf("DiscardLocalRepoBranchChanges [branch: %s]: %v", opts.OldBranch, err) - } else if err = repo.UpdateLocalCopyBranch(opts.OldBranch); err != nil { - return fmt.Errorf("UpdateLocalCopyBranch [branch: %s]: %v", opts.OldBranch, err) - } - - if opts.OldBranch != opts.NewBranch { - if err = repo.CheckoutNewBranch(opts.OldBranch, opts.NewBranch); err != nil { - return fmt.Errorf("CheckoutNewBranch [old_branch: %s, new_branch: %s]: %v", opts.OldBranch, opts.NewBranch, err) - } - } - - localPath := repo.LocalCopyPath() - dirPath := path.Join(localPath, opts.TreePath) - - if err := os.MkdirAll(dirPath, os.ModePerm); err != nil { - return fmt.Errorf("Failed to create dir %s: %v", dirPath, err) - } - - // Copy uploaded files into repository. - for _, upload := range uploads { - tmpPath := upload.LocalPath() - targetPath := path.Join(dirPath, upload.Name) - if !com.IsFile(tmpPath) { - continue - } - - if err = com.Copy(tmpPath, targetPath); err != nil { - return fmt.Errorf("Copy: %v", err) - } - } - - if err = git.AddChanges(localPath, true); err != nil { - return fmt.Errorf("git add --all: %v", err) - } else if err = git.CommitChanges(localPath, git.CommitChangesOptions{ - Committer: doer.NewGitSig(), - Message: opts.Message, - }); err != nil { - return fmt.Errorf("CommitChanges: %v", err) - } else if err = git.Push(localPath, git.PushOptions{ - Remote: "origin", - Branch: opts.NewBranch, - }); err != nil { - return fmt.Errorf("git push origin %s: %v", opts.NewBranch, err) - } - - gitRepo, err := git.OpenRepository(repo.RepoPath()) - if err != nil { - log.Error(4, "OpenRepository: %v", err) - return nil - } - commit, err := gitRepo.GetBranchCommit(opts.NewBranch) - if err != nil { - log.Error(4, "GetBranchCommit [branch: %s]: %v", opts.NewBranch, err) - return nil - } - - // Simulate push event. - oldCommitID := opts.LastCommitID - if opts.NewBranch != opts.OldBranch { - oldCommitID = git.EmptySHA - } - - if err = repo.GetOwner(); err != nil { - return fmt.Errorf("GetOwner: %v", err) - } - err = PushUpdate( - opts.NewBranch, - PushUpdateOptions{ - PusherID: doer.ID, - PusherName: doer.Name, - RepoUserName: repo.Owner.Name, - RepoName: repo.Name, - RefFullName: git.BranchPrefix + opts.NewBranch, - OldCommitID: oldCommitID, - NewCommitID: commit.ID.String(), - }, - ) - if err != nil { - return fmt.Errorf("PushUpdate: %v", err) - } - - return DeleteUploads(uploads...) -} diff --git a/models/upload.go b/models/upload.go new file mode 100644 index 000000000..65ce1223c --- /dev/null +++ b/models/upload.go @@ -0,0 +1,155 @@ +// Copyright 2016 The Gogs Authors. All rights reserved. +// Copyright 2019 The Gitea Authors. All rights reserved. +// Use of this source code is governed by a MIT-style +// license that can be found in the LICENSE file. + +package models + +import ( + "fmt" + "io" + "mime/multipart" + "os" + "path" + + "github.com/Unknwon/com" + gouuid "github.com/satori/go.uuid" + + "code.gitea.io/gitea/modules/setting" +) + +// ____ ___ .__ .___ ___________.___.__ +// | | \______ | | _________ __| _/ \_ _____/| | | ____ ______ +// | | /\____ \| | / _ \__ \ / __ | | __) | | | _/ __ \ / ___/ +// | | / | |_> > |_( <_> ) __ \_/ /_/ | | \ | | |_\ ___/ \___ \ +// |______/ | __/|____/\____(____ /\____ | \___ / |___|____/\___ >____ > +// |__| \/ \/ \/ \/ \/ +// + +// Upload represent a uploaded file to a repo to be deleted when moved +type Upload struct { + ID int64 `xorm:"pk autoincr"` + UUID string `xorm:"uuid UNIQUE"` + Name string +} + +// UploadLocalPath returns where uploads is stored in local file system based on given UUID. +func UploadLocalPath(uuid string) string { + return path.Join(setting.Repository.Upload.TempPath, uuid[0:1], uuid[1:2], uuid) +} + +// LocalPath returns where uploads are temporarily stored in local file system. +func (upload *Upload) LocalPath() string { + return UploadLocalPath(upload.UUID) +} + +// NewUpload creates a new upload object. +func NewUpload(name string, buf []byte, file multipart.File) (_ *Upload, err error) { + upload := &Upload{ + UUID: gouuid.NewV4().String(), + Name: name, + } + + localPath := upload.LocalPath() + if err = os.MkdirAll(path.Dir(localPath), os.ModePerm); err != nil { + return nil, fmt.Errorf("MkdirAll: %v", err) + } + + fw, err := os.Create(localPath) + if err != nil { + return nil, fmt.Errorf("Create: %v", err) + } + defer fw.Close() + + if _, err = fw.Write(buf); err != nil { + return nil, fmt.Errorf("Write: %v", err) + } else if _, err = io.Copy(fw, file); err != nil { + return nil, fmt.Errorf("Copy: %v", err) + } + + if _, err := x.Insert(upload); err != nil { + return nil, err + } + + return upload, nil +} + +// GetUploadByUUID returns the Upload by UUID +func GetUploadByUUID(uuid string) (*Upload, error) { + upload := &Upload{UUID: uuid} + has, err := x.Get(upload) + if err != nil { + return nil, err + } else if !has { + return nil, ErrUploadNotExist{0, uuid} + } + return upload, nil +} + +// GetUploadsByUUIDs returns multiple uploads by UUIDS +func GetUploadsByUUIDs(uuids []string) ([]*Upload, error) { + if len(uuids) == 0 { + return []*Upload{}, nil + } + + // Silently drop invalid uuids. + uploads := make([]*Upload, 0, len(uuids)) + return uploads, x.In("uuid", uuids).Find(&uploads) +} + +// DeleteUploads deletes multiple uploads +func DeleteUploads(uploads ...*Upload) (err error) { + if len(uploads) == 0 { + return nil + } + + sess := x.NewSession() + defer sess.Close() + if err = sess.Begin(); err != nil { + return err + } + + ids := make([]int64, len(uploads)) + for i := 0; i < len(uploads); i++ { + ids[i] = uploads[i].ID + } + if _, err = sess. + In("id", ids). + Delete(new(Upload)); err != nil { + return fmt.Errorf("delete uploads: %v", err) + } + + if err = sess.Commit(); err != nil { + return err + } + + for _, upload := range uploads { + localPath := upload.LocalPath() + if !com.IsFile(localPath) { + continue + } + + if err := os.Remove(localPath); err != nil { + return fmt.Errorf("remove upload: %v", err) + } + } + + return nil +} + +// DeleteUploadByUUID deletes a upload by UUID +func DeleteUploadByUUID(uuid string) error { + upload, err := GetUploadByUUID(uuid) + if err != nil { + if IsErrUploadNotExist(err) { + return nil + } + return fmt.Errorf("GetUploadByUUID: %v", err) + } + + if err := DeleteUploads(upload); err != nil { + return fmt.Errorf("DeleteUpload: %v", err) + } + + return nil +} diff --git a/modules/uploader/delete.go b/modules/uploader/delete.go new file mode 100644 index 000000000..fbe451c5d --- /dev/null +++ b/modules/uploader/delete.go @@ -0,0 +1,100 @@ +// Copyright 2019 The Gitea Authors. All rights reserved. +// Use of this source code is governed by a MIT-style +// license that can be found in the LICENSE file. + +package uploader + +import ( + "fmt" + + "code.gitea.io/git" + "code.gitea.io/gitea/models" +) + +// DeleteRepoFileOptions holds the repository delete file options +type DeleteRepoFileOptions struct { + LastCommitID string + OldBranch string + NewBranch string + TreePath string + Message string +} + +// DeleteRepoFile deletes a file in the given repository +func DeleteRepoFile(repo *models.Repository, doer *models.User, opts *DeleteRepoFileOptions) error { + t, err := NewTemporaryUploadRepository(repo) + defer t.Close() + if err != nil { + return err + } + if err := t.Clone(opts.OldBranch); err != nil { + return err + } + if err := t.SetDefaultIndex(); err != nil { + return err + } + + filesInIndex, err := t.LsFiles(opts.TreePath) + if err != nil { + return fmt.Errorf("UpdateRepoFile: %v", err) + } + + inFilelist := false + for _, file := range filesInIndex { + if file == opts.TreePath { + inFilelist = true + } + } + if !inFilelist { + return git.ErrNotExist{RelPath: opts.TreePath} + } + + if err := t.RemoveFilesFromIndex(opts.TreePath); err != nil { + return err + } + + // Now write the tree + treeHash, err := t.WriteTree() + if err != nil { + return err + } + + // Now commit the tree + commitHash, err := t.CommitTree(doer, treeHash, opts.Message) + if err != nil { + return err + } + + // Then push this tree to NewBranch + if err := t.Push(doer, commitHash, opts.NewBranch); err != nil { + return err + } + + // Simulate push event. + oldCommitID := opts.LastCommitID + if opts.NewBranch != opts.OldBranch { + oldCommitID = git.EmptySHA + } + + if err = repo.GetOwner(); err != nil { + return fmt.Errorf("GetOwner: %v", err) + } + err = models.PushUpdate( + opts.NewBranch, + models.PushUpdateOptions{ + PusherID: doer.ID, + PusherName: doer.Name, + RepoUserName: repo.Owner.Name, + RepoName: repo.Name, + RefFullName: git.BranchPrefix + opts.NewBranch, + OldCommitID: oldCommitID, + NewCommitID: commitHash, + }, + ) + if err != nil { + return fmt.Errorf("PushUpdate: %v", err) + } + + // FIXME: Should we UpdateRepoIndexer(repo) here? + return nil +} diff --git a/modules/uploader/diff.go b/modules/uploader/diff.go new file mode 100644 index 000000000..e01947ea6 --- /dev/null +++ b/modules/uploader/diff.go @@ -0,0 +1,38 @@ +// Copyright 2019 The Gitea Authors. All rights reserved. +// Use of this source code is governed by a MIT-style +// license that can be found in the LICENSE file. + +package uploader + +import ( + "strings" + + "code.gitea.io/gitea/models" +) + +// GetDiffPreview produces and returns diff result of a file which is not yet committed. +func GetDiffPreview(repo *models.Repository, branch, treePath, content string) (*models.Diff, error) { + t, err := NewTemporaryUploadRepository(repo) + defer t.Close() + if err != nil { + return nil, err + } + if err := t.Clone(branch); err != nil { + return nil, err + } + if err := t.SetDefaultIndex(); err != nil { + return nil, err + } + + // Add the object to the database + objectHash, err := t.HashObject(strings.NewReader(content)) + if err != nil { + return nil, err + } + + // Add the object to the index + if err := t.AddObjectToIndex("100644", objectHash, treePath); err != nil { + return nil, err + } + return t.DiffIndex() +} diff --git a/modules/uploader/repo.go b/modules/uploader/repo.go new file mode 100644 index 000000000..33cc160ca --- /dev/null +++ b/modules/uploader/repo.go @@ -0,0 +1,359 @@ +// Copyright 2019 The Gitea Authors. All rights reserved. +// Use of this source code is governed by a MIT-style +// license that can be found in the LICENSE file. + +package uploader + +import ( + "bytes" + "context" + "fmt" + "io" + "os" + "os/exec" + "path" + "strings" + "time" + + "code.gitea.io/gitea/models" + "code.gitea.io/gitea/modules/process" + "code.gitea.io/gitea/modules/setting" + + "github.com/Unknwon/com" +) + +// TemporaryUploadRepository is a type to wrap our upload repositories +type TemporaryUploadRepository struct { + repo *models.Repository + basePath string +} + +// NewTemporaryUploadRepository creates a new temporary upload repository +func NewTemporaryUploadRepository(repo *models.Repository) (*TemporaryUploadRepository, error) { + timeStr := com.ToStr(time.Now().Nanosecond()) // SHOULD USE SOMETHING UNIQUE + basePath := path.Join(models.LocalCopyPath(), "upload-"+timeStr+".git") + if err := os.MkdirAll(path.Dir(basePath), os.ModePerm); err != nil { + return nil, fmt.Errorf("Failed to create dir %s: %v", basePath, err) + } + t := &TemporaryUploadRepository{repo: repo, basePath: basePath} + return t, nil +} + +// Close the repository cleaning up all files +func (t *TemporaryUploadRepository) Close() { + if _, err := os.Stat(t.basePath); !os.IsNotExist(err) { + os.RemoveAll(t.basePath) + } +} + +// Clone the base repository to our path and set branch as the HEAD +func (t *TemporaryUploadRepository) Clone(branch string) error { + if _, stderr, err := process.GetManager().ExecTimeout(5*time.Minute, + fmt.Sprintf("Clone (git clone -s --bare): %s", t.basePath), + "git", "clone", "-s", "--bare", "-b", branch, t.repo.RepoPath(), t.basePath); err != nil { + return fmt.Errorf("Clone: %v %s", err, stderr) + } + return nil +} + +// SetDefaultIndex sets the git index to our HEAD +func (t *TemporaryUploadRepository) SetDefaultIndex() error { + if _, stderr, err := process.GetManager().ExecDir(5*time.Minute, + t.basePath, + fmt.Sprintf("SetDefaultIndex (git read-tree HEAD): %s", t.basePath), + "git", "read-tree", "HEAD"); err != nil { + return fmt.Errorf("SetDefaultIndex: %v %s", err, stderr) + } + return nil +} + +// LsFiles checks if the given filename arguments are in the index +func (t *TemporaryUploadRepository) LsFiles(filenames ...string) ([]string, error) { + stdOut := new(bytes.Buffer) + stdErr := new(bytes.Buffer) + + timeout := 5 * time.Minute + ctx, cancel := context.WithTimeout(context.Background(), timeout) + defer cancel() + + cmdArgs := []string{"ls-files", "-z", "--"} + for _, arg := range filenames { + if arg != "" { + cmdArgs = append(cmdArgs, arg) + } + } + + cmd := exec.CommandContext(ctx, "git", cmdArgs...) + desc := fmt.Sprintf("lsFiles: (git ls-files) %v", cmdArgs) + cmd.Dir = t.basePath + cmd.Stdout = stdOut + cmd.Stderr = stdErr + + if err := cmd.Start(); err != nil { + return nil, fmt.Errorf("exec(%s) failed: %v(%v)", desc, err, ctx.Err()) + } + + pid := process.GetManager().Add(desc, cmd) + err := cmd.Wait() + process.GetManager().Remove(pid) + + if err != nil { + err = fmt.Errorf("exec(%d:%s) failed: %v(%v) stdout: %v stderr: %v", pid, desc, err, ctx.Err(), stdOut, stdErr) + return nil, err + } + + filelist := make([]string, len(filenames)) + for _, line := range bytes.Split(stdOut.Bytes(), []byte{'\000'}) { + filelist = append(filelist, string(line)) + } + + return filelist, err +} + +// RemoveFilesFromIndex removes the given files from the index +func (t *TemporaryUploadRepository) RemoveFilesFromIndex(filenames ...string) error { + stdOut := new(bytes.Buffer) + stdErr := new(bytes.Buffer) + stdIn := new(bytes.Buffer) + for _, file := range filenames { + if file != "" { + stdIn.WriteString("0 0000000000000000000000000000000000000000\t") + stdIn.WriteString(file) + stdIn.WriteByte('\000') + } + } + + timeout := 5 * time.Minute + ctx, cancel := context.WithTimeout(context.Background(), timeout) + defer cancel() + + cmdArgs := []string{"update-index", "--remove", "-z", "--index-info"} + cmd := exec.CommandContext(ctx, "git", cmdArgs...) + desc := fmt.Sprintf("removeFilesFromIndex: (git update-index) %v", filenames) + cmd.Dir = t.basePath + cmd.Stdout = stdOut + cmd.Stderr = stdErr + cmd.Stdin = bytes.NewReader(stdIn.Bytes()) + + if err := cmd.Start(); err != nil { + return fmt.Errorf("exec(%s) failed: %v(%v)", desc, err, ctx.Err()) + } + + pid := process.GetManager().Add(desc, cmd) + err := cmd.Wait() + process.GetManager().Remove(pid) + + if err != nil { + err = fmt.Errorf("exec(%d:%s) failed: %v(%v) stdout: %v stderr: %v", pid, desc, err, ctx.Err(), stdOut, stdErr) + } + + return err +} + +// HashObject writes the provided content to the object db and returns its hash +func (t *TemporaryUploadRepository) HashObject(content io.Reader) (string, error) { + timeout := 5 * time.Minute + ctx, cancel := context.WithTimeout(context.Background(), timeout) + defer cancel() + + hashCmd := exec.CommandContext(ctx, "git", "hash-object", "-w", "--stdin") + hashCmd.Dir = t.basePath + hashCmd.Stdin = content + stdOutBuffer := new(bytes.Buffer) + stdErrBuffer := new(bytes.Buffer) + hashCmd.Stdout = stdOutBuffer + hashCmd.Stderr = stdErrBuffer + desc := fmt.Sprintf("hashObject: (git hash-object)") + if err := hashCmd.Start(); err != nil { + return "", fmt.Errorf("git hash-object: %s", err) + } + + pid := process.GetManager().Add(desc, hashCmd) + err := hashCmd.Wait() + process.GetManager().Remove(pid) + + if err != nil { + err = fmt.Errorf("exec(%d:%s) failed: %v(%v) stdout: %v stderr: %v", pid, desc, err, ctx.Err(), stdOutBuffer, stdErrBuffer) + return "", err + } + + return strings.TrimSpace(stdOutBuffer.String()), nil +} + +// AddObjectToIndex adds the provided object hash to the index with the provided mode and path +func (t *TemporaryUploadRepository) AddObjectToIndex(mode, objectHash, objectPath string) error { + if _, stderr, err := process.GetManager().ExecDir(5*time.Minute, + t.basePath, + fmt.Sprintf("addObjectToIndex (git update-index): %s", t.basePath), + "git", "update-index", "--add", "--replace", "--cacheinfo", mode, objectHash, objectPath); err != nil { + return fmt.Errorf("git update-index: %s", stderr) + } + return nil +} + +// WriteTree writes the current index as a tree to the object db and returns its hash +func (t *TemporaryUploadRepository) WriteTree() (string, error) { + treeHash, stderr, err := process.GetManager().ExecDir(5*time.Minute, + t.basePath, + fmt.Sprintf("WriteTree (git write-tree): %s", t.basePath), + "git", "write-tree") + if err != nil { + return "", fmt.Errorf("git write-tree: %s", stderr) + } + return strings.TrimSpace(treeHash), nil + +} + +// CommitTree creates a commit from a given tree for the user with provided message +func (t *TemporaryUploadRepository) CommitTree(doer *models.User, treeHash string, message string) (string, error) { + commitTimeStr := time.Now().Format(time.UnixDate) + sig := doer.NewGitSig() + + // FIXME: Should we add SSH_ORIGINAL_COMMAND to this + // Because this may call hooks we should pass in the environment + env := append(os.Environ(), + "GIT_AUTHOR_NAME="+sig.Name, + "GIT_AUTHOR_EMAIL="+sig.Email, + "GIT_AUTHOR_DATE="+commitTimeStr, + "GIT_COMMITTER_NAME="+sig.Name, + "GIT_COMMITTER_EMAIL="+sig.Email, + "GIT_COMMITTER_DATE="+commitTimeStr, + ) + commitHash, stderr, err := process.GetManager().ExecDirEnv(5*time.Minute, + t.basePath, + fmt.Sprintf("commitTree (git commit-tree): %s", t.basePath), + env, + "git", "commit-tree", treeHash, "-p", "HEAD", "-m", message) + if err != nil { + return "", fmt.Errorf("git commit-tree: %s", stderr) + } + return strings.TrimSpace(commitHash), nil +} + +// Push the provided commitHash to the repository branch by the provided user +func (t *TemporaryUploadRepository) Push(doer *models.User, commitHash string, branch string) error { + isWiki := "false" + if strings.HasSuffix(t.repo.Name, ".wiki") { + isWiki = "true" + } + + sig := doer.NewGitSig() + + // FIXME: Should we add SSH_ORIGINAL_COMMAND to this + // Because calls hooks we need to pass in the environment + env := append(os.Environ(), + "GIT_AUTHOR_NAME="+sig.Name, + "GIT_AUTHOR_EMAIL="+sig.Email, + "GIT_COMMITTER_NAME="+sig.Name, + "GIT_COMMITTER_EMAIL="+sig.Email, + models.EnvRepoName+"="+t.repo.Name, + models.EnvRepoUsername+"="+t.repo.OwnerName, + models.EnvRepoIsWiki+"="+isWiki, + models.EnvPusherName+"="+doer.Name, + models.EnvPusherID+"="+fmt.Sprintf("%d", doer.ID), + models.ProtectedBranchRepoID+"="+fmt.Sprintf("%d", t.repo.ID), + ) + + if _, stderr, err := process.GetManager().ExecDirEnv(5*time.Minute, + t.basePath, + fmt.Sprintf("actuallyPush (git push): %s", t.basePath), + env, + "git", "push", t.repo.RepoPath(), strings.TrimSpace(commitHash)+":refs/heads/"+strings.TrimSpace(branch)); err != nil { + return fmt.Errorf("git push: %s", stderr) + } + return nil +} + +// DiffIndex returns a Diff of the current index to the head +func (t *TemporaryUploadRepository) DiffIndex() (diff *models.Diff, err error) { + timeout := 5 * time.Minute + ctx, cancel := context.WithTimeout(context.Background(), timeout) + defer cancel() + + stdErr := new(bytes.Buffer) + + cmd := exec.CommandContext(ctx, "git", "diff-index", "--cached", "-p", "HEAD") + cmd.Dir = t.basePath + cmd.Stderr = stdErr + + stdout, err := cmd.StdoutPipe() + if err != nil { + return nil, fmt.Errorf("StdoutPipe: %v stderr %s", err, stdErr.String()) + } + + if err = cmd.Start(); err != nil { + return nil, fmt.Errorf("Start: %v stderr %s", err, stdErr.String()) + } + + pid := process.GetManager().Add(fmt.Sprintf("diffIndex [repo_path: %s]", t.repo.RepoPath()), cmd) + defer process.GetManager().Remove(pid) + + diff, err = models.ParsePatch(setting.Git.MaxGitDiffLines, setting.Git.MaxGitDiffLineCharacters, setting.Git.MaxGitDiffFiles, stdout) + if err != nil { + return nil, fmt.Errorf("ParsePatch: %v", err) + } + + if err = cmd.Wait(); err != nil { + return nil, fmt.Errorf("Wait: %v", err) + } + + return diff, nil +} + +// CheckAttribute checks the given attribute of the provided files +func (t *TemporaryUploadRepository) CheckAttribute(attribute string, args ...string) (map[string]map[string]string, error) { + stdOut := new(bytes.Buffer) + stdErr := new(bytes.Buffer) + + timeout := 5 * time.Minute + ctx, cancel := context.WithTimeout(context.Background(), timeout) + defer cancel() + + cmdArgs := []string{"check-attr", "-z", attribute, "--cached", "--"} + for _, arg := range args { + if arg != "" { + cmdArgs = append(cmdArgs, arg) + } + } + + cmd := exec.CommandContext(ctx, "git", cmdArgs...) + desc := fmt.Sprintf("checkAttr: (git check-attr) %s %v", attribute, cmdArgs) + cmd.Dir = t.basePath + cmd.Stdout = stdOut + cmd.Stderr = stdErr + + if err := cmd.Start(); err != nil { + return nil, fmt.Errorf("exec(%s) failed: %v(%v)", desc, err, ctx.Err()) + } + + pid := process.GetManager().Add(desc, cmd) + err := cmd.Wait() + process.GetManager().Remove(pid) + + if err != nil { + err = fmt.Errorf("exec(%d:%s) failed: %v(%v) stdout: %v stderr: %v", pid, desc, err, ctx.Err(), stdOut, stdErr) + return nil, err + } + + fields := bytes.Split(stdOut.Bytes(), []byte{'\000'}) + + if len(fields)%3 != 1 { + return nil, fmt.Errorf("Wrong number of fields in return from check-attr") + } + + var name2attribute2info = make(map[string]map[string]string) + + for i := 0; i < (len(fields) / 3); i++ { + filename := string(fields[3*i]) + attribute := string(fields[3*i+1]) + info := string(fields[3*i+2]) + attribute2info := name2attribute2info[filename] + if attribute2info == nil { + attribute2info = make(map[string]string) + } + attribute2info[attribute] = info + name2attribute2info[filename] = attribute2info + } + + return name2attribute2info, err +} diff --git a/modules/uploader/update.go b/modules/uploader/update.go new file mode 100644 index 000000000..08caf11ee --- /dev/null +++ b/modules/uploader/update.go @@ -0,0 +1,159 @@ +// Copyright 2019 The Gitea Authors. All rights reserved. +// Use of this source code is governed by a MIT-style +// license that can be found in the LICENSE file. + +package uploader + +import ( + "fmt" + "strings" + + "code.gitea.io/git" + "code.gitea.io/gitea/models" + "code.gitea.io/gitea/modules/lfs" + "code.gitea.io/gitea/modules/setting" +) + +// UpdateRepoFileOptions holds the repository file update options +type UpdateRepoFileOptions struct { + LastCommitID string + OldBranch string + NewBranch string + OldTreeName string + NewTreeName string + Message string + Content string + IsNewFile bool +} + +// UpdateRepoFile adds or updates a file in the given repository +func UpdateRepoFile(repo *models.Repository, doer *models.User, opts *UpdateRepoFileOptions) error { + t, err := NewTemporaryUploadRepository(repo) + defer t.Close() + if err != nil { + return err + } + if err := t.Clone(opts.OldBranch); err != nil { + return err + } + if err := t.SetDefaultIndex(); err != nil { + return err + } + + filesInIndex, err := t.LsFiles(opts.NewTreeName, opts.OldTreeName) + if err != nil { + return fmt.Errorf("UpdateRepoFile: %v", err) + } + + if opts.IsNewFile { + for _, file := range filesInIndex { + if file == opts.NewTreeName { + return models.ErrRepoFileAlreadyExist{FileName: opts.NewTreeName} + } + } + } + + //var stdout string + if opts.OldTreeName != opts.NewTreeName && len(filesInIndex) > 0 { + for _, file := range filesInIndex { + if file == opts.OldTreeName { + if err := t.RemoveFilesFromIndex(opts.OldTreeName); err != nil { + return err + } + } + } + + } + + // Check there is no way this can return multiple infos + filename2attribute2info, err := t.CheckAttribute("filter", opts.NewTreeName) + if err != nil { + return err + } + + content := opts.Content + var lfsMetaObject *models.LFSMetaObject + + if filename2attribute2info[opts.NewTreeName] != nil && filename2attribute2info[opts.NewTreeName]["filter"] == "lfs" { + // OK so we are supposed to LFS this data! + oid, err := models.GenerateLFSOid(strings.NewReader(opts.Content)) + if err != nil { + return err + } + lfsMetaObject = &models.LFSMetaObject{Oid: oid, Size: int64(len(opts.Content)), RepositoryID: repo.ID} + content = lfsMetaObject.Pointer() + } + + // Add the object to the database + objectHash, err := t.HashObject(strings.NewReader(content)) + if err != nil { + return err + } + + // Add the object to the index + if err := t.AddObjectToIndex("100644", objectHash, opts.NewTreeName); err != nil { + return err + } + + // Now write the tree + treeHash, err := t.WriteTree() + if err != nil { + return err + } + + // Now commit the tree + commitHash, err := t.CommitTree(doer, treeHash, opts.Message) + if err != nil { + return err + } + + if lfsMetaObject != nil { + // We have an LFS object - create it + lfsMetaObject, err = models.NewLFSMetaObject(lfsMetaObject) + if err != nil { + return err + } + contentStore := &lfs.ContentStore{BasePath: setting.LFS.ContentPath} + if !contentStore.Exists(lfsMetaObject) { + if err := contentStore.Put(lfsMetaObject, strings.NewReader(opts.Content)); err != nil { + if err2 := repo.RemoveLFSMetaObjectByOid(lfsMetaObject.Oid); err2 != nil { + return fmt.Errorf("Error whilst removing failed inserted LFS object %s: %v (Prev Error: %v)", lfsMetaObject.Oid, err2, err) + } + return err + } + } + } + + // Then push this tree to NewBranch + if err := t.Push(doer, commitHash, opts.NewBranch); err != nil { + return err + } + + // Simulate push event. + oldCommitID := opts.LastCommitID + if opts.NewBranch != opts.OldBranch { + oldCommitID = git.EmptySHA + } + + if err = repo.GetOwner(); err != nil { + return fmt.Errorf("GetOwner: %v", err) + } + err = models.PushUpdate( + opts.NewBranch, + models.PushUpdateOptions{ + PusherID: doer.ID, + PusherName: doer.Name, + RepoUserName: repo.Owner.Name, + RepoName: repo.Name, + RefFullName: git.BranchPrefix + opts.NewBranch, + OldCommitID: oldCommitID, + NewCommitID: commitHash, + }, + ) + if err != nil { + return fmt.Errorf("PushUpdate: %v", err) + } + models.UpdateRepoIndexer(repo) + + return nil +} diff --git a/modules/uploader/upload.go b/modules/uploader/upload.go new file mode 100644 index 000000000..bee3f1b9b --- /dev/null +++ b/modules/uploader/upload.go @@ -0,0 +1,206 @@ +// Copyright 2019 The Gitea Authors. All rights reserved. +// Use of this source code is governed by a MIT-style +// license that can be found in the LICENSE file. + +package uploader + +import ( + "fmt" + "os" + "path" + "strings" + + "code.gitea.io/gitea/modules/lfs" + "code.gitea.io/gitea/modules/setting" + + "code.gitea.io/git" + "code.gitea.io/gitea/models" +) + +// UploadRepoFileOptions contains the uploaded repository file options +type UploadRepoFileOptions struct { + LastCommitID string + OldBranch string + NewBranch string + TreePath string + Message string + Files []string // In UUID format. +} + +type uploadInfo struct { + upload *models.Upload + lfsMetaObject *models.LFSMetaObject +} + +func cleanUpAfterFailure(infos *[]uploadInfo, t *TemporaryUploadRepository, original error) error { + for _, info := range *infos { + if info.lfsMetaObject == nil { + continue + } + if !info.lfsMetaObject.Existing { + if err := t.repo.RemoveLFSMetaObjectByOid(info.lfsMetaObject.Oid); err != nil { + original = fmt.Errorf("%v, %v", original, err) + } + } + } + return original +} + +// UploadRepoFiles uploads files to the given repository +func UploadRepoFiles(repo *models.Repository, doer *models.User, opts *UploadRepoFileOptions) error { + if len(opts.Files) == 0 { + return nil + } + + uploads, err := models.GetUploadsByUUIDs(opts.Files) + if err != nil { + return fmt.Errorf("GetUploadsByUUIDs [uuids: %v]: %v", opts.Files, err) + } + + t, err := NewTemporaryUploadRepository(repo) + defer t.Close() + if err != nil { + return err + } + if err := t.Clone(opts.OldBranch); err != nil { + return err + } + if err := t.SetDefaultIndex(); err != nil { + return err + } + + names := make([]string, len(uploads)) + infos := make([]uploadInfo, len(uploads)) + for i, upload := range uploads { + names[i] = upload.Name + infos[i] = uploadInfo{upload: upload} + } + + filename2attribute2info, err := t.CheckAttribute("filter", names...) + if err != nil { + return err + } + + // Copy uploaded files into repository. + for i, uploadInfo := range infos { + file, err := os.Open(uploadInfo.upload.LocalPath()) + if err != nil { + return err + } + defer file.Close() + + var objectHash string + if filename2attribute2info[uploadInfo.upload.Name] != nil && filename2attribute2info[uploadInfo.upload.Name]["filter"] == "lfs" { + // Handle LFS + // FIXME: Inefficient! this should probably happen in models.Upload + oid, err := models.GenerateLFSOid(file) + if err != nil { + return err + } + fileInfo, err := file.Stat() + if err != nil { + return err + } + + uploadInfo.lfsMetaObject = &models.LFSMetaObject{Oid: oid, Size: fileInfo.Size(), RepositoryID: t.repo.ID} + + if objectHash, err = t.HashObject(strings.NewReader(uploadInfo.lfsMetaObject.Pointer())); err != nil { + return err + } + infos[i] = uploadInfo + + } else { + if objectHash, err = t.HashObject(file); err != nil { + return err + } + } + + // Add the object to the index + if err := t.AddObjectToIndex("100644", objectHash, path.Join(opts.TreePath, uploadInfo.upload.Name)); err != nil { + return err + + } + } + + // Now write the tree + treeHash, err := t.WriteTree() + if err != nil { + return err + } + + // Now commit the tree + commitHash, err := t.CommitTree(doer, treeHash, opts.Message) + if err != nil { + return err + } + + // Now deal with LFS objects + for _, uploadInfo := range infos { + if uploadInfo.lfsMetaObject == nil { + continue + } + uploadInfo.lfsMetaObject, err = models.NewLFSMetaObject(uploadInfo.lfsMetaObject) + if err != nil { + // OK Now we need to cleanup + return cleanUpAfterFailure(&infos, t, err) + } + // Don't move the files yet - we need to ensure that + // everything can be inserted first + } + + // OK now we can insert the data into the store - there's no way to clean up the store + // once it's in there, it's in there. + contentStore := &lfs.ContentStore{BasePath: setting.LFS.ContentPath} + for _, uploadInfo := range infos { + if uploadInfo.lfsMetaObject == nil { + continue + } + if !contentStore.Exists(uploadInfo.lfsMetaObject) { + file, err := os.Open(uploadInfo.upload.LocalPath()) + if err != nil { + return cleanUpAfterFailure(&infos, t, err) + } + defer file.Close() + // FIXME: Put regenerates the hash and copies the file over. + // I guess this strictly ensures the soundness of the store but this is inefficient. + if err := contentStore.Put(uploadInfo.lfsMetaObject, file); err != nil { + // OK Now we need to cleanup + // Can't clean up the store, once uploaded there they're there. + return cleanUpAfterFailure(&infos, t, err) + } + } + } + + // Then push this tree to NewBranch + if err := t.Push(doer, commitHash, opts.NewBranch); err != nil { + return err + } + + // Simulate push event. + oldCommitID := opts.LastCommitID + if opts.NewBranch != opts.OldBranch { + oldCommitID = git.EmptySHA + } + + if err = repo.GetOwner(); err != nil { + return fmt.Errorf("GetOwner: %v", err) + } + err = models.PushUpdate( + opts.NewBranch, + models.PushUpdateOptions{ + PusherID: doer.ID, + PusherName: doer.Name, + RepoUserName: repo.Owner.Name, + RepoName: repo.Name, + RefFullName: git.BranchPrefix + opts.NewBranch, + OldCommitID: oldCommitID, + NewCommitID: commitHash, + }, + ) + if err != nil { + return fmt.Errorf("PushUpdate: %v", err) + } + // FIXME: Should we models.UpdateRepoIndexer(repo) here? + + return models.DeleteUploads(uploads...) +} diff --git a/routers/repo/editor.go b/routers/repo/editor.go index 4e3557dbb..01963d8dc 100644 --- a/routers/repo/editor.go +++ b/routers/repo/editor.go @@ -19,6 +19,7 @@ import ( "code.gitea.io/gitea/modules/log" "code.gitea.io/gitea/modules/setting" "code.gitea.io/gitea/modules/templates" + "code.gitea.io/gitea/modules/uploader" ) const ( @@ -62,6 +63,16 @@ func editFile(ctx *context.Context, isNewFile bool) { ctx.Data["RequireSimpleMDE"] = true canCommit := renderCommitRights(ctx) + treePath := cleanUploadFileName(ctx.Repo.TreePath) + if treePath != ctx.Repo.TreePath { + if isNewFile { + ctx.Redirect(path.Join(ctx.Repo.RepoLink, "_new", ctx.Repo.BranchName, treePath)) + } else { + ctx.Redirect(path.Join(ctx.Repo.RepoLink, "_edit", ctx.Repo.BranchName, treePath)) + } + return + } + treeNames, treePaths := getParentTreeFields(ctx.Repo.TreePath) if !isNewFile { @@ -155,7 +166,7 @@ func editFilePost(ctx *context.Context, form auth.EditRepoFileForm, isNewFile bo oldBranchName := ctx.Repo.BranchName branchName := oldBranchName - oldTreePath := ctx.Repo.TreePath + oldTreePath := cleanUploadFileName(ctx.Repo.TreePath) lastCommit := form.LastCommit form.LastCommit = ctx.Repo.Commit.ID.String() @@ -298,7 +309,7 @@ func editFilePost(ctx *context.Context, form auth.EditRepoFileForm, isNewFile bo message += "\n\n" + form.CommitMessage } - if err := ctx.Repo.Repository.UpdateRepoFile(ctx.User, models.UpdateRepoFileOptions{ + if err := uploader.UpdateRepoFile(ctx.Repo.Repository, ctx.User, &uploader.UpdateRepoFileOptions{ LastCommitID: lastCommit, OldBranch: oldBranchName, NewBranch: branchName, @@ -328,7 +339,11 @@ func NewFilePost(ctx *context.Context, form auth.EditRepoFileForm) { // DiffPreviewPost render preview diff page func DiffPreviewPost(ctx *context.Context, form auth.EditPreviewDiffForm) { - treePath := ctx.Repo.TreePath + treePath := cleanUploadFileName(ctx.Repo.TreePath) + if len(treePath) == 0 { + ctx.Error(500, "file name to diff is invalid") + return + } entry, err := ctx.Repo.Commit.GetTreeEntryByPath(treePath) if err != nil { @@ -339,7 +354,7 @@ func DiffPreviewPost(ctx *context.Context, form auth.EditPreviewDiffForm) { return } - diff, err := ctx.Repo.Repository.GetDiffPreview(ctx.Repo.BranchName, treePath, form.Content) + diff, err := uploader.GetDiffPreview(ctx.Repo.Repository, ctx.Repo.BranchName, treePath, form.Content) if err != nil { ctx.Error(500, "GetDiffPreview: "+err.Error()) return @@ -358,7 +373,14 @@ func DiffPreviewPost(ctx *context.Context, form auth.EditPreviewDiffForm) { func DeleteFile(ctx *context.Context) { ctx.Data["PageIsDelete"] = true ctx.Data["BranchLink"] = ctx.Repo.RepoLink + "/src/" + ctx.Repo.BranchNameSubURL() - ctx.Data["TreePath"] = ctx.Repo.TreePath + treePath := cleanUploadFileName(ctx.Repo.TreePath) + + if treePath != ctx.Repo.TreePath { + ctx.Redirect(path.Join(ctx.Repo.RepoLink, "_delete", ctx.Repo.BranchName, treePath)) + return + } + + ctx.Data["TreePath"] = treePath canCommit := renderCommitRights(ctx) ctx.Data["commit_summary"] = "" @@ -426,7 +448,7 @@ func DeleteFilePost(ctx *context.Context, form auth.DeleteRepoFileForm) { message += "\n\n" + form.CommitMessage } - if err := ctx.Repo.Repository.DeleteRepoFile(ctx.User, models.DeleteRepoFileOptions{ + if err := uploader.DeleteRepoFile(ctx.Repo.Repository, ctx.User, &uploader.DeleteRepoFileOptions{ LastCommitID: ctx.Repo.CommitID, OldBranch: oldBranchName, NewBranch: branchName, @@ -453,6 +475,12 @@ func UploadFile(ctx *context.Context) { ctx.Data["PageIsUpload"] = true renderUploadSettings(ctx) canCommit := renderCommitRights(ctx) + treePath := cleanUploadFileName(ctx.Repo.TreePath) + if treePath != ctx.Repo.TreePath { + ctx.Redirect(path.Join(ctx.Repo.RepoLink, "_upload", ctx.Repo.BranchName, treePath)) + return + } + ctx.Repo.TreePath = treePath treeNames, treePaths := getParentTreeFields(ctx.Repo.TreePath) if len(treeNames) == 0 { @@ -489,10 +517,6 @@ func UploadFilePost(ctx *context.Context, form auth.UploadRepoFileForm) { } form.TreePath = cleanUploadFileName(form.TreePath) - if len(form.TreePath) == 0 { - ctx.Error(500, "Upload file name is invalid") - return - } treeNames, treePaths := getParentTreeFields(form.TreePath) if len(treeNames) == 0 { @@ -559,7 +583,7 @@ func UploadFilePost(ctx *context.Context, form auth.UploadRepoFileForm) { message += "\n\n" + form.CommitMessage } - if err := ctx.Repo.Repository.UploadRepoFiles(ctx.User, models.UploadRepoFileOptions{ + if err := uploader.UploadRepoFiles(ctx.Repo.Repository, ctx.User, &uploader.UploadRepoFileOptions{ LastCommitID: ctx.Repo.CommitID, OldBranch: oldBranchName, NewBranch: branchName, @@ -576,12 +600,13 @@ func UploadFilePost(ctx *context.Context, form auth.UploadRepoFileForm) { } func cleanUploadFileName(name string) string { - name = strings.TrimLeft(name, "./\\") - name = strings.Replace(name, "../", "", -1) - name = strings.Replace(name, "..\\", "", -1) - name = strings.TrimPrefix(path.Clean(name), ".git/") - if name == ".git" { - return "" + // Rebase the filename + name = strings.Trim(path.Clean("/"+name), " /") + // Git disallows any filenames to have a .git directory in them. + for _, part := range strings.Split(name, "/") { + if strings.ToLower(part) == ".git" { + return "" + } } return name } diff --git a/routers/repo/editor_test.go b/routers/repo/editor_test.go index e5b957020..b3d4314c2 100644 --- a/routers/repo/editor_test.go +++ b/routers/repo/editor_test.go @@ -15,16 +15,24 @@ func TestCleanUploadName(t *testing.T) { models.PrepareTestEnv(t) var kases = map[string]string{ - ".git/refs/master": "git/refs/master", - "/root/abc": "root/abc", - "./../../abc": "abc", - "a/../.git": "a/.git", - "a/../../../abc": "a/abc", - "../../../acd": "acd", - "../../.git/abc": "git/abc", - "..\\..\\.git/abc": "git/abc", + ".git/refs/master": "", + "/root/abc": "root/abc", + "./../../abc": "abc", + "a/../.git": "", + "a/../../../abc": "abc", + "../../../acd": "acd", + "../../.git/abc": "", + "..\\..\\.git/abc": "..\\..\\.git/abc", + "..\\../.git/abc": "", + "..\\../.git": "", + "abc/../def": "def", + ".drone.yml": ".drone.yml", + ".abc/def/.drone.yml": ".abc/def/.drone.yml", + "..drone.yml.": "..drone.yml.", + "..a.dotty...name...": "..a.dotty...name...", + "..a.dotty../.folder../.name...": "..a.dotty../.folder../.name...", } for k, v := range kases { - assert.EqualValues(t, v, cleanUploadFileName(k)) + assert.EqualValues(t, cleanUploadFileName(k), v) } }