From 15a5c10d33fd3c0b9483316b19712e981bf2391d Mon Sep 17 00:00:00 2001 From: John Olheiser <42128690+jolheiser@users.noreply.github.com> Date: Sat, 30 Nov 2019 00:54:47 -0600 Subject: [PATCH] Variable expansion in repository templates (#9163) * Start expansion Signed-off-by: jolheiser * _template rather than .template Signed-off-by: jolheiser * Use ioutil Signed-off-by: jolheiser * Add descriptions to mapping * Start globbing Signed-off-by: jolheiser * Tune globbing Signed-off-by: jolheiser * Re-arrange imports Signed-off-by: jolheiser * Don't expand git hooks Signed-off-by: jolheiser * Add glob tests for .giteatemplate Signed-off-by: jolheiser * Parse globs separately so they can be tested more easily Signed-off-by: jolheiser * Change template location and add docs Signed-off-by: jolheiser * nit Signed-off-by: jolheiser * Update docs/content/doc/features/gitea-directory.md Co-Authored-By: guillep2k <18600385+guillep2k@users.noreply.github.com> * Update docs/content/doc/features/gitea-directory.md Co-Authored-By: guillep2k <18600385+guillep2k@users.noreply.github.com> * Add upper-lower case match Signed-off-by: jolheiser * Nits Signed-off-by: jolheiser * Update models/repo_generate.go Co-Authored-By: guillep2k <18600385+guillep2k@users.noreply.github.com> --- docs/content/doc/features/gitea-directory.md | 56 ++++++ models/repo.go | 48 ----- models/repo_generate.go | 184 ++++++++++++++++++- models/repo_generate_test.go | 57 ++++++ modules/git/repo.go | 4 + 5 files changed, 298 insertions(+), 51 deletions(-) create mode 100644 docs/content/doc/features/gitea-directory.md create mode 100644 models/repo_generate_test.go diff --git a/docs/content/doc/features/gitea-directory.md b/docs/content/doc/features/gitea-directory.md new file mode 100644 index 0000000000..e598969bcd --- /dev/null +++ b/docs/content/doc/features/gitea-directory.md @@ -0,0 +1,56 @@ +--- +date: "2019-11-28:00:00+02:00" +title: "The .gitea Directory" +slug: "gitea-directory" +weight: 40 +toc: true +draft: false +menu: + sidebar: + parent: "features" + name: "The .gitea Directory" + weight: 50 + identifier: "gitea-directory" +--- + +# The .gitea directory +Gitea repositories can include a `.gitea` directory at their base which will store settings/configurations for certain features. + +## Templates +Gitea includes template repositories, and one feature implemented with them is auto-expansion of specific variables within your template files. +To tell Gitea which files to expand, you must include a `template` file inside the `.gitea` directory of the template repository. +Gitea uses [gobwas/glob](https://github.com/gobwas/glob) for its glob syntax. It closely resembles a traditional `.gitignore`, however there may be slight differences. + +### Example `.gitea/template` file +All paths are relative to the base of the repository +```gitignore +# All .go files, anywhere in the repository +**.go + +# All text files in the text directory +text/*.txt + +# A specific file +a/b/c/d.json + +# Batch files in both upper or lower case can be matched +**.[bB][aA][tT] +``` +**NOTE:** The `template` file will be removed from the `.gitea` directory when a repository is generated from the template. + +### Variable Expansion +In any file matched by the above globs, certain variables will be expanded. +All variables must be of the form `$VAR` or `${VAR}`. To escape an expansion, use a double `$$`, such as `$$VAR` or `$${VAR}` + +| Variable | Expands To | +|----------------------|-----------------------------------------------------| +| REPO_NAME | The name of the generated repository | +| TEMPLATE_NAME | The name of the template repository | +| REPO_DESCRIPTION | The description of the generated repository | +| TEMPLATE_DESCRIPTION | The description of the template repository | +| REPO_LINK | The URL to the generated repository | +| TEMPLATE_LINK | The URL to the template repository | +| REPO_HTTPS_URL | The HTTP(S) clone link for the generated repository | +| TEMPLATE_HTTPS_URL | The HTTP(S) clone link for the template repository | +| REPO_SSH_URL | The SSH clone link for the generated repository | +| TEMPLATE_SSH_URL | The SSH clone link for the template repository | diff --git a/models/repo.go b/models/repo.go index cbe1ccc4f6..0ccf786db3 100644 --- a/models/repo.go +++ b/models/repo.go @@ -1361,54 +1361,6 @@ func prepareRepoCommit(e Engine, repo *Repository, tmpDir, repoPath string, opts return nil } -func generateRepoCommit(e Engine, repo, templateRepo *Repository, tmpDir string) error { - commitTimeStr := time.Now().Format(time.RFC3339) - authorSig := repo.Owner.NewGitSig() - - // Because this may call hooks we should pass in the environment - env := append(os.Environ(), - "GIT_AUTHOR_NAME="+authorSig.Name, - "GIT_AUTHOR_EMAIL="+authorSig.Email, - "GIT_AUTHOR_DATE="+commitTimeStr, - "GIT_COMMITTER_NAME="+authorSig.Name, - "GIT_COMMITTER_EMAIL="+authorSig.Email, - "GIT_COMMITTER_DATE="+commitTimeStr, - ) - - // Clone to temporary path and do the init commit. - templateRepoPath := templateRepo.repoPath(e) - _, stderr, err := process.GetManager().ExecDirEnv( - -1, "", - fmt.Sprintf("generateRepoCommit(git clone): %s", templateRepoPath), - env, - git.GitExecutable, "clone", "--depth", "1", templateRepoPath, tmpDir, - ) - if err != nil { - return fmt.Errorf("git clone: %v - %s", err, stderr) - } - - if err := os.RemoveAll(path.Join(tmpDir, ".git")); err != nil { - return fmt.Errorf("remove git dir: %v", err) - } - - if err := git.InitRepository(tmpDir, false); err != nil { - return err - } - - repoPath := repo.repoPath(e) - _, stderr, err = process.GetManager().ExecDirEnv( - -1, tmpDir, - fmt.Sprintf("generateRepoCommit(git remote add): %s", repoPath), - env, - git.GitExecutable, "remote", "add", "origin", repoPath, - ) - if err != nil { - return fmt.Errorf("git remote add: %v - %s", err, stderr) - } - - return initRepoCommit(tmpDir, repo.Owner) -} - func checkInitRepository(repoPath string) (err error) { // Somehow the directory could exist. if com.IsExist(repoPath) { diff --git a/models/repo_generate.go b/models/repo_generate.go index 234bdc27f2..56a3940ac1 100644 --- a/models/repo_generate.go +++ b/models/repo_generate.go @@ -6,7 +6,9 @@ package models import ( "fmt" + "io/ioutil" "os" + "path" "path/filepath" "strconv" "strings" @@ -14,7 +16,10 @@ import ( "code.gitea.io/gitea/modules/git" "code.gitea.io/gitea/modules/log" + "code.gitea.io/gitea/modules/process" + "code.gitea.io/gitea/modules/util" + "github.com/gobwas/glob" "github.com/unknwon/com" ) @@ -36,8 +41,148 @@ func (gro GenerateRepoOptions) IsValid() bool { return gro.GitContent || gro.Topics || gro.GitHooks || gro.Webhooks || gro.Avatar || gro.IssueLabels // or other items as they are added } +// GiteaTemplate holds information about a .gitea/template file +type GiteaTemplate struct { + Path string + Content []byte + + globs []glob.Glob +} + +// Globs parses the .gitea/template globs or returns them if they were already parsed +func (gt GiteaTemplate) Globs() []glob.Glob { + if gt.globs != nil { + return gt.globs + } + + gt.globs = make([]glob.Glob, 0) + lines := strings.Split(string(util.NormalizeEOL(gt.Content)), "\n") + for _, line := range lines { + line = strings.TrimSpace(line) + if line == "" || strings.HasPrefix(line, "#") { + continue + } + g, err := glob.Compile(line, '/') + if err != nil { + log.Info("Invalid glob expression '%s' (skipped): %v", line, err) + continue + } + gt.globs = append(gt.globs, g) + } + return gt.globs +} + +func checkGiteaTemplate(tmpDir string) (*GiteaTemplate, error) { + gtPath := filepath.Join(tmpDir, ".gitea", "template") + if _, err := os.Stat(gtPath); os.IsNotExist(err) { + return nil, nil + } else if err != nil { + return nil, err + } + + content, err := ioutil.ReadFile(gtPath) + if err != nil { + return nil, err + } + + gt := &GiteaTemplate{ + Path: gtPath, + Content: content, + } + + return gt, nil +} + +func generateRepoCommit(e Engine, repo, templateRepo, generateRepo *Repository, tmpDir string) error { + commitTimeStr := time.Now().Format(time.RFC3339) + authorSig := repo.Owner.NewGitSig() + + // Because this may call hooks we should pass in the environment + env := append(os.Environ(), + "GIT_AUTHOR_NAME="+authorSig.Name, + "GIT_AUTHOR_EMAIL="+authorSig.Email, + "GIT_AUTHOR_DATE="+commitTimeStr, + "GIT_COMMITTER_NAME="+authorSig.Name, + "GIT_COMMITTER_EMAIL="+authorSig.Email, + "GIT_COMMITTER_DATE="+commitTimeStr, + ) + + // Clone to temporary path and do the init commit. + templateRepoPath := templateRepo.repoPath(e) + if err := git.Clone(templateRepoPath, tmpDir, git.CloneRepoOptions{ + Depth: 1, + }); err != nil { + return fmt.Errorf("git clone: %v", err) + } + + if err := os.RemoveAll(path.Join(tmpDir, ".git")); err != nil { + return fmt.Errorf("remove git dir: %v", err) + } + + // Variable expansion + gt, err := checkGiteaTemplate(tmpDir) + if err != nil { + return fmt.Errorf("checkGiteaTemplate: %v", err) + } + + if err := os.Remove(gt.Path); err != nil { + return fmt.Errorf("remove .giteatemplate: %v", err) + } + + // Avoid walking tree if there are no globs + if len(gt.Globs()) > 0 { + tmpDirSlash := strings.TrimSuffix(filepath.ToSlash(tmpDir), "/") + "/" + if err := filepath.Walk(tmpDirSlash, func(path string, info os.FileInfo, walkErr error) error { + if walkErr != nil { + return walkErr + } + + if info.IsDir() { + return nil + } + + base := strings.TrimPrefix(filepath.ToSlash(path), tmpDirSlash) + for _, g := range gt.Globs() { + if g.Match(base) { + content, err := ioutil.ReadFile(path) + if err != nil { + return err + } + + if err := ioutil.WriteFile(path, + []byte(generateExpansion(string(content), templateRepo, generateRepo)), + 0644); err != nil { + return err + } + break + } + } + return nil + }); err != nil { + return err + } + } + + if err := git.InitRepository(tmpDir, false); err != nil { + return err + } + + repoPath := repo.repoPath(e) + _, stderr, err := process.GetManager().ExecDirEnv( + -1, tmpDir, + fmt.Sprintf("generateRepoCommit(git remote add): %s", repoPath), + env, + git.GitExecutable, "remote", "add", "origin", repoPath, + ) + if err != nil { + return fmt.Errorf("git remote add: %v - %s", err, stderr) + } + + return initRepoCommit(tmpDir, repo.Owner) +} + // generateRepository initializes repository from template -func generateRepository(e Engine, repo, templateRepo *Repository) (err error) { +func generateRepository(e Engine, repo, templateRepo, generateRepo *Repository) (err error) { tmpDir := filepath.Join(os.TempDir(), "gitea-"+repo.Name+"-"+com.ToStr(time.Now().Nanosecond())) if err := os.MkdirAll(tmpDir, os.ModePerm); err != nil { @@ -50,7 +195,7 @@ func generateRepository(e Engine, repo, templateRepo *Repository) (err error) { } }() - if err = generateRepoCommit(e, repo, templateRepo, tmpDir); err != nil { + if err = generateRepoCommit(e, repo, templateRepo, generateRepo, tmpDir); err != nil { return fmt.Errorf("generateRepoCommit: %v", err) } @@ -95,7 +240,7 @@ func GenerateRepository(ctx DBContext, doer, owner *User, templateRepo *Reposito // GenerateGitContent generates git content from a template repository func GenerateGitContent(ctx DBContext, templateRepo, generateRepo *Repository) error { - if err := generateRepository(ctx.e, generateRepo, templateRepo); err != nil { + if err := generateRepository(ctx.e, generateRepo, templateRepo, generateRepo); err != nil { return err } @@ -210,3 +355,36 @@ func GenerateIssueLabels(ctx DBContext, templateRepo, generateRepo *Repository) } return nil } + +func generateExpansion(src string, templateRepo, generateRepo *Repository) string { + return os.Expand(src, func(key string) string { + switch key { + case "REPO_NAME": + return generateRepo.Name + case "TEMPLATE_NAME": + return templateRepo.Name + case "REPO_DESCRIPTION": + return generateRepo.Description + case "TEMPLATE_DESCRIPTION": + return templateRepo.Description + case "REPO_OWNER": + return generateRepo.MustOwnerName() + case "TEMPLATE_OWNER": + return templateRepo.MustOwnerName() + case "REPO_LINK": + return generateRepo.Link() + case "TEMPLATE_LINK": + return templateRepo.Link() + case "REPO_HTTPS_URL": + return generateRepo.CloneLink().HTTPS + case "TEMPLATE_HTTPS_URL": + return templateRepo.CloneLink().HTTPS + case "REPO_SSH_URL": + return generateRepo.CloneLink().SSH + case "TEMPLATE_SSH_URL": + return templateRepo.CloneLink().SSH + default: + return key + } + }) +} diff --git a/models/repo_generate_test.go b/models/repo_generate_test.go new file mode 100644 index 0000000000..53ab4fcd3d --- /dev/null +++ b/models/repo_generate_test.go @@ -0,0 +1,57 @@ +// Copyright 2019 The Gitea Authors. All rights reserved. +// Use of this source code is governed by a MIT-style +// license that can be found in the LICENSE file. + +package models + +import ( + "testing" + + "github.com/stretchr/testify/assert" +) + +var giteaTemplate = []byte(` +# Header + +# All .go files +**.go + +# All text files in /text/ +text/*.txt + +# All files in modules folders +**/modules/* +`) + +func TestGiteaTemplate(t *testing.T) { + gt := GiteaTemplate{Content: giteaTemplate} + assert.Equal(t, len(gt.Globs()), 3) + + tt := []struct { + Path string + Match bool + }{ + {Path: "main.go", Match: true}, + {Path: "a/b/c/d/e.go", Match: true}, + {Path: "main.txt", Match: false}, + {Path: "a/b.txt", Match: false}, + {Path: "text/a.txt", Match: true}, + {Path: "text/b.txt", Match: true}, + {Path: "text/c.json", Match: false}, + {Path: "a/b/c/modules/README.md", Match: true}, + {Path: "a/b/c/modules/d/README.md", Match: false}, + } + + for _, tc := range tt { + t.Run(tc.Path, func(t *testing.T) { + match := false + for _, g := range gt.Globs() { + if g.Match(tc.Path) { + match = true + break + } + } + assert.Equal(t, tc.Match, match) + }) + } +} diff --git a/modules/git/repo.go b/modules/git/repo.go index e277f896bf..03296d56ab 100644 --- a/modules/git/repo.go +++ b/modules/git/repo.go @@ -161,6 +161,7 @@ type CloneRepoOptions struct { Branch string Shared bool NoCheckout bool + Depth int } // Clone clones original repository to target path. @@ -193,6 +194,9 @@ func CloneWithArgs(from, to string, args []string, opts CloneRepoOptions) (err e if opts.NoCheckout { cmd.AddArguments("--no-checkout") } + if opts.Depth > 0 { + cmd.AddArguments("--depth", strconv.Itoa(opts.Depth)) + } if len(opts.Branch) > 0 { cmd.AddArguments("-b", opts.Branch)