Nulo/forgejo
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity

Variable expansion in repository templates (#9163)

Browse source 
* Start expansion

Signed-off-by: jolheiser <john.olheiser@gmail.com>

* _template rather than .template

Signed-off-by: jolheiser <john.olheiser@gmail.com>

* Use ioutil

Signed-off-by: jolheiser <john.olheiser@gmail.com>

* Add descriptions to mapping

* Start globbing

Signed-off-by: jolheiser <john.olheiser@gmail.com>

* Tune globbing

Signed-off-by: jolheiser <john.olheiser@gmail.com>

* Re-arrange imports

Signed-off-by: jolheiser <john.olheiser@gmail.com>

* Don't expand git hooks

Signed-off-by: jolheiser <john.olheiser@gmail.com>

* Add glob tests for .giteatemplate

Signed-off-by: jolheiser <john.olheiser@gmail.com>

* Parse globs separately so they can be tested more easily

Signed-off-by: jolheiser <john.olheiser@gmail.com>

* Change template location and add docs

Signed-off-by: jolheiser <john.olheiser@gmail.com>

* nit

Signed-off-by: jolheiser <john.olheiser@gmail.com>

* Update docs/content/doc/features/gitea-directory.md

Co-Authored-By: guillep2k <18600385+guillep2k@users.noreply.github.com>

* Update docs/content/doc/features/gitea-directory.md

Co-Authored-By: guillep2k <18600385+guillep2k@users.noreply.github.com>

* Add upper-lower case match

Signed-off-by: jolheiser <john.olheiser@gmail.com>

* Nits

Signed-off-by: jolheiser <john.olheiser@gmail.com>

* Update models/repo_generate.go

Co-Authored-By: guillep2k <18600385+guillep2k@users.noreply.github.com>
This commit is contained in:
John Olheiser 2019-11-30 00:54:47 -06:00 committed by techknowlogick
parent c9d50bcab5
commit 15a5c10d33
5 changed files with 298 additions and 51 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

56
docs/content/doc/features/gitea-directory.md Normal file
View file

@ -0,0 +1,56 @@
---
date: "2019-11-28:00:00+02:00"
title: "The .gitea Directory"
slug: "gitea-directory"
weight: 40
toc: true
draft: false
menu:
sidebar:
parent: "features"
name: "The .gitea Directory"
weight: 50
identifier: "gitea-directory"
---
# The .gitea directory
Gitea repositories can include a `.gitea` directory at their base which will store settings/configurations for certain features.
## Templates
Gitea includes template repositories, and one feature implemented with them is auto-expansion of specific variables within your template files.
To tell Gitea which files to expand, you must include a `template` file inside the `.gitea` directory of the template repository.
Gitea uses [gobwas/glob](https://github.com/gobwas/glob) for its glob syntax. It closely resembles a traditional `.gitignore`, however there may be slight differences.
### Example `.gitea/template` file
All paths are relative to the base of the repository
```gitignore
# All .go files, anywhere in the repository
**.go
# All text files in the text directory
text/*.txt
# A specific file
a/b/c/d.json
# Batch files in both upper or lower case can be matched
**.[bB][aA][tT]
```
**NOTE:** The `template` file will be removed from the `.gitea` directory when a repository is generated from the template.
### Variable Expansion
In any file matched by the above globs, certain variables will be expanded.
All variables must be of the form `$VAR` or `${VAR}`. To escape an expansion, use a double `$$`, such as `$$VAR` or `$${VAR}`
| Variable | Expands To |
|----------------------|-----------------------------------------------------|
| REPO_NAME | The name of the generated repository |
| TEMPLATE_NAME | The name of the template repository |
| REPO_DESCRIPTION | The description of the generated repository |
| TEMPLATE_DESCRIPTION | The description of the template repository |
| REPO_LINK | The URL to the generated repository |
| TEMPLATE_LINK | The URL to the template repository |
| REPO_HTTPS_URL | The HTTP(S) clone link for the generated repository |
| TEMPLATE_HTTPS_URL | The HTTP(S) clone link for the template repository |
| REPO_SSH_URL | The SSH clone link for the generated repository |
| TEMPLATE_SSH_URL | The SSH clone link for the template repository |

48
models/repo.go
View file

@ -1361,54 +1361,6 @@ func prepareRepoCommit(e Engine, repo *Repository, tmpDir, repoPath string, opts
return nil
}
func generateRepoCommit(e Engine, repo, templateRepo *Repository, tmpDir string) error {
commitTimeStr := time.Now().Format(time.RFC3339)
authorSig := repo.Owner.NewGitSig()
// Because this may call hooks we should pass in the environment
env := append(os.Environ(),
"GIT_AUTHOR_NAME="+authorSig.Name,
"GIT_AUTHOR_EMAIL="+authorSig.Email,
"GIT_AUTHOR_DATE="+commitTimeStr,
"GIT_COMMITTER_NAME="+authorSig.Name,
"GIT_COMMITTER_EMAIL="+authorSig.Email,
"GIT_COMMITTER_DATE="+commitTimeStr,
)
// Clone to temporary path and do the init commit.
templateRepoPath := templateRepo.repoPath(e)
_, stderr, err := process.GetManager().ExecDirEnv(
-1, "",
fmt.Sprintf("generateRepoCommit(git clone): %s", templateRepoPath),
env,
git.GitExecutable, "clone", "--depth", "1", templateRepoPath, tmpDir,
)
if err != nil {
return fmt.Errorf("git clone: %v - %s", err, stderr)
}
if err := os.RemoveAll(path.Join(tmpDir, ".git")); err != nil {
return fmt.Errorf("remove git dir: %v", err)
}
if err := git.InitRepository(tmpDir, false); err != nil {
return err
}
repoPath := repo.repoPath(e)
_, stderr, err = process.GetManager().ExecDirEnv(
-1, tmpDir,
fmt.Sprintf("generateRepoCommit(git remote add): %s", repoPath),
env,
git.GitExecutable, "remote", "add", "origin", repoPath,
)
if err != nil {
return fmt.Errorf("git remote add: %v - %s", err, stderr)
}
return initRepoCommit(tmpDir, repo.Owner)
}
func checkInitRepository(repoPath string) (err error) {
// Somehow the directory could exist.
if com.IsExist(repoPath) {

184
models/repo_generate.go
View file

@ -6,7 +6,9 @@ package models
import (
"fmt"
"io/ioutil"
"os"
"path"
"path/filepath"
"strconv"
"strings"
@ -14,7 +16,10 @@ import (
"code.gitea.io/gitea/modules/git"
"code.gitea.io/gitea/modules/log"
"code.gitea.io/gitea/modules/process"
"code.gitea.io/gitea/modules/util"
"github.com/gobwas/glob"
"github.com/unknwon/com"
)
@ -36,8 +41,148 @@ func (gro GenerateRepoOptions) IsValid() bool {
return gro.GitContent || gro.Topics || gro.GitHooks || gro.Webhooks || gro.Avatar || gro.IssueLabels // or other items as they are added
}
// GiteaTemplate holds information about a .gitea/template file
type GiteaTemplate struct {
Path string
Content []byte
globs []glob.Glob
}
// Globs parses the .gitea/template globs or returns them if they were already parsed
func (gt GiteaTemplate) Globs() []glob.Glob {
if gt.globs != nil {
return gt.globs
}
gt.globs = make([]glob.Glob, 0)
lines := strings.Split(string(util.NormalizeEOL(gt.Content)), "\n")
for _, line := range lines {
line = strings.TrimSpace(line)
if line == "" || strings.HasPrefix(line, "#") {
continue
}
g, err := glob.Compile(line, '/')
if err != nil {
log.Info("Invalid glob expression '%s' (skipped): %v", line, err)
continue
}
gt.globs = append(gt.globs, g)
}
return gt.globs
}
func checkGiteaTemplate(tmpDir string) (*GiteaTemplate, error) {
gtPath := filepath.Join(tmpDir, ".gitea", "template")
if _, err := os.Stat(gtPath); os.IsNotExist(err) {
return nil, nil
} else if err != nil {
return nil, err
}
content, err := ioutil.ReadFile(gtPath)
if err != nil {
return nil, err
}
gt := &GiteaTemplate{
Path: gtPath,
Content: content,
}
return gt, nil
}
func generateRepoCommit(e Engine, repo, templateRepo, generateRepo *Repository, tmpDir string) error {
commitTimeStr := time.Now().Format(time.RFC3339)
authorSig := repo.Owner.NewGitSig()
// Because this may call hooks we should pass in the environment
env := append(os.Environ(),
"GIT_AUTHOR_NAME="+authorSig.Name,
"GIT_AUTHOR_EMAIL="+authorSig.Email,
"GIT_AUTHOR_DATE="+commitTimeStr,
"GIT_COMMITTER_NAME="+authorSig.Name,
"GIT_COMMITTER_EMAIL="+authorSig.Email,
"GIT_COMMITTER_DATE="+commitTimeStr,
)
// Clone to temporary path and do the init commit.
templateRepoPath := templateRepo.repoPath(e)
if err := git.Clone(templateRepoPath, tmpDir, git.CloneRepoOptions{
Depth: 1,
}); err != nil {
return fmt.Errorf("git clone: %v", err)
}
if err := os.RemoveAll(path.Join(tmpDir, ".git")); err != nil {
return fmt.Errorf("remove git dir: %v", err)
}
// Variable expansion
gt, err := checkGiteaTemplate(tmpDir)
if err != nil {
return fmt.Errorf("checkGiteaTemplate: %v", err)
}
if err := os.Remove(gt.Path); err != nil {
return fmt.Errorf("remove .giteatemplate: %v", err)
}
// Avoid walking tree if there are no globs
if len(gt.Globs()) > 0 {
tmpDirSlash := strings.TrimSuffix(filepath.ToSlash(tmpDir), "/") + "/"
if err := filepath.Walk(tmpDirSlash, func(path string, info os.FileInfo, walkErr error) error {
if walkErr != nil {
return walkErr
}
if info.IsDir() {
return nil
}
base := strings.TrimPrefix(filepath.ToSlash(path), tmpDirSlash)
for _, g := range gt.Globs() {
if g.Match(base) {
content, err := ioutil.ReadFile(path)
if err != nil {
return err
}
if err := ioutil.WriteFile(path,
[]byte(generateExpansion(string(content), templateRepo, generateRepo)),
0644); err != nil {
return err
}
break
}
}
return nil
}); err != nil {
return err
}
}
if err := git.InitRepository(tmpDir, false); err != nil {
return err
}
repoPath := repo.repoPath(e)
_, stderr, err := process.GetManager().ExecDirEnv(
-1, tmpDir,
fmt.Sprintf("generateRepoCommit(git remote add): %s", repoPath),
env,
git.GitExecutable, "remote", "add", "origin", repoPath,
)
if err != nil {
return fmt.Errorf("git remote add: %v - %s", err, stderr)
}
return initRepoCommit(tmpDir, repo.Owner)
}
// generateRepository initializes repository from template
func generateRepository(e Engine, repo, templateRepo *Repository) (err error) {
func generateRepository(e Engine, repo, templateRepo, generateRepo *Repository) (err error) {
tmpDir := filepath.Join(os.TempDir(), "gitea-"+repo.Name+"-"+com.ToStr(time.Now().Nanosecond()))
if err := os.MkdirAll(tmpDir, os.ModePerm); err != nil {
@ -50,7 +195,7 @@ func generateRepository(e Engine, repo, templateRepo *Repository) (err error) {
}
}()
if err = generateRepoCommit(e, repo, templateRepo, tmpDir); err != nil {
if err = generateRepoCommit(e, repo, templateRepo, generateRepo, tmpDir); err != nil {
return fmt.Errorf("generateRepoCommit: %v", err)
}
@ -95,7 +240,7 @@ func GenerateRepository(ctx DBContext, doer, owner *User, templateRepo *Reposito
// GenerateGitContent generates git content from a template repository
func GenerateGitContent(ctx DBContext, templateRepo, generateRepo *Repository) error {
if err := generateRepository(ctx.e, generateRepo, templateRepo); err != nil {
if err := generateRepository(ctx.e, generateRepo, templateRepo, generateRepo); err != nil {
return err
}
@ -210,3 +355,36 @@ func GenerateIssueLabels(ctx DBContext, templateRepo, generateRepo *Repository)
}
return nil
}
func generateExpansion(src string, templateRepo, generateRepo *Repository) string {
return os.Expand(src, func(key string) string {
switch key {
case "REPO_NAME":
return generateRepo.Name
case "TEMPLATE_NAME":
return templateRepo.Name
case "REPO_DESCRIPTION":
return generateRepo.Description
case "TEMPLATE_DESCRIPTION":
return templateRepo.Description
case "REPO_OWNER":
return generateRepo.MustOwnerName()
case "TEMPLATE_OWNER":
return templateRepo.MustOwnerName()
case "REPO_LINK":
return generateRepo.Link()
case "TEMPLATE_LINK":
return templateRepo.Link()
case "REPO_HTTPS_URL":
return generateRepo.CloneLink().HTTPS
case "TEMPLATE_HTTPS_URL":
return templateRepo.CloneLink().HTTPS
case "REPO_SSH_URL":
return generateRepo.CloneLink().SSH
case "TEMPLATE_SSH_URL":
return templateRepo.CloneLink().SSH
default:
return key
}
})
}

57
models/repo_generate_test.go Normal file
View file

@ -0,0 +1,57 @@
// Copyright 2019 The Gitea Authors. All rights reserved.
// Use of this source code is governed by a MIT-style
// license that can be found in the LICENSE file.
package models
import (
"testing"
"github.com/stretchr/testify/assert"
)
var giteaTemplate = []byte(`
# Header
# All .go files
**.go
# All text files in /text/
text/*.txt
# All files in modules folders
**/modules/*
`)
func TestGiteaTemplate(t *testing.T) {
gt := GiteaTemplate{Content: giteaTemplate}
assert.Equal(t, len(gt.Globs()), 3)
tt := []struct {
Path string
Match bool
}{
{Path: "main.go", Match: true},
{Path: "a/b/c/d/e.go", Match: true},
{Path: "main.txt", Match: false},
{Path: "a/b.txt", Match: false},
{Path: "text/a.txt", Match: true},
{Path: "text/b.txt", Match: true},
{Path: "text/c.json", Match: false},
{Path: "a/b/c/modules/README.md", Match: true},
{Path: "a/b/c/modules/d/README.md", Match: false},
}
for _, tc := range tt {
t.Run(tc.Path, func(t *testing.T) {
match := false
for _, g := range gt.Globs() {
if g.Match(tc.Path) {
match = true
break
}
}
assert.Equal(t, tc.Match, match)
})
}
}

4
modules/git/repo.go
View file

@ -161,6 +161,7 @@ type CloneRepoOptions struct {
Branch string
Shared bool
NoCheckout bool
Depth int
}
// Clone clones original repository to target path.
@ -193,6 +194,9 @@ func CloneWithArgs(from, to string, args []string, opts CloneRepoOptions) (err e
if opts.NoCheckout {
cmd.AddArguments("--no-checkout")
}
if opts.Depth > 0 {
cmd.AddArguments("--depth", strconv.Itoa(opts.Depth))
}
if len(opts.Branch) > 0 {
cmd.AddArguments("-b", opts.Branch)