diff --git a/modules/markup/csv/csv.go b/modules/markup/csv/csv.go index 077947e77..1e3acc9b4 100644 --- a/modules/markup/csv/csv.go +++ b/modules/markup/csv/csv.go @@ -9,12 +9,18 @@ import ( "encoding/csv" "html" "io" + "regexp" + "strings" "code.gitea.io/gitea/modules/markup" + "code.gitea.io/gitea/modules/util" ) +var quoteRegexp = regexp.MustCompile(`["'][\s\S]+?["']`) + func init() { markup.RegisterParser(Parser{}) + } // Parser implements markup.Parser for orgmode @@ -28,12 +34,13 @@ func (Parser) Name() string { // Extensions implements markup.Parser func (Parser) Extensions() []string { - return []string{".csv"} + return []string{".csv", ".tsv"} } // Render implements markup.Parser -func (Parser) Render(rawBytes []byte, urlPrefix string, metas map[string]string, isWiki bool) []byte { +func (p Parser) Render(rawBytes []byte, urlPrefix string, metas map[string]string, isWiki bool) []byte { rd := csv.NewReader(bytes.NewReader(rawBytes)) + rd.Comma = p.bestDelimiter(rawBytes) var tmpBlock bytes.Buffer tmpBlock.WriteString(``) for { @@ -50,9 +57,57 @@ func (Parser) Render(rawBytes []byte, urlPrefix string, metas map[string]string, tmpBlock.WriteString(html.EscapeString(field)) tmpBlock.WriteString("") } - tmpBlock.WriteString("") + tmpBlock.WriteString("") } tmpBlock.WriteString("
") return tmpBlock.Bytes() } + +// bestDelimiter scores the input CSV data against delimiters, and returns the best match. +// Reads at most 10k bytes & 10 lines. +func (p Parser) bestDelimiter(data []byte) rune { + maxLines := 10 + maxBytes := util.Min(len(data), 1e4) + text := string(data[:maxBytes]) + text = quoteRegexp.ReplaceAllLiteralString(text, "") + lines := strings.SplitN(text, "\n", maxLines+1) + lines = lines[:util.Min(maxLines, len(lines))] + + delimiters := []rune{',', ';', '\t', '|'} + bestDelim := delimiters[0] + bestScore := 0.0 + for _, delim := range delimiters { + score := p.scoreDelimiter(lines, delim) + if score > bestScore { + bestScore = score + bestDelim = delim + } + } + + return bestDelim +} + +// scoreDelimiter uses a count & regularity metric to evaluate a delimiter against lines of CSV +func (Parser) scoreDelimiter(lines []string, delim rune) (score float64) { + countTotal := 0 + countLineMax := 0 + linesNotEqual := 0 + + for _, line := range lines { + if len(line) == 0 { + continue + } + + countLine := strings.Count(line, string(delim)) + countTotal += countLine + if countLine != countLineMax { + if countLineMax != 0 { + linesNotEqual++ + } + countLineMax = util.Max(countLine, countLineMax) + } + } + + return float64(countTotal) * (1 - float64(linesNotEqual)/float64(len(lines))) +} diff --git a/modules/markup/csv/csv_test.go b/modules/markup/csv/csv_test.go index f050296ce..4d4e0871e 100644 --- a/modules/markup/csv/csv_test.go +++ b/modules/markup/csv/csv_test.go @@ -13,9 +13,14 @@ import ( func TestRenderCSV(t *testing.T) { var parser Parser var kases = map[string]string{ - "a": "
a
", - "1,2": "
12
", - "
": "
<br/>
", + "a": "
a
", + "1,2": "
12
", + "1;2": "
12
", + "1\t2": "
12
", + "1|2": "
12
", + "1,2,3;4,5,6;7,8,9\na;b;c": "
1,2,34,5,67,8,9
abc
", + "\"1,2,3,4\";\"a\nb\"\nc;d": "
1,2,3,4a\nb
cd
", + "
": "
<br/>
", } for k, v := range kases { diff --git a/public/css/index.css b/public/css/index.css index b19b85ad3..6b906cc4a 100644 --- a/public/css/index.css +++ b/public/css/index.css @@ -489,6 +489,7 @@ footer .ui.left,footer .ui.right{line-height:40px} .repository.file.list .non-diff-file-content .view-raw img{padding:5px 5px 0 5px} .repository.file.list .non-diff-file-content .plain-text{padding:1em 2em 1em 2em} .repository.file.list .non-diff-file-content .plain-text pre{word-break:break-word;white-space:pre-wrap} +.repository.file.list .non-diff-file-content .csv{overflow-x:auto} .repository.file.list .non-diff-file-content pre{overflow:auto} .repository.file.list .sidebar{padding-left:0} .repository.file.list .sidebar .octicon{width:16px} diff --git a/public/less/_repository.less b/public/less/_repository.less index eb183c162..ef05beb6f 100644 --- a/public/less/_repository.less +++ b/public/less/_repository.less @@ -400,6 +400,10 @@ } } + .csv { + overflow-x: auto; + } + pre { overflow: auto; } diff --git a/routers/repo/view.go b/routers/repo/view.go index b534ae0e1..9d24506b4 100644 --- a/routers/repo/view.go +++ b/routers/repo/view.go @@ -162,8 +162,9 @@ func renderDirectory(ctx *context.Context, treeLink string) { d, _ := ioutil.ReadAll(dataRc) buf = charset.ToUTF8WithFallback(append(buf, d...)) - if markup.Type(readmeFile.Name()) != "" { + if markupType := markup.Type(readmeFile.Name()); markupType != "" { ctx.Data["IsMarkup"] = true + ctx.Data["MarkupType"] = string(markupType) ctx.Data["FileContent"] = string(markup.Render(readmeFile.Name(), buf, treeLink, ctx.Repo.Repository.ComposeMetas())) } else { ctx.Data["IsRenderedHTML"] = true @@ -282,8 +283,9 @@ func renderFile(ctx *context.Context, entry *git.TreeEntry, treeLink, rawLink st readmeExist := markup.IsReadmeFile(blob.Name()) ctx.Data["ReadmeExist"] = readmeExist - if markup.Type(blob.Name()) != "" { + if markupType := markup.Type(blob.Name()); markupType != "" { ctx.Data["IsMarkup"] = true + ctx.Data["MarkupType"] = markupType ctx.Data["FileContent"] = string(markup.Render(blob.Name(), buf, path.Dir(treeLink), ctx.Repo.Repository.ComposeMetas())) } else if readmeExist { ctx.Data["IsRenderedHTML"] = true diff --git a/templates/repo/view_file.tmpl b/templates/repo/view_file.tmpl index 72b1ae7a8..895a72aae 100644 --- a/templates/repo/view_file.tmpl +++ b/templates/repo/view_file.tmpl @@ -45,7 +45,7 @@
-
+
{{if .IsMarkup}} {{if .FileContent}}{{.FileContent | Safe}}{{end}} {{else if .IsRenderedHTML}}