From 44637f03ccef3f005da41da5d1880df80cceebb2 Mon Sep 17 00:00:00 2001 From: Unknwon Date: Sun, 27 Dec 2015 17:02:36 -0500 Subject: [PATCH] #2282 fast detection of utf-8 --- gogs.go | 2 +- modules/base/tool.go | 6 ++++++ modules/template/template.go | 5 ++--- templates/.VERSION | 2 +- 4 files changed, 10 insertions(+), 5 deletions(-) diff --git a/gogs.go b/gogs.go index 46d7e3386..2a424d26c 100644 --- a/gogs.go +++ b/gogs.go @@ -17,7 +17,7 @@ import ( "github.com/gogits/gogs/modules/setting" ) -const APP_VER = "0.8.13.1225" +const APP_VER = "0.8.13.1227" func init() { runtime.GOMAXPROCS(runtime.NumCPU()) diff --git a/modules/base/tool.go b/modules/base/tool.go index 75f475fa4..5927dad01 100644 --- a/modules/base/tool.go +++ b/modules/base/tool.go @@ -18,6 +18,7 @@ import ( "regexp" "strings" "time" + "unicode/utf8" "github.com/Unknwon/com" "github.com/Unknwon/i18n" @@ -53,6 +54,11 @@ func ShortSha(sha1 string) string { } func DetectEncoding(content []byte) string { + if utf8.Valid(content[:1024]) { + log.Debug("Detected encoding: utf-8 (fast)") + return "utf-8" + } + _, name, certain := charset.DetermineEncoding(content, "") if name != "utf-8" && len(setting.Repository.AnsiCharset) > 0 { log.Debug("Using default AnsiCharset: %s", setting.Repository.AnsiCharset) diff --git a/modules/template/template.go b/modules/template/template.go index 4149de451..6c070b700 100644 --- a/modules/template/template.go +++ b/modules/template/template.go @@ -12,7 +12,6 @@ import ( "runtime" "strings" "time" - "unicode/utf8" "golang.org/x/net/html/charset" "golang.org/x/text/transform" @@ -131,11 +130,11 @@ func Sha1(str string) string { } func ToUtf8WithErr(content []byte) (error, string) { - if utf8.Valid(content[:1024]) { + charsetLabel := base.DetectEncoding(content) + if charsetLabel == "utf-8" { return nil, string(content) } - charsetLabel := base.DetectEncoding(content) encoding, _ := charset.Lookup(charsetLabel) if encoding == nil { return fmt.Errorf("Unknown encoding: %s", charsetLabel), string(content) diff --git a/templates/.VERSION b/templates/.VERSION index e0c3ee2f2..cc6aa49ac 100644 --- a/templates/.VERSION +++ b/templates/.VERSION @@ -1 +1 @@ -0.8.13.1225 \ No newline at end of file +0.8.13.1227 \ No newline at end of file