From 19ae6439b0956a578100b50cb09b1cbd40a01942 Mon Sep 17 00:00:00 2001 From: Lunny Xiao Date: Thu, 24 Dec 2020 12:25:17 +0800 Subject: [PATCH] Improve vfsgen to not unzip bindata files but send to browser directly (#7109) * Don't unzip files from bindata but send to browser directly * remove dependent for httpgzip * Add tests for parseAcceptEncoding * Update docs for ENABLE_GZIP * Fix bug * Fix bug Co-authored-by: zeripath --- custom/conf/app.example.ini | 2 +- .../doc/advanced/config-cheat-sheet.en-us.md | 2 +- .../doc/advanced/config-cheat-sheet.zh-cn.md | 2 +- modules/public/dynamic.go | 12 +++++- modules/public/public.go | 12 +++++- modules/public/public_test.go | 40 +++++++++++++++++++ modules/public/static.go | 40 +++++++++++++++++++ 7 files changed, 105 insertions(+), 5 deletions(-) create mode 100644 modules/public/public_test.go diff --git a/custom/conf/app.example.ini b/custom/conf/app.example.ini index b89bbf894e..e26e9e4d56 100644 --- a/custom/conf/app.example.ini +++ b/custom/conf/app.example.ini @@ -361,7 +361,7 @@ KEY_FILE = https/key.pem STATIC_ROOT_PATH = ; Default path for App data APP_DATA_PATH = data -; Application level GZIP support +; Enable gzip compression for runtime-generated content, static resources excluded ENABLE_GZIP = false ; Application profiling (memory and cpu) ; For "web" command it listens on localhost:6060 diff --git a/docs/content/doc/advanced/config-cheat-sheet.en-us.md b/docs/content/doc/advanced/config-cheat-sheet.en-us.md index d482523f79..43f42b95e0 100644 --- a/docs/content/doc/advanced/config-cheat-sheet.en-us.md +++ b/docs/content/doc/advanced/config-cheat-sheet.en-us.md @@ -264,7 +264,7 @@ Values containing `#` or `;` must be quoted using `` ` `` or `"""`. - `STATIC_ROOT_PATH`: **./**: Upper level of template and static files path. - `APP_DATA_PATH`: **data** (**/data/gitea** on docker): Default path for application data. - `STATIC_CACHE_TIME`: **6h**: Web browser cache time for static resources on `custom/`, `public/` and all uploaded avatars. Note that this cache is disabled when `RUN_MODE` is "dev". -- `ENABLE_GZIP`: **false**: Enables application-level GZIP support. +- `ENABLE_GZIP`: **false**: Enable gzip compression for runtime-generated content, static resources excluded. - `ENABLE_PPROF`: **false**: Application profiling (memory and cpu). For "web" command it listens on localhost:6060. For "serv" command it dumps to disk at `PPROF_DATA_PATH` as `(cpuprofile|memprofile)__` - `PPROF_DATA_PATH`: **data/tmp/pprof**: `PPROF_DATA_PATH`, use an absolute path when you start gitea as service - `LANDING_PAGE`: **home**: Landing page for unauthenticated users \[home, explore, organizations, login\]. diff --git a/docs/content/doc/advanced/config-cheat-sheet.zh-cn.md b/docs/content/doc/advanced/config-cheat-sheet.zh-cn.md index c1f7e836c0..da2d02c11d 100644 --- a/docs/content/doc/advanced/config-cheat-sheet.zh-cn.md +++ b/docs/content/doc/advanced/config-cheat-sheet.zh-cn.md @@ -70,7 +70,7 @@ menu: - `KEY_FILE`: 启用HTTPS的密钥文件。 - `STATIC_ROOT_PATH`: 存放模板和静态文件的根目录，默认是 Gitea 的根目录。 - `STATIC_CACHE_TIME`: **6h**: 静态资源文件，包括 `custom/`, `public/` 和所有上传的头像的浏览器缓存时间。 -- `ENABLE_GZIP`: 启用应用级别的 GZIP 压缩。 +- `ENABLE_GZIP`: 启用实时生成的数据启用 GZIP 压缩，不包括静态资源。 - `LANDING_PAGE`: 未登录用户的默认页面，可选 `home` 或 `explore`。 - `LFS_START_SERVER`: 是否启用 git-lfs 支持. 可以为 `true` 或 `false`， 默认是 `false`。 diff --git a/modules/public/dynamic.go b/modules/public/dynamic.go index f1a4dbb1a3..f634c598a3 100644 --- a/modules/public/dynamic.go +++ b/modules/public/dynamic.go @@ -6,9 +6,19 @@ package public -import "net/http" +import ( + "io" + "net/http" + "os" + "time" +) // Static implements the macaron static handler for serving assets. func Static(opts *Options) func(next http.Handler) http.Handler { return opts.staticHandler(opts.Directory) } + +// ServeContent serve http content +func ServeContent(w http.ResponseWriter, req *http.Request, fi os.FileInfo, modtime time.Time, content io.ReadSeeker) { + http.ServeContent(w, req, fi.Name(), modtime, content) +} diff --git a/modules/public/public.go b/modules/public/public.go index fc933637d8..c8148e6db3 100644 --- a/modules/public/public.go +++ b/modules/public/public.go @@ -87,6 +87,16 @@ func (opts *Options) staticHandler(dir string) func(next http.Handler) http.Hand } } +// parseAcceptEncoding parse Accept-Encoding: deflate, gzip;q=1.0, *;q=0.5 as compress methods +func parseAcceptEncoding(val string) map[string]bool { + parts := strings.Split(val, ";") + var types = make(map[string]bool) + for _, v := range strings.Split(parts[0], ",") { + types[strings.TrimSpace(v)] = true + } + return types +} + func (opts *Options) handle(w http.ResponseWriter, req *http.Request, opt *Options) bool { if req.Method != "GET" && req.Method != "HEAD" { return false @@ -157,6 +167,6 @@ func (opts *Options) handle(w http.ResponseWriter, req *http.Request, opt *Optio return true } - http.ServeContent(w, req, file, fi.ModTime(), f) + ServeContent(w, req, fi, fi.ModTime(), f) return true } diff --git a/modules/public/public_test.go b/modules/public/public_test.go new file mode 100644 index 0000000000..cf8dced431 --- /dev/null +++ b/modules/public/public_test.go @@ -0,0 +1,40 @@ +// Copyright 2020 The Gitea Authors. All rights reserved. +// Use of this source code is governed by a MIT-style +// license that can be found in the LICENSE file. + +package public + +import ( + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestParseAcceptEncoding(t *testing.T) { + var kases = []struct { + Header string + Expected map[string]bool + }{ + { + Header: "deflate, gzip;q=1.0, *;q=0.5", + Expected: map[string]bool{ + "deflate": true, + "gzip": true, + }, + }, + { + Header: " gzip, deflate, br", + Expected: map[string]bool{ + "deflate": true, + "gzip": true, + "br": true, + }, + }, + } + + for _, kase := range kases { + t.Run(kase.Header, func(t *testing.T) { + assert.EqualValues(t, kase.Expected, parseAcceptEncoding(kase.Header)) + }) + } +} diff --git a/modules/public/static.go b/modules/public/static.go index 8da10567ea..c4dd7a1eca 100644 --- a/modules/public/static.go +++ b/modules/public/static.go @@ -7,8 +7,17 @@ package public import ( + "bytes" + "compress/gzip" + "io" "io/ioutil" + "mime" "net/http" + "os" + "path/filepath" + "time" + + "code.gitea.io/gitea/modules/log" ) // Static implements the macaron static handler for serving assets. @@ -49,3 +58,34 @@ func AssetIsDir(name string) (bool, error) { } } } + +// ServeContent serve http content +func ServeContent(w http.ResponseWriter, req *http.Request, fi os.FileInfo, modtime time.Time, content io.ReadSeeker) { + encodings := parseAcceptEncoding(req.Header.Get("Accept-Encoding")) + if encodings["gzip"] { + if cf, ok := fi.(*vfsgen۰CompressedFileInfo); ok { + rd := bytes.NewReader(cf.GzipBytes()) + w.Header().Set("Content-Encoding", "gzip") + ctype := mime.TypeByExtension(filepath.Ext(fi.Name())) + if ctype == "" { + // read a chunk to decide between utf-8 text and binary + var buf [512]byte + grd, _ := gzip.NewReader(rd) + n, _ := io.ReadFull(grd, buf[:]) + ctype = http.DetectContentType(buf[:n]) + _, err := rd.Seek(0, io.SeekStart) // rewind to output whole file + if err != nil { + log.Error("rd.Seek error: %v", err) + http.Error(w, http.StatusText(500), 500) + return + } + } + w.Header().Set("Content-Type", ctype) + http.ServeContent(w, req, fi.Name(), modtime, rd) + return + } + } + + http.ServeContent(w, req, fi.Name(), modtime, content) + return +}