From 9f5ddca57c38f566fb18c9add339fc9f5aeb25de Mon Sep 17 00:00:00 2001 From: zeripath Date: Mon, 9 May 2022 16:54:51 +0100 Subject: [PATCH] Set the LastModified header for raw files (#18356) Although the use of LastModified dates for caching of git objects should be discouraged (as it is not native to git - and there are a LOT of ways this could be incorrect) - LastModified dates can be a helpful somewhat more human way of caching for simple cases. This PR adds this header and handles the If-Modified-Since header to the /raw/ routes. Fix #18354 Signed-off-by: Andrew Thornton Co-authored-by: 6543 <6543@obermui.de> --- modules/httpcache/httpcache.go | 39 ++++++++++++++++- routers/api/v1/repo/file.go | 43 +++++++++++++++++-- routers/common/repo.go | 5 ++- routers/web/repo/download.go | 76 ++++++++++++++++++++++++---------- routers/web/repo/wiki.go | 3 +- 5 files changed, 135 insertions(+), 31 deletions(-) diff --git a/modules/httpcache/httpcache.go b/modules/httpcache/httpcache.go index 11b63148d..5797e981c 100644 --- a/modules/httpcache/httpcache.go +++ b/modules/httpcache/httpcache.go @@ -37,18 +37,23 @@ func generateETag(fi os.FileInfo) string { // HandleTimeCache handles time-based caching for a HTTP request func HandleTimeCache(req *http.Request, w http.ResponseWriter, fi os.FileInfo) (handled bool) { + return HandleGenericTimeCache(req, w, fi.ModTime()) +} + +// HandleGenericTimeCache handles time-based caching for a HTTP request +func HandleGenericTimeCache(req *http.Request, w http.ResponseWriter, lastModified time.Time) (handled bool) { AddCacheControlToHeader(w.Header(), setting.StaticCacheTime) ifModifiedSince := req.Header.Get("If-Modified-Since") if ifModifiedSince != "" { t, err := time.Parse(http.TimeFormat, ifModifiedSince) - if err == nil && fi.ModTime().Unix() <= t.Unix() { + if err == nil && lastModified.Unix() <= t.Unix() { w.WriteHeader(http.StatusNotModified) return true } } - w.Header().Set("Last-Modified", fi.ModTime().Format(http.TimeFormat)) + w.Header().Set("Last-Modified", lastModified.Format(http.TimeFormat)) return false } @@ -85,3 +90,33 @@ func checkIfNoneMatchIsValid(req *http.Request, etag string) bool { } return false } + +// HandleGenericETagTimeCache handles ETag-based caching with Last-Modified caching for a HTTP request. +// It returns true if the request was handled. +func HandleGenericETagTimeCache(req *http.Request, w http.ResponseWriter, etag string, lastModified time.Time) (handled bool) { + if len(etag) > 0 { + w.Header().Set("Etag", etag) + } + if !lastModified.IsZero() { + w.Header().Set("Last-Modified", lastModified.Format(http.TimeFormat)) + } + + if len(etag) > 0 { + if checkIfNoneMatchIsValid(req, etag) { + w.WriteHeader(http.StatusNotModified) + return true + } + } + if !lastModified.IsZero() { + ifModifiedSince := req.Header.Get("If-Modified-Since") + if ifModifiedSince != "" { + t, err := time.Parse(http.TimeFormat, ifModifiedSince) + if err == nil && lastModified.Unix() <= t.Unix() { + w.WriteHeader(http.StatusNotModified) + return true + } + } + } + AddCacheControlToHeader(w.Header(), setting.StaticCacheTime) + return false +} diff --git a/routers/api/v1/repo/file.go b/routers/api/v1/repo/file.go index b8b72f95e..2a4c4ad97 100644 --- a/routers/api/v1/repo/file.go +++ b/routers/api/v1/repo/file.go @@ -9,13 +9,16 @@ import ( "encoding/base64" "fmt" "net/http" + "path" "time" "code.gitea.io/gitea/models" repo_model "code.gitea.io/gitea/models/repo" "code.gitea.io/gitea/models/unit" + "code.gitea.io/gitea/modules/cache" "code.gitea.io/gitea/modules/context" "code.gitea.io/gitea/modules/git" + "code.gitea.io/gitea/modules/setting" api "code.gitea.io/gitea/modules/structs" "code.gitea.io/gitea/modules/web" "code.gitea.io/gitea/routers/common" @@ -62,18 +65,50 @@ func GetRawFile(ctx *context.APIContext) { return } - blob, err := ctx.Repo.Commit.GetBlobByPath(ctx.Repo.TreePath) + blob, lastModified := getBlobForEntry(ctx) + if ctx.Written() { + return + } + + if err := common.ServeBlob(ctx.Context, blob, lastModified); err != nil { + ctx.Error(http.StatusInternalServerError, "ServeBlob", err) + } +} + +func getBlobForEntry(ctx *context.APIContext) (blob *git.Blob, lastModified time.Time) { + entry, err := ctx.Repo.Commit.GetTreeEntryByPath(ctx.Repo.TreePath) if err != nil { if git.IsErrNotExist(err) { ctx.NotFound() } else { - ctx.Error(http.StatusInternalServerError, "GetBlobByPath", err) + ctx.Error(http.StatusInternalServerError, "GetTreeEntryByPath", err) } return } - if err = common.ServeBlob(ctx.Context, blob); err != nil { - ctx.Error(http.StatusInternalServerError, "ServeBlob", err) + + if entry.IsDir() || entry.IsSubModule() { + ctx.NotFound("getBlobForEntry", nil) + return } + + var c *git.LastCommitCache + if setting.CacheService.LastCommit.Enabled && ctx.Repo.CommitsCount >= setting.CacheService.LastCommit.CommitsCount { + c = git.NewLastCommitCache(ctx.Repo.Repository.FullName(), ctx.Repo.GitRepo, setting.LastCommitCacheTTLSeconds, cache.GetCache()) + } + + info, _, err := git.Entries([]*git.TreeEntry{entry}).GetCommitsInfo(ctx, ctx.Repo.Commit, path.Dir("/" + ctx.Repo.TreePath)[1:], c) + if err != nil { + ctx.Error(http.StatusInternalServerError, "GetCommitsInfo", err) + return + } + + if len(info) == 1 { + // Not Modified + lastModified = info[0].Commit.Committer.When + } + blob = entry.Blob() + + return } // GetArchive get archive of a repository diff --git a/routers/common/repo.go b/routers/common/repo.go index b0e14b63f..d037e151f 100644 --- a/routers/common/repo.go +++ b/routers/common/repo.go @@ -10,6 +10,7 @@ import ( "path" "path/filepath" "strings" + "time" "code.gitea.io/gitea/modules/charset" "code.gitea.io/gitea/modules/context" @@ -22,8 +23,8 @@ import ( ) // ServeBlob download a git.Blob -func ServeBlob(ctx *context.Context, blob *git.Blob) error { - if httpcache.HandleGenericETagCache(ctx.Req, ctx.Resp, `"`+blob.ID.String()+`"`) { +func ServeBlob(ctx *context.Context, blob *git.Blob, lastModified time.Time) error { + if httpcache.HandleGenericETagTimeCache(ctx.Req, ctx.Resp, `"`+blob.ID.String()+`"`, lastModified) { return nil } diff --git a/routers/web/repo/download.go b/routers/web/repo/download.go index 72d34cb93..eae61bb8e 100644 --- a/routers/web/repo/download.go +++ b/routers/web/repo/download.go @@ -6,7 +6,11 @@ package repo import ( + "path" + "time" + "code.gitea.io/gitea/models" + "code.gitea.io/gitea/modules/cache" "code.gitea.io/gitea/modules/context" "code.gitea.io/gitea/modules/git" "code.gitea.io/gitea/modules/httpcache" @@ -18,8 +22,8 @@ import ( ) // ServeBlobOrLFS download a git.Blob redirecting to LFS if necessary -func ServeBlobOrLFS(ctx *context.Context, blob *git.Blob) error { - if httpcache.HandleGenericETagCache(ctx.Req, ctx.Resp, `"`+blob.ID.String()+`"`) { +func ServeBlobOrLFS(ctx *context.Context, blob *git.Blob, lastModified time.Time) error { + if httpcache.HandleGenericETagTimeCache(ctx.Req, ctx.Resp, `"`+blob.ID.String()+`"`, lastModified) { return nil } @@ -45,7 +49,7 @@ func ServeBlobOrLFS(ctx *context.Context, blob *git.Blob) error { log.Error("ServeBlobOrLFS: Close: %v", err) } closed = true - return common.ServeBlob(ctx, blob) + return common.ServeBlob(ctx, blob, lastModified) } if httpcache.HandleGenericETagCache(ctx.Req, ctx.Resp, `"`+pointer.Oid+`"`) { return nil @@ -76,37 +80,65 @@ func ServeBlobOrLFS(ctx *context.Context, blob *git.Blob) error { } closed = true - return common.ServeBlob(ctx, blob) + return common.ServeBlob(ctx, blob, lastModified) +} + +func getBlobForEntry(ctx *context.Context) (blob *git.Blob, lastModified time.Time) { + entry, err := ctx.Repo.Commit.GetTreeEntryByPath(ctx.Repo.TreePath) + if err != nil { + if git.IsErrNotExist(err) { + ctx.NotFound("GetTreeEntryByPath", err) + } else { + ctx.ServerError("GetTreeEntryByPath", err) + } + return + } + + if entry.IsDir() || entry.IsSubModule() { + ctx.NotFound("getBlobForEntry", nil) + return + } + + var c *git.LastCommitCache + if setting.CacheService.LastCommit.Enabled && ctx.Repo.CommitsCount >= setting.CacheService.LastCommit.CommitsCount { + c = git.NewLastCommitCache(ctx.Repo.Repository.FullName(), ctx.Repo.GitRepo, setting.LastCommitCacheTTLSeconds, cache.GetCache()) + } + + info, _, err := git.Entries([]*git.TreeEntry{entry}).GetCommitsInfo(ctx, ctx.Repo.Commit, path.Dir("/" + ctx.Repo.TreePath)[1:], c) + if err != nil { + ctx.ServerError("GetCommitsInfo", err) + return + } + + if len(info) == 1 { + // Not Modified + lastModified = info[0].Commit.Committer.When + } + blob = entry.Blob() + + return } // SingleDownload download a file by repos path func SingleDownload(ctx *context.Context) { - blob, err := ctx.Repo.Commit.GetBlobByPath(ctx.Repo.TreePath) - if err != nil { - if git.IsErrNotExist(err) { - ctx.NotFound("GetBlobByPath", nil) - } else { - ctx.ServerError("GetBlobByPath", err) - } + blob, lastModified := getBlobForEntry(ctx) + if blob == nil { return } - if err = common.ServeBlob(ctx, blob); err != nil { + + if err := common.ServeBlob(ctx, blob, lastModified); err != nil { ctx.ServerError("ServeBlob", err) } } // SingleDownloadOrLFS download a file by repos path redirecting to LFS if necessary func SingleDownloadOrLFS(ctx *context.Context) { - blob, err := ctx.Repo.Commit.GetBlobByPath(ctx.Repo.TreePath) - if err != nil { - if git.IsErrNotExist(err) { - ctx.NotFound("GetBlobByPath", nil) - } else { - ctx.ServerError("GetBlobByPath", err) - } + blob, lastModified := getBlobForEntry(ctx) + if blob == nil { return } - if err = ServeBlobOrLFS(ctx, blob); err != nil { + + if err := ServeBlobOrLFS(ctx, blob, lastModified); err != nil { ctx.ServerError("ServeBlobOrLFS", err) } } @@ -122,7 +154,7 @@ func DownloadByID(ctx *context.Context) { } return } - if err = common.ServeBlob(ctx, blob); err != nil { + if err = common.ServeBlob(ctx, blob, time.Time{}); err != nil { ctx.ServerError("ServeBlob", err) } } @@ -138,7 +170,7 @@ func DownloadByIDOrLFS(ctx *context.Context) { } return } - if err = ServeBlobOrLFS(ctx, blob); err != nil { + if err = ServeBlobOrLFS(ctx, blob, time.Time{}); err != nil { ctx.ServerError("ServeBlob", err) } } diff --git a/routers/web/repo/wiki.go b/routers/web/repo/wiki.go index 8bdb6fbc0..77f60a1df 100644 --- a/routers/web/repo/wiki.go +++ b/routers/web/repo/wiki.go @@ -13,6 +13,7 @@ import ( "net/url" "path/filepath" "strings" + "time" "code.gitea.io/gitea/models" "code.gitea.io/gitea/models/unit" @@ -627,7 +628,7 @@ func WikiRaw(ctx *context.Context) { } if entry != nil { - if err = common.ServeBlob(ctx, entry.Blob()); err != nil { + if err = common.ServeBlob(ctx, entry.Blob(), time.Time{}); err != nil { ctx.ServerError("ServeBlob", err) } return