From ff4e292b3fe49ee3086f45de1717301981e5f5f2 Mon Sep 17 00:00:00 2001 From: KN4CK3R Date: Fri, 25 Nov 2022 12:46:28 +0100 Subject: [PATCH] Add support for HEAD requests in Maven registry (#21834) (#21929) Backport of #21834 Co-authored-by: Lunny Xiao --- modules/context/context.go | 18 +++++-- routers/api/packages/api.go | 1 + routers/api/packages/composer/composer.go | 5 +- routers/api/packages/conan/conan.go | 5 +- routers/api/packages/generic/generic.go | 5 +- routers/api/packages/helm/helm.go | 5 +- routers/api/packages/maven/api.go | 7 +-- routers/api/packages/maven/maven.go | 57 ++++++++++++++++++-- routers/api/packages/npm/npm.go | 10 +++- routers/api/packages/nuget/nuget.go | 10 +++- routers/api/packages/pub/pub.go | 5 +- routers/api/packages/pypi/pypi.go | 5 +- routers/api/packages/rubygems/rubygems.go | 5 +- routers/api/packages/vagrant/vagrant.go | 5 +- routers/api/v1/repo/file.go | 6 ++- routers/common/repo.go | 13 +++-- routers/web/repo/repo.go | 5 +- routers/web/user/package.go | 5 +- tests/integration/api_packages_maven_test.go | 30 ++++++++++- 19 files changed, 162 insertions(+), 40 deletions(-) diff --git a/modules/context/context.go b/modules/context/context.go index 697eb7690..47368bb28 100644 --- a/modules/context/context.go +++ b/modules/context/context.go @@ -349,9 +349,11 @@ func (ctx *Context) RespHeader() http.Header { type ServeHeaderOptions struct { ContentType string // defaults to "application/octet-stream" ContentTypeCharset string + ContentLength *int64 Disposition string // defaults to "attachment" Filename string CacheDuration time.Duration // defaults to 5 minutes + LastModified time.Time } // SetServeHeaders sets necessary content serve headers @@ -369,6 +371,10 @@ func (ctx *Context) SetServeHeaders(opts *ServeHeaderOptions) { header.Set("Content-Type", contentType) header.Set("X-Content-Type-Options", "nosniff") + if opts.ContentLength != nil { + header.Set("Content-Length", strconv.FormatInt(*opts.ContentLength, 10)) + } + if opts.Filename != "" { disposition := opts.Disposition if disposition == "" { @@ -385,14 +391,16 @@ func (ctx *Context) SetServeHeaders(opts *ServeHeaderOptions) { duration = 5 * time.Minute } httpcache.AddCacheControlToHeader(header, duration) + + if !opts.LastModified.IsZero() { + header.Set("Last-Modified", opts.LastModified.UTC().Format(http.TimeFormat)) + } } // ServeContent serves content to http request -func (ctx *Context) ServeContent(name string, r io.ReadSeeker, modTime time.Time) { - ctx.SetServeHeaders(&ServeHeaderOptions{ - Filename: name, - }) - http.ServeContent(ctx.Resp, ctx.Req, name, modTime, r) +func (ctx *Context) ServeContent(r io.ReadSeeker, opts *ServeHeaderOptions) { + ctx.SetServeHeaders(opts) + http.ServeContent(ctx.Resp, ctx.Req, opts.Filename, opts.LastModified, r) } // UploadStream returns the request body or the first form file diff --git a/routers/api/packages/api.go b/routers/api/packages/api.go index 6f53bc4ae..2b3d7de3d 100644 --- a/routers/api/packages/api.go +++ b/routers/api/packages/api.go @@ -179,6 +179,7 @@ func Routes(ctx gocontext.Context) *web.Route { r.Group("/maven", func() { r.Put("/*", reqPackageAccess(perm.AccessModeWrite), maven.UploadPackageFile) r.Get("/*", maven.DownloadPackageFile) + r.Head("/*", maven.ProvidePackageFileHeader) }, reqPackageAccess(perm.AccessModeRead)) r.Group("/nuget", func() { r.Group("", func() { // Needs to be unauthenticated for the NuGet client. diff --git a/routers/api/packages/composer/composer.go b/routers/api/packages/composer/composer.go index 86ef7cbd9..476a2c236 100644 --- a/routers/api/packages/composer/composer.go +++ b/routers/api/packages/composer/composer.go @@ -184,7 +184,10 @@ func DownloadPackageFile(ctx *context.Context) { } defer s.Close() - ctx.ServeContent(pf.Name, s, pf.CreatedUnix.AsLocalTime()) + ctx.ServeContent(s, &context.ServeHeaderOptions{ + Filename: pf.Name, + LastModified: pf.CreatedUnix.AsLocalTime(), + }) } // UploadPackage creates a new package diff --git a/routers/api/packages/conan/conan.go b/routers/api/packages/conan/conan.go index dd078d6ad..ac99a48e9 100644 --- a/routers/api/packages/conan/conan.go +++ b/routers/api/packages/conan/conan.go @@ -473,7 +473,10 @@ func downloadFile(ctx *context.Context, fileFilter container.Set[string], fileKe } defer s.Close() - ctx.ServeContent(pf.Name, s, pf.CreatedUnix.AsLocalTime()) + ctx.ServeContent(s, &context.ServeHeaderOptions{ + Filename: pf.Name, + LastModified: pf.CreatedUnix.AsLocalTime(), + }) } // DeleteRecipeV1 deletes the requested recipe(s) diff --git a/routers/api/packages/generic/generic.go b/routers/api/packages/generic/generic.go index 81891bec2..f2bc1dc59 100644 --- a/routers/api/packages/generic/generic.go +++ b/routers/api/packages/generic/generic.go @@ -53,7 +53,10 @@ func DownloadPackageFile(ctx *context.Context) { } defer s.Close() - ctx.ServeContent(pf.Name, s, pf.CreatedUnix.AsLocalTime()) + ctx.ServeContent(s, &context.ServeHeaderOptions{ + Filename: pf.Name, + LastModified: pf.CreatedUnix.AsLocalTime(), + }) } // UploadPackage uploads the specific generic package. diff --git a/routers/api/packages/helm/helm.go b/routers/api/packages/helm/helm.go index 9c85e0874..43dafe629 100644 --- a/routers/api/packages/helm/helm.go +++ b/routers/api/packages/helm/helm.go @@ -138,7 +138,10 @@ func DownloadPackageFile(ctx *context.Context) { } defer s.Close() - ctx.ServeContent(pf.Name, s, pf.CreatedUnix.AsLocalTime()) + ctx.ServeContent(s, &context.ServeHeaderOptions{ + Filename: pf.Name, + LastModified: pf.CreatedUnix.AsLocalTime(), + }) } // UploadPackage creates a new package diff --git a/routers/api/packages/maven/api.go b/routers/api/packages/maven/api.go index b60a31781..4ca541dd6 100644 --- a/routers/api/packages/maven/api.go +++ b/routers/api/packages/maven/api.go @@ -6,7 +6,6 @@ package maven import ( "encoding/xml" - "sort" "strings" packages_model "code.gitea.io/gitea/models/packages" @@ -23,12 +22,8 @@ type MetadataResponse struct { Version []string `xml:"versioning>versions>version"` } +// pds is expected to be sorted ascending by CreatedUnix func createMetadataResponse(pds []*packages_model.PackageDescriptor) *MetadataResponse { - sort.Slice(pds, func(i, j int) bool { - // Maven and Gradle order packages by their creation timestamp and not by their version string - return pds[i].Version.CreatedUnix < pds[j].Version.CreatedUnix - }) - var release *packages_model.PackageDescriptor versions := make([]string, 0, len(pds)) diff --git a/routers/api/packages/maven/maven.go b/routers/api/packages/maven/maven.go index bf00c199f..de8832880 100644 --- a/routers/api/packages/maven/maven.go +++ b/routers/api/packages/maven/maven.go @@ -16,6 +16,8 @@ import ( "net/http" "path/filepath" "regexp" + "sort" + "strconv" "strings" packages_model "code.gitea.io/gitea/models/packages" @@ -34,6 +36,10 @@ const ( extensionSHA1 = ".sha1" extensionSHA256 = ".sha256" extensionSHA512 = ".sha512" + extensionPom = ".pom" + extensionJar = ".jar" + contentTypeJar = "application/java-archive" + contentTypeXML = "text/xml" ) var ( @@ -49,6 +55,15 @@ func apiError(ctx *context.Context, status int, obj interface{}) { // DownloadPackageFile serves the content of a package func DownloadPackageFile(ctx *context.Context) { + handlePackageFile(ctx, true) +} + +// ProvidePackageFileHeader provides only the headers describing a package +func ProvidePackageFileHeader(ctx *context.Context) { + handlePackageFile(ctx, false) +} + +func handlePackageFile(ctx *context.Context, serveContent bool) { params, err := extractPathParameters(ctx) if err != nil { apiError(ctx, http.StatusBadRequest, err) @@ -58,7 +73,7 @@ func DownloadPackageFile(ctx *context.Context) { if params.IsMeta && params.Version == "" { serveMavenMetadata(ctx, params) } else { - servePackageFile(ctx, params) + servePackageFile(ctx, params, serveContent) } } @@ -82,6 +97,11 @@ func serveMavenMetadata(ctx *context.Context, params parameters) { return } + sort.Slice(pds, func(i, j int) bool { + // Maven and Gradle order packages by their creation timestamp and not by their version string + return pds[i].Version.CreatedUnix < pds[j].Version.CreatedUnix + }) + xmlMetadata, err := xml.Marshal(createMetadataResponse(pds)) if err != nil { apiError(ctx, http.StatusInternalServerError, err) @@ -89,6 +109,9 @@ func serveMavenMetadata(ctx *context.Context, params parameters) { } xmlMetadataWithHeader := append([]byte(xml.Header), xmlMetadata...) + latest := pds[len(pds)-1] + ctx.Resp.Header().Set("Last-Modified", latest.Version.CreatedUnix.Format(http.TimeFormat)) + ext := strings.ToLower(filepath.Ext(params.Filename)) if isChecksumExtension(ext) { var hash []byte @@ -110,10 +133,15 @@ func serveMavenMetadata(ctx *context.Context, params parameters) { return } - ctx.PlainTextBytes(http.StatusOK, xmlMetadataWithHeader) + ctx.Resp.Header().Set("Content-Length", strconv.Itoa(len(xmlMetadataWithHeader))) + ctx.Resp.Header().Set("Content-Type", contentTypeXML) + + if _, err := ctx.Resp.Write(xmlMetadataWithHeader); err != nil { + log.Error("write bytes failed: %v", err) + } } -func servePackageFile(ctx *context.Context, params parameters) { +func servePackageFile(ctx *context.Context, params parameters, serveContent bool) { packageName := params.GroupID + "-" + params.ArtifactID pv, err := packages_model.GetVersionByNameAndVersion(ctx, ctx.Package.Owner.ID, packages_model.TypeMaven, packageName, params.Version) @@ -165,6 +193,23 @@ func servePackageFile(ctx *context.Context, params parameters) { return } + opts := &context.ServeHeaderOptions{ + ContentLength: &pb.Size, + LastModified: pf.CreatedUnix.AsLocalTime(), + } + switch ext { + case extensionJar: + opts.ContentType = contentTypeJar + case extensionPom: + opts.ContentType = contentTypeXML + } + + if !serveContent { + ctx.SetServeHeaders(opts) + ctx.Status(http.StatusOK) + return + } + s, err := packages_module.NewContentStore().Get(packages_module.BlobHash256Key(pb.HashSHA256)) if err != nil { apiError(ctx, http.StatusInternalServerError, err) @@ -177,7 +222,9 @@ func servePackageFile(ctx *context.Context, params parameters) { } } - ctx.ServeContent(pf.Name, s, pf.CreatedUnix.AsLocalTime()) + opts.Filename = pf.Name + + ctx.ServeContent(s, opts) } // UploadPackageFile adds a file to the package. If the package does not exist, it gets created. @@ -272,7 +319,7 @@ func UploadPackageFile(ctx *context.Context) { } // If it's the package pom file extract the metadata - if ext == ".pom" { + if ext == extensionPom { pfci.IsLead = true var err error diff --git a/routers/api/packages/npm/npm.go b/routers/api/packages/npm/npm.go index 82dae0cf4..a9676b919 100644 --- a/routers/api/packages/npm/npm.go +++ b/routers/api/packages/npm/npm.go @@ -103,7 +103,10 @@ func DownloadPackageFile(ctx *context.Context) { } defer s.Close() - ctx.ServeContent(pf.Name, s, pf.CreatedUnix.AsLocalTime()) + ctx.ServeContent(s, &context.ServeHeaderOptions{ + Filename: pf.Name, + LastModified: pf.CreatedUnix.AsLocalTime(), + }) } // DownloadPackageFileByName finds the version and serves the contents of a package @@ -146,7 +149,10 @@ func DownloadPackageFileByName(ctx *context.Context) { } defer s.Close() - ctx.ServeContent(pf.Name, s, pf.CreatedUnix.AsLocalTime()) + ctx.ServeContent(s, &context.ServeHeaderOptions{ + Filename: pf.Name, + LastModified: pf.CreatedUnix.AsLocalTime(), + }) } // UploadPackage creates a new package diff --git a/routers/api/packages/nuget/nuget.go b/routers/api/packages/nuget/nuget.go index e84aef316..5267bca8f 100644 --- a/routers/api/packages/nuget/nuget.go +++ b/routers/api/packages/nuget/nuget.go @@ -342,7 +342,10 @@ func DownloadPackageFile(ctx *context.Context) { } defer s.Close() - ctx.ServeContent(pf.Name, s, pf.CreatedUnix.AsLocalTime()) + ctx.ServeContent(s, &context.ServeHeaderOptions{ + Filename: pf.Name, + LastModified: pf.CreatedUnix.AsLocalTime(), + }) } // UploadPackage creates a new package with the metadata contained in the uploaded nupgk file @@ -552,7 +555,10 @@ func DownloadSymbolFile(ctx *context.Context) { } defer s.Close() - ctx.ServeContent(pf.Name, s, pf.CreatedUnix.AsLocalTime()) + ctx.ServeContent(s, &context.ServeHeaderOptions{ + Filename: pf.Name, + LastModified: pf.CreatedUnix.AsLocalTime(), + }) } // DeletePackage hard deletes the package diff --git a/routers/api/packages/pub/pub.go b/routers/api/packages/pub/pub.go index 9af0ceeb0..672669c98 100644 --- a/routers/api/packages/pub/pub.go +++ b/routers/api/packages/pub/pub.go @@ -271,5 +271,8 @@ func DownloadPackageFile(ctx *context.Context) { } defer s.Close() - ctx.ServeContent(pf.Name, s, pf.CreatedUnix.AsLocalTime()) + ctx.ServeContent(s, &context.ServeHeaderOptions{ + Filename: pf.Name, + LastModified: pf.CreatedUnix.AsLocalTime(), + }) } diff --git a/routers/api/packages/pypi/pypi.go b/routers/api/packages/pypi/pypi.go index 4c8041c30..2063fc37c 100644 --- a/routers/api/packages/pypi/pypi.go +++ b/routers/api/packages/pypi/pypi.go @@ -95,7 +95,10 @@ func DownloadPackageFile(ctx *context.Context) { } defer s.Close() - ctx.ServeContent(pf.Name, s, pf.CreatedUnix.AsLocalTime()) + ctx.ServeContent(s, &context.ServeHeaderOptions{ + Filename: pf.Name, + LastModified: pf.CreatedUnix.AsLocalTime(), + }) } // UploadPackageFile adds a file to the package. If the package does not exist, it gets created. diff --git a/routers/api/packages/rubygems/rubygems.go b/routers/api/packages/rubygems/rubygems.go index a13810ccd..d83120ca2 100644 --- a/routers/api/packages/rubygems/rubygems.go +++ b/routers/api/packages/rubygems/rubygems.go @@ -192,7 +192,10 @@ func DownloadPackageFile(ctx *context.Context) { } defer s.Close() - ctx.ServeContent(pf.Name, s, pf.CreatedUnix.AsLocalTime()) + ctx.ServeContent(s, &context.ServeHeaderOptions{ + Filename: pf.Name, + LastModified: pf.CreatedUnix.AsLocalTime(), + }) } // UploadPackageFile adds a file to the package. If the package does not exist, it gets created. diff --git a/routers/api/packages/vagrant/vagrant.go b/routers/api/packages/vagrant/vagrant.go index 7750e5dc4..35643e315 100644 --- a/routers/api/packages/vagrant/vagrant.go +++ b/routers/api/packages/vagrant/vagrant.go @@ -235,5 +235,8 @@ func DownloadPackageFile(ctx *context.Context) { } defer s.Close() - ctx.ServeContent(pf.Name, s, pf.CreatedUnix.AsLocalTime()) + ctx.ServeContent(s, &context.ServeHeaderOptions{ + Filename: pf.Name, + LastModified: pf.CreatedUnix.AsLocalTime(), + }) } diff --git a/routers/api/v1/repo/file.go b/routers/api/v1/repo/file.go index 6dead81e6..aba5b1f9e 100644 --- a/routers/api/v1/repo/file.go +++ b/routers/api/v1/repo/file.go @@ -341,7 +341,11 @@ func download(ctx *context.APIContext, archiveName string, archiver *repo_model. return } defer fr.Close() - ctx.ServeContent(downloadName, fr, archiver.CreatedUnix.AsLocalTime()) + + ctx.ServeContent(fr, &context.ServeHeaderOptions{ + Filename: downloadName, + LastModified: archiver.CreatedUnix.AsLocalTime(), + }) } // GetEditorconfig get editor config of a repository diff --git a/routers/common/repo.go b/routers/common/repo.go index f4b813d6b..340eb1809 100644 --- a/routers/common/repo.go +++ b/routers/common/repo.go @@ -5,7 +5,6 @@ package common import ( - "fmt" "io" "path" "path/filepath" @@ -52,16 +51,16 @@ func ServeData(ctx *context.Context, filePath string, size int64, reader io.Read buf = buf[:n] } - if size >= 0 { - ctx.Resp.Header().Set("Content-Length", fmt.Sprintf("%d", size)) - } else { - log.Error("ServeData called to serve data: %s with size < 0: %d", filePath, size) - } - opts := &context.ServeHeaderOptions{ Filename: path.Base(filePath), } + if size >= 0 { + opts.ContentLength = &size + } else { + log.Error("ServeData called to serve data: %s with size < 0: %d", filePath, size) + } + sniffedType := typesniffer.DetectContentType(buf) isPlain := sniffedType.IsText() || ctx.FormBool("render") diff --git a/routers/web/repo/repo.go b/routers/web/repo/repo.go index 3e746d3f0..0bad3f5de 100644 --- a/routers/web/repo/repo.go +++ b/routers/web/repo/repo.go @@ -426,7 +426,10 @@ func download(ctx *context.Context, archiveName string, archiver *repo_model.Rep } defer fr.Close() - ctx.ServeContent(downloadName, fr, archiver.CreatedUnix.AsLocalTime()) + ctx.ServeContent(fr, &context.ServeHeaderOptions{ + Filename: downloadName, + LastModified: archiver.CreatedUnix.AsLocalTime(), + }) } // InitiateDownload will enqueue an archival request, as needed. It may submit diff --git a/routers/web/user/package.go b/routers/web/user/package.go index 7179e2df9..7be37b6a5 100644 --- a/routers/web/user/package.go +++ b/routers/web/user/package.go @@ -402,5 +402,8 @@ func DownloadPackageFile(ctx *context.Context) { } defer s.Close() - ctx.ServeContent(pf.Name, s, pf.CreatedUnix.AsLocalTime()) + ctx.ServeContent(s, &context.ServeHeaderOptions{ + Filename: pf.Name, + LastModified: pf.CreatedUnix.AsLocalTime(), + }) } diff --git a/tests/integration/api_packages_maven_test.go b/tests/integration/api_packages_maven_test.go index 87d95557c..e71e1ff03 100644 --- a/tests/integration/api_packages_maven_test.go +++ b/tests/integration/api_packages_maven_test.go @@ -7,6 +7,7 @@ package integration import ( "fmt" "net/http" + "strconv" "strings" "testing" @@ -39,6 +40,12 @@ func TestPackageMaven(t *testing.T) { MakeRequest(t, req, expectedStatus) } + checkHeaders := func(t *testing.T, h http.Header, contentType string, contentLength int64) { + assert.Equal(t, contentType, h.Get("Content-Type")) + assert.Equal(t, strconv.FormatInt(contentLength, 10), h.Get("Content-Length")) + assert.NotEmpty(t, h.Get("Last-Modified")) + } + t.Run("Upload", func(t *testing.T) { defer tests.PrintCurrentTest(t)() @@ -77,10 +84,18 @@ func TestPackageMaven(t *testing.T) { t.Run("Download", func(t *testing.T) { defer tests.PrintCurrentTest(t)() - req := NewRequest(t, "GET", fmt.Sprintf("%s/%s/%s", root, packageVersion, filename)) + req := NewRequest(t, "HEAD", fmt.Sprintf("%s/%s/%s", root, packageVersion, filename)) req = AddBasicAuthHeader(req, user.Name) resp := MakeRequest(t, req, http.StatusOK) + checkHeaders(t, resp.Header(), "application/java-archive", 4) + + req = NewRequest(t, "GET", fmt.Sprintf("%s/%s/%s", root, packageVersion, filename)) + req = AddBasicAuthHeader(req, user.Name) + resp = MakeRequest(t, req, http.StatusOK) + + checkHeaders(t, resp.Header(), "application/java-archive", 4) + assert.Equal(t, []byte("test"), resp.Body.Bytes()) pvs, err := packages.GetVersionsByPackageType(db.DefaultContext, user.ID, packages.TypeMaven) @@ -150,10 +165,18 @@ func TestPackageMaven(t *testing.T) { t.Run("DownloadPOM", func(t *testing.T) { defer tests.PrintCurrentTest(t)() - req := NewRequest(t, "GET", fmt.Sprintf("%s/%s/%s.pom", root, packageVersion, filename)) + req := NewRequest(t, "HEAD", fmt.Sprintf("%s/%s/%s.pom", root, packageVersion, filename)) req = AddBasicAuthHeader(req, user.Name) resp := MakeRequest(t, req, http.StatusOK) + checkHeaders(t, resp.Header(), "text/xml", int64(len(pomContent))) + + req = NewRequest(t, "GET", fmt.Sprintf("%s/%s/%s.pom", root, packageVersion, filename)) + req = AddBasicAuthHeader(req, user.Name) + resp = MakeRequest(t, req, http.StatusOK) + + checkHeaders(t, resp.Header(), "text/xml", int64(len(pomContent))) + assert.Equal(t, []byte(pomContent), resp.Body.Bytes()) pvs, err := packages.GetVersionsByPackageType(db.DefaultContext, user.ID, packages.TypeMaven) @@ -191,6 +214,9 @@ func TestPackageMaven(t *testing.T) { resp := MakeRequest(t, req, http.StatusOK) expectedMetadata := `` + "\ncom.giteatest-project1.0.11.0.11.0.1" + + checkHeaders(t, resp.Header(), "text/xml", int64(len(expectedMetadata))) + assert.Equal(t, expectedMetadata, resp.Body.String()) for key, checksum := range map[string]string{